def get_filesnames_recursively(self, root_path, *, file_ext='log', file_format='common_log'):
"""Setup file read processing."""
self.root_path = root_path
self.file_ext = file_ext
self.file_format = file_format
if file_format not in (self.ALLOWED_FILE_FORMATS.COMMON_LOG.value, self.ALLOWED_FILE_FORMATS.JSON.value):
raise FileFormatNotSupported("File format {} is not supported".format(file_format))
self.files = [filename for filename in Path(root_path).glob('**/*.{}'.format(self.file_ext))]
def read_file(self, filepath, storage_attribute):
"""Check if file is supported and loop parse each file."""
data = []
with open(filepath, "r") as fp:
if filepath.suffix == ".json":
data = json.load(fp)
elif filepath.suffix == ".log":
# Here we are loading in data from common log format Columns [0]= timestamp [1]=severity [2]=msg
for line in fp:
message_field = self.extract_message(line)
data.append({"message": message_field})
else:
raise FileFormatNotSupported(
"File format is not supported json and common log format (which ends with '.log') .")
if storage_attribute.false_data is not None:
data.extend(storage_attribute.false_data)
return data