def find_log_files(self, sp_key, filecontents=True, filehandles=False):
"""
Return matches log files of interest.
:param sp_key: Search pattern key specified in config
:param filehandles: Set to true to return a file handle instead of slurped file contents
:return: Yields a dict with filename (fn), root directory (root), cleaned sample name
generated from the filename (s_name) and either the file contents or file handle
for the current matched file (f).
As yield is used, the results can be iterated over without loading all files at once
"""
# Pick up path filters if specified.
# Allows modules to be called multiple times with different sets of files
path_filters = getattr(self, 'mod_cust_config', {}).get('path_filters')
# Old, depreciated syntax support. Likely to be removed in a future version.
if isinstance(sp_key, dict):
report.files[self.name] = list()
for sf in report.searchfiles:
if report.search_file(sp_key, {'fn': sf[0], 'root': sf[1]}):
report.files[self.name].append({'fn': sf[0], 'root': sf[1]})
sp_key = self.name
logwarn = "Depreciation Warning: {} - Please use new style for find_log_files()".format(self.name)
if len(report.files[self.name]) > 0:
logger.warn(logwarn)
else:
logger.debug(logwarn)
elif not isinstance(sp_key, str):
logger.warn("Did not understand find_log_files() search key")
return
for f in report.files[sp_key]:
# If path_filters is given, skip unless match
if path_filters is not None and len(path_filters) > 0:
if not all([ fnmatch.fnmatch(f['fn'], pf) for pf in path_filters ]):
logger.debug("{} - Skipping '{}' as didn't match module path filters".format(sp_key, f['fn']))
continue
# Make a note of the filename so that we can report it if something crashes
report.last_found_file = os.path.join(f['root'],f['fn'])
# Make a sample name from the filename
f['s_name'] = self.clean_s_name(f['fn'], f['root'])
if filehandles or filecontents:
try:
with io.open (os.path.join(f['root'],f['fn']), "r", encoding='utf-8') as fh:
if filehandles:
f['f'] = fh
yield f
elif filecontents:
f['f'] = fh.read()
yield f
except (IOError, OSError, ValueError, UnicodeDecodeError):
if config.report_readerrors:
logger.debug("Couldn't open filehandle when returning file: {}".format(f['fn']))
f['f'] = None
else:
yield f
def find_log_files(self, sp_key, filecontents=True, filehandles=False):
"""
Return matches log files of interest.
:param sp_key: Search pattern key specified in config
:param filehandles: Set to true to return a file handle instead of slurped file contents
:return: Yields a dict with filename (fn), root directory (root), cleaned sample name
generated from the filename (s_name) and either the file contents or file handle
for the current matched file (f).
As yield is used, the results can be iterated over without loading all files at once
"""
# Old, depreciated syntax support. Likely to be removed in a future version.
if isinstance(sp_key, dict):
report.files[self.name] = list()
for sf in report.searchfiles:
if report.search_file(sp_key, {'fn': sf[0], 'root': sf[1]}):
report.files[self.name].append({
'fn': sf[0],
'root': sf[1]
})
sp_key = self.name
logwarn = "Depreciation Warning: {} - Please use new style for find_log_files()".format(
self.name)
if len(report.files[self.name]) > 0:
logger.warn(logwarn)
else:
logger.debug(logwarn)
elif not isinstance(sp_key, str):
logger.warn("Did not understand find_log_files() search key")
return
for f in report.files[sp_key]:
# Make a sample name from the filename
f['s_name'] = self.clean_s_name(f['fn'], f['root'])
if filehandles or filecontents:
try:
with io.open(os.path.join(f['root'], f['fn']),
"r",
encoding='utf-8') as fh:
if filehandles:
f['f'] = fh
yield f
elif filecontents:
f['f'] = fh.read()
yield f
except (IOError, OSError, ValueError, UnicodeDecodeError):
if config.report_readerrors:
logger.debug(
"Couldn't open filehandle when returning file: {}".
format(f['fn']))
f['f'] = None
else:
yield f
def find_log_files(self, sp_key, filecontents=True, filehandles=False):
"""
Return matches log files of interest.
:param sp_key: Search pattern key specified in config
:param filehandles: Set to true to return a file handle instead of slurped file contents
:return: Yields a dict with filename (fn), root directory (root), cleaned sample name
generated from the filename (s_name) and either the file contents or file handle
for the current matched file (f).
As yield is used, the results can be iterated over without loading all files at once
"""
# Pick up path filters if specified.
# Allows modules to be called multiple times with different sets of files
path_filters = getattr(self, "mod_cust_config", {}).get("path_filters")
path_filters_exclude = getattr(self, "mod_cust_config",
{}).get("path_filters_exclude")
if type(path_filters) == str:
path_filters = [path_filters]
if type(path_filters_exclude) == str:
path_filters_exclude = [path_filters_exclude]
# Old, depreciated syntax support. Likely to be removed in a future version.
if isinstance(sp_key, dict):
report.files[self.name] = list()
for sf in report.searchfiles:
if report.search_file(sp_key, {
"fn": sf[0],
"root": sf[1]
},
module_key=None):
report.files[self.name].append({
"fn": sf[0],
"root": sf[1]
})
sp_key = self.name
logwarn = "Depreciation Warning: {} - Please use new style for find_log_files()".format(
self.name)
if len(report.files[self.name]) > 0:
logger.warning(logwarn)
else:
logger.debug(logwarn)
elif not isinstance(sp_key, str):
logger.warning("Did not understand find_log_files() search key")
return
for f in report.files[sp_key]:
# Make a note of the filename so that we can report it if something crashes
report.last_found_file = os.path.join(f["root"], f["fn"])
# Filter out files based on exclusion patterns
if path_filters_exclude and len(path_filters_exclude) > 0:
exlusion_hits = (fnmatch.fnmatch(report.last_found_file, pfe)
for pfe in path_filters_exclude)
if any(exlusion_hits):
logger.debug(
f"{sp_key} - Skipping '{report.last_found_file}' as it matched the path_filters_exclude for '{self.name}'"
)
continue
# Filter out files based on inclusion patterns
if path_filters and len(path_filters) > 0:
inclusion_hits = (fnmatch.fnmatch(report.last_found_file, pf)
for pf in path_filters)
if not any(inclusion_hits):
logger.debug(
f"{sp_key} - Skipping '{report.last_found_file}' as it didn't match the path_filters for '{self.name}'"
)
continue
else:
logger.debug(
f"{sp_key} - Selecting '{report.last_found_file}' as it matched the path_filters for '{self.name}'"
)
# Make a sample name from the filename
f["sp_key"] = sp_key
f["s_name"] = self.clean_s_name(f["fn"], f)
if filehandles or filecontents:
try:
# Custom content module can now handle image files
(ftype, encoding) = mimetypes.guess_type(
os.path.join(f["root"], f["fn"]))
if ftype is not None and ftype.startswith("image"):
with io.open(os.path.join(f["root"], f["fn"]),
"rb") as fh:
# always return file handles
f["f"] = fh
yield f
else:
# Everything else - should be all text files
with io.open(os.path.join(f["root"], f["fn"]),
"r",
encoding="utf-8") as fh:
if filehandles:
f["f"] = fh
yield f
elif filecontents:
f["f"] = fh.read()
yield f
except (IOError, OSError, ValueError, UnicodeDecodeError) as e:
logger.debug(
"Couldn't open filehandle when returning file: {}\n{}".
format(f["fn"], e))
f["f"] = None
else:
yield f
