def parse_reports(resultlist,
groups=[],
ugly=True,
includeMetadata=False,
python=False):
"""Turn report dictionaries into json output. Returns a string.
resultlist - A list of the scan return values
groups - A list of modules types that will be grouped together for the report
ugly - If True the return json will not be formatted
includeMetadata - If True module metadata will be included in the report
python - If true a python dictionary is returned instead of a json string
"""
files = {}
metadatas = {}
for item in resultlist:
if item is not None:
(result, metadata) = item
else:
continue
for (fname, hit) in result:
if fname not in files:
files[fname] = {}
# Group together module results if configured
if metadata['Type'] in groups:
if not files[fname].get(metadata['Type'], False):
files[fname][metadata['Type']] = {}
files[fname][metadata['Type']][metadata['Name']] = hit
# Else put it in the root of the file
else:
files[fname][metadata['Name']] = hit
# This is to prevent some modules from showing in metadata reports.
if includeMetadata:
if metadata['Name'] not in metadatas and metadata.get(
"Include", True):
metadatas[metadata['Name']] = metadata
if includeMetadata:
finaldata = {"Files": files, "Metadata": metadatas}
else:
finaldata = files
if python:
return finaldata
finaldata = convert_encoding(finaldata)
if not ugly:
return json.dumps(finaldata,
sort_keys=True,
indent=3,
ensure_ascii=False)
else:
return json.dumps(finaldata,
sort_keys=True,
separators=(',', ':'),
ensure_ascii=False)
def scan(filelist, conf=DEFAULTCONF):
results = []
libmagicresults, libmagicmeta = REQUIRES[0]
for fname, libmagicresult in libmagicresults:
if fname not in filelist:
print("DEBUG: File not in filelist")
if not libmagicresult.startswith('PE32'):
continue
result = {}
pe = pefile.PE(fname)
result['pehash'] = _get_pehash(pe)
check, sha = _get_rich_header(pe)
if check:
result['rich_header_checksum'] = check
if sha:
result['rich_header_sha256'] = sha
if callable(getattr(pe, 'get_imphash', None)):
try:
result['import_hash'] = pe.get_imphash()
except:
pass
if hasattr(pe, 'DIRECTORY_ENTRY_RESOURCE'):
result['resource_data'] = _dump_resource_data("ROOT",
pe.DIRECTORY_ENTRY_RESOURCE,
pe,
False)
result['sections'] = _get_sections(pe)
if hasattr(pe, 'DIRECTORY_ENTRY_IMPORT'):
result['imports'] = _get_imports(pe)
else:
result['imports'] = None
if hasattr(pe, 'DIRECTORY_ENTRY_EXPORT'):
result['exports'] = _get_exports(pe)
else:
result['exports'] = None
result['pe_timestamp'] = _get_timestamp(pe)
if hasattr(pe, 'DIRECTORY_ENTRY_DEBUG'):
result['debug_info'] = _get_debug_info(pe)
if hasattr(pe, 'VS_VERSIONINFO'):
result['version_info'] = _get_version_info(pe)
if hasattr(pe, 'DIRECTORY_ENTRY_TLS'):
ret = _get_tls_info(pe)
if ret:
result['tls_callback_info'] = ret
result = convert_encoding(result)
results.append((fname, result))
metadata = {}
metadata["Name"] = NAME
metadata["Type"] = TYPE
metadata["Version"] = pefile.__version__
metadata["Include"] = False
return (results, metadata)
def parse_reports(resultlist, groups=[], ugly=True, includeMetadata=False, python=False):
"""Turn report dictionaries into json output. Returns a string.
resultlist - A list of the scan return values
groups - A list of modules types that will be grouped together for the report
ugly - If True the return json will not be formatted
includeMetadata - If True module metadata will be included in the report
python - If true a python dictionary is returned instead of a json string
"""
files = {}
metadatas = {}
for item in resultlist:
if item is not None:
(result, metadata) = item
else:
continue
for (fname, hit) in result:
if fname not in files:
files[fname] = {}
# Group together module results if configured
if metadata['Type'] in groups:
if not files[fname].get(metadata['Type'], False):
files[fname][metadata['Type']] = {}
files[fname][metadata['Type']][metadata['Name']] = hit
# Else put it in the root of the file
else:
files[fname][metadata['Name']] = hit
# This is to prevent some modules from showing in metadata reports.
if includeMetadata:
if metadata['Name'] not in metadatas and metadata.get("Include", True):
metadatas[metadata['Name']] = metadata
if includeMetadata:
finaldata = {"Files": files, "Metadata": metadatas}
else:
finaldata = files
if python:
return finaldata
finaldata = convert_encoding(finaldata)
if not ugly:
return json.dumps(finaldata, sort_keys=True, indent=3, ensure_ascii=False)
else:
return json.dumps(finaldata, sort_keys=True, separators=(',', ':'), ensure_ascii=False)
def _main():
# Force all prints to go to stderr
stdout = sys.stdout
sys.stdout = sys.stderr
# Import dependencies only needed by _main()
import zipfile
# Get args
args = _parse_args()
# Set verbose
if args.verbose:
global VERBOSE
VERBOSE = args.verbose
# Checks if user is trying to initialize
if args.Files == ['init'] and not os.path.isfile('init'):
_init(args)
if not os.path.isfile(args.config):
config_init(args.config)
# Make sure report is not a dir
if args.json:
if os.path.isdir(args.json):
print('ERROR:', args.json, 'is a directory, a file is expected')
return False
# Parse the file list
parsedlist = parseFileList(args.Files, recursive=args.recursive)
# Unzip zip files if asked to
if args.extractzips:
for fname in parsedlist:
if zipfile.is_zipfile(fname):
unzip_dir = os.path.join('_tmp', os.path.basename(fname))
z = zipfile.ZipFile(fname)
if PY3:
args.password = bytes(args.password, 'utf-8')
try:
z.extractall(path=unzip_dir, pwd=args.password)
for uzfile in z.namelist():
parsedlist.append(os.path.join(unzip_dir, uzfile))
except RuntimeError as e:
print("ERROR: Failed to extract ", fname, ' - ', e, sep='')
parsedlist.remove(fname)
# Resume from report
if args.resume:
i = len(parsedlist)
try:
reportfile = codecs.open(args.json, 'r', 'utf-8')
except Exception as e:
print("ERROR: Could not open report file")
exit(1)
for line in reportfile:
line = json.loads(line)
for fname in line:
if fname in parsedlist:
parsedlist.remove(fname)
reportfile.close()
i = i - len(parsedlist)
if VERBOSE:
print("Skipping", i, "files which are in the report already")
# Do multiple runs if there are too many files
filelists = []
if len(parsedlist) > args.numberper:
while len(parsedlist) > args.numberper:
filelists.append(parsedlist[:args.numberper])
parsedlist = parsedlist[args.numberper:]
if parsedlist:
filelists.append(parsedlist)
for filelist in filelists:
# Record start time for metadata
starttime = str(datetime.datetime.now())
# Run the multiscan
results = multiscan(filelist, recursive=None, configfile=args.config)
# We need to read in the config for the parseReports call
Config = configparser.SafeConfigParser()
Config.optionxform = str
Config.read(args.config)
config = _get_main_config(Config)
# Make sure we have a group-types
if "group-types" not in config:
config["group-types"] = []
elif not config["group-types"]:
config["group-types"] = []
# Add in script metadata
endtime = str(datetime.datetime.now())
# For windows compatibility
try:
username = os.getlogin()
except:
username = os.getenv('USERNAME')
results.append(([], {"Name": "MultiScanner",
"Start Time": starttime,
"End Time": endtime,
# "Command Line":list2cmdline(sys.argv),
"Run by": username
}))
if args.show or not stdout.isatty():
# TODO: Make this output something readable
# Parse Results
report = parse_reports(results, groups=config["group-types"], ugly=args.ugly, includeMetadata=args.metadata)
# Print report
try:
print(convert_encoding(report, encoding='ascii', errors='replace'), file=stdout)
stdout.flush()
except Exception as e:
print('ERROR: Can\'t print report -', e)
report = parse_reports(results, groups=config["group-types"], includeMetadata=args.metadata, python=True)
update_conf = None
if args.json:
update_conf = {'File': {'path': args.json}}
if args.json.endswith('.gz') or args.json.endswith('.gzip'):
update_conf['File']['gzip'] = True
if 'storage-config' not in config:
config["storage-config"] = None
storage_handle = storage.StorageHandler(configfile=config["storage-config"], config=update_conf)
storage_handle.store(report)
storage_handle.close()
# Cleanup zip extracted files
if args.extractzips:
shutil.rmtree('_tmp')
def _main():
# Force all prints to go to stderr
stdout = sys.stdout
sys.stdout = sys.stderr
# Import dependencies only needed by _main()
import zipfile
# Get args
args = _parse_args()
# Set verbose
if args.verbose:
global VERBOSE
VERBOSE = args.verbose
# Checks if user is trying to initialize
if str(args.Files) == "['init']" and not os.path.isfile('init'):
_init(args)
if not os.path.isfile(args.config):
config_init(args.config)
# Make sure report is not a dir
if args.json:
if os.path.isdir(args.json):
print('ERROR:', args.json, 'is a directory, a file is expected')
return False
# Parse the file list
parsedlist = parseFileList(args.Files, recursive=args.recursive)
# Unzip zip files if asked to
if args.extractzips:
for fname in parsedlist:
if zipfile.is_zipfile(fname):
unzip_dir = os.path.join('_tmp', os.path.basename(fname))
z = zipfile.ZipFile(fname)
if PY3:
args.password = bytes(args.password, 'utf-8')
try:
z.extractall(path=unzip_dir, pwd=args.password)
for uzfile in z.namelist():
parsedlist.append(os.path.join(unzip_dir, uzfile))
except RuntimeError as e:
print("ERROR: Failed to extract ", fname, ' - ', e, sep='')
parsedlist.remove(fname)
# Resume from report
if args.resume:
i = len(parsedlist)
try:
reportfile = codecs.open(args.json, 'r', 'utf-8')
except Exception as e:
print("ERROR: Could not open report file")
exit(1)
for line in reportfile:
line = json.loads(line)
for fname in line:
if fname in parsedlist:
parsedlist.remove(fname)
reportfile.close()
i = i - len(parsedlist)
if VERBOSE:
print("Skipping", i, "files which are in the report already")
# Do multiple runs if there are too many files
filelists = []
if len(parsedlist) > args.numberper:
while len(parsedlist) > args.numberper:
filelists.append(parsedlist[:args.numberper])
parsedlist = parsedlist[args.numberper:]
if parsedlist:
filelists.append(parsedlist)
for filelist in filelists:
# Record start time for metadata
starttime = str(datetime.datetime.now())
# Run the multiscan
results = multiscan(filelist, recursive=None, configfile=args.config)
# We need to read in the config for the parseReports call
Config = configparser.SafeConfigParser()
Config.optionxform = str
Config.read(args.config)
config = _get_main_config(Config)
# Make sure we have a group-types
if "group-types" not in config:
config["group-types"] = []
elif not config["group-types"]:
config["group-types"] = []
# Add in script metadata
endtime = str(datetime.datetime.now())
# For windows compatibility
try:
username = os.getlogin()
except:
username = os.getenv('USERNAME')
results.append((
[],
{
"Name": "MultiScanner",
"Start Time": starttime,
"End Time": endtime,
# "Command Line":list2cmdline(sys.argv),
"Run by": username
}))
if args.show or not stdout.isatty():
# TODO: Make this output something readable
# Parse Results
report = parse_reports(results,
groups=config["group-types"],
ugly=args.ugly,
includeMetadata=args.metadata)
# Print report
try:
print(convert_encoding(report,
encoding='ascii',
errors='replace'),
file=stdout)
stdout.flush()
except Exception as e:
print('ERROR: Can\'t print report -', e)
report = parse_reports(results,
groups=config["group-types"],
includeMetadata=args.metadata,
python=True)
update_conf = None
if args.json:
update_conf = {'File': {'path': args.json}}
if args.json.endswith('.gz') or args.json.endswith('.gzip'):
update_conf['File']['gzip'] = True
if 'storage-config' not in config:
config["storage-config"] = None
storage_handle = storage.StorageHandler(
configfile=config["storage-config"], config=update_conf)
storage_handle.store(report)
storage_handle.close()
# Cleanup zip extracted files
if args.extractzips:
shutil.rmtree('_tmp')
