def generate_linux_package_report(report_name: str, vulnerabilities: List,
folder_id: str, drive_id: str) -> None:
rows = []
rows.append(
'Device,Criticality,Vulnerability,Upgrade/Update,CVE,Info'.split(','))
sheet_id = drive.create_file(folder_id, report_name,
'application/vnd.google-apps.spreadsheet',
google_user_for_service_account, drive_id)
if sheet_id is not None:
criticality_of_vulnerabilities = categorize_vulnerabilities_on_criticality(
vulnerabilities)
for category in criticality_of_vulnerabilities:
devices = {}
for vulnerability in criticality_of_vulnerabilities[category]:
if vulnerability.dns:
if vulnerability.dns in devices:
devices[vulnerability.dns].append(vulnerability)
else:
devices[vulnerability.dns] = [vulnerability]
else:
if vulnerability.ip in devices:
devices[vulnerability.ip].append(vulnerability)
else:
devices[vulnerability.ip] = [vulnerability]
logger.info(
'Found %d devices that marked as %s for linux security updates'
% (len(devices), category))
for device in devices:
packages = []
plugins = []
cves = []
info_links = []
for vulnerability in devices[device]:
packages.extend(vulnerability.resolution.split(','))
plugins.append(vulnerability.plugin_name)
if vulnerability.cves:
cves.extend(vulnerability.cves)
info_links.extend(vulnerability.additional_links)
packages = '\r'.join(list(set(packages))).replace("'",
'').replace(
'"', '')
plugins = '\r'.join(list(set(plugins))).replace("'",
'').replace(
'"', '')
cves = ', '.join(list(set(cves))).replace("'",
'').replace('"', '')
info_links = '\r'.join(list(set(info_links))).replace(
"'", '').replace('"', '')
rows.append(("%s|%s|%s|%s|%s|%s" %
(device, category, plugins, packages, cves,
info_links)).split('|'))
sheet.update_sheet(report_name, rows, google_user_for_service_account,
sheet_id)
def check_and_create_report_corp_folders(
drive_id: str, sub_sub_folder_id: str) -> Tuple[str, str]:
# Check Corporate Folder
corporate_folder_id = drive.find_folder('Corporate',
google_user_for_service_account,
drive_id, sub_sub_folder_id)
if corporate_folder_id is not None:
# Check Windows Folder Corporate
corp_windows_folder_id = drive.find_folder(
'Windows', google_user_for_service_account, drive_id,
corporate_folder_id)
if corp_windows_folder_id is None:
corp_windows_folder_id = drive.create_file(
corporate_folder_id, 'Windows',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
# Check Mac Folder Corporate
corp_mac_folder_id = drive.find_folder(
'Macs', google_user_for_service_account, drive_id,
corporate_folder_id)
if corp_mac_folder_id is None:
corp_mac_folder_id = drive.create_file(
corporate_folder_id, 'Macs',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
else:
corporate_folder_id = drive.create_file(
sub_sub_folder_id, 'Corporate',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
corp_windows_folder_id = drive.create_file(
corporate_folder_id, 'Windows',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
corp_mac_folder_id = drive.create_file(
corporate_folder_id, 'Macs', 'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
return corp_windows_folder_id, corp_mac_folder_id
def check_and_create_report_prod_folders(
drive_id: str, sub_sub_folder_id: str) -> Tuple[str, str, str]:
# Check Production Folder
production_folder_id = drive.find_folder('Production',
google_user_for_service_account,
drive_id, sub_sub_folder_id)
if production_folder_id is not None:
# Check GCP Folder
prod_gcp_folder_id = drive.find_folder(
'GCP', google_user_for_service_account, drive_id,
production_folder_id)
if prod_gcp_folder_id is None:
prod_gcp_folder_id = drive.create_file(
production_folder_id, 'GCP',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
# Check Windows Folder Prod
prod_windows_folder_id = drive.find_folder(
'Windows', google_user_for_service_account, drive_id,
production_folder_id)
if prod_windows_folder_id is None:
prod_windows_folder_id = drive.create_file(
production_folder_id, 'Windows',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
# Check Linux Folder
prod_linux_folder_id = drive.find_folder(
'Linux', google_user_for_service_account, drive_id,
production_folder_id)
if prod_linux_folder_id is None:
prod_linux_folder_id = drive.create_file(
production_folder_id, 'Linux',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
else:
production_folder_id = drive.create_file(
sub_sub_folder_id, 'Production',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
prod_windows_folder_id = drive.create_file(
production_folder_id, 'Windows',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
prod_linux_folder_id = drive.create_file(
production_folder_id, 'Linux',
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
prod_gcp_folder_id = drive.create_file(
production_folder_id, 'GCP', 'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
return prod_gcp_folder_id, prod_windows_folder_id, prod_linux_folder_id
def check_and_create_report_root_folders() -> Tuple[str, str]:
drive_id = drive.find_drive(google_team_drive,
google_user_for_service_account)
# Check main folder in Security Engineering
root_folder_id = drive.find_folder(gdrive_folder_in_team_drive,
google_user_for_service_account,
drive_id)
year = datetime.datetime.now().strftime('%Y')
month_year = datetime.datetime.now().strftime('%B - %Y')
if root_folder_id is not None:
# Check year folder in main folder
sub_folder_id = drive.find_folder(year,
google_user_for_service_account,
drive_id, root_folder_id)
if sub_folder_id is not None:
# Check month year folder in year sub-folder
sub_sub_folder_id = drive.find_folder(
month_year, google_user_for_service_account, drive_id,
sub_folder_id)
if sub_sub_folder_id is None:
sub_sub_folder_id = drive.create_file(
sub_folder_id, month_year,
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
else:
sub_folder_id = drive.create_file(
root_folder_id, year, 'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
sub_sub_folder_id = drive.create_file(
sub_folder_id, month_year,
'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
else:
root_folder_id = drive.create_file(
root_folder_id, gdrive_folder_in_team_drive,
'application/vnd.google-apps.folder',
drive.google_user_for_service_account, drive_id)
sub_folder_id = drive.create_file(
root_folder_id, year, 'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
sub_sub_folder_id = drive.create_file(
sub_folder_id, month_year, 'application/vnd.google-apps.folder',
google_user_for_service_account, drive_id)
return drive_id, sub_sub_folder_id
def main(duration: int):
# Extract chrome extensions from tenable scan
chrome_extensions_per_device_per_user, extension_ids_per_version, extension_ids_per_name = tenable.get_chrome_extensions(duration)
risky_objects = {}
extensions_to_run_reports_on = {}
paid_extensions = []
not_in_store = []
extensions_to_scan_again = []
new_extensions_to_scan_again = []
not_an_extension = []
latest_version_extension = {}
# Submit all extensions to crxcavator to reduce the number of extensions to actually get a report on and classify the result accordingly
logger.info('Found %d extensions' % len(extension_ids_per_version))
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
fs = [executor.submit(crxcavator.submit_extension, extension_id) for extension_id in extension_ids_per_version]
for future in concurrent.futures.as_completed(fs):
scan_result = future.result()
if scan_result['extension'] and not scan_result['not_free'] and not scan_result['removed_from_store']:
extensions_to_run_reports_on[scan_result['id']] = extension_ids_per_version[scan_result['id']]
if scan_result['version']:
latest_version_extension[scan_result['id']] = scan_result['version']
elif scan_result['not_free']:
paid_extensions.append(scan_result['id'])
elif scan_result['removed_from_store']:
not_in_store.append(scan_result['id'])
elif scan_result['run_again']:
extensions_to_scan_again.append(scan_result['id'])
elif not scan_result['extension']:
not_an_extension.append(scan_result['id'])
# Try to scan extensions again that errored when submitted to crxcavator
if extensions_to_scan_again:
logger.info('Scanning %d extensions again' % len(extensions_to_scan_again))
seconds_to_sleep = 90
logger.info('Sleeping %d seconds' % seconds_to_sleep)
time.sleep(seconds_to_sleep)
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
fs = [executor.submit(crxcavator.submit_extension, extension_id) for extension_id in extensions_to_scan_again]
for future in concurrent.futures.as_completed(fs):
scan_result = future.result()
if scan_result['extension'] and not scan_result['not_free'] and not scan_result['removed_from_store']:
extensions_to_run_reports_on[scan_result['id']] = extension_ids_per_version[scan_result['id']]
if scan_result['version']:
latest_version_extension[scan_result['id']] = scan_result['version']
elif scan_result['not_free']:
paid_extensions.append(scan_result['id'])
elif scan_result['removed_from_store']:
not_in_store.append(scan_result['id'])
elif scan_result['run_again']:
new_extensions_to_scan_again.append(scan_result['id'])
elif not scan_result['extension']:
not_an_extension.append(scan_result['id'])
logger.info('Unable to retreive %d extensions from webstore' % len(new_extensions_to_scan_again))
logger.info('Found %d free extensions, %d non-extensions, %d paid-extensions & %d extensions not in store' % (len(extensions_to_run_reports_on), len(not_an_extension), len(paid_extensions), len(not_in_store)))
# Fetch reports on the extension for a particular version
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
fs = [executor.submit(crxcavator.fetch_risk_details, extension_id, extension_ids_per_version[extension_id]) for extension_id in extensions_to_run_reports_on]
for future in concurrent.futures.as_completed(fs):
riskobjs = future.result()
if riskobjs:
for riskobj in riskobjs:
risky_extension_obj = RiskyExtensions(riskobj.id, riskobj.version)
#risky_extension_obj.name = extension_ids_per_name[risky_extension_obj.id]
risky_extension_obj.name = riskobj.name
risky_extension_obj.risk_csp = riskobj.risk_csp
risky_extension_obj.risk_external_javascript = riskobj.risk_external_javascript
risky_extension_obj.risk_external_calls = riskobj.risk_external_calls
risky_extension_obj.risk_score = riskobj.risk_score
risky_extension_obj.entry_points = riskobj.entry_points
risky_extension_obj.dangerous_functions = riskobj.dangerous_functions
risky_extension_obj.chrome_link = riskobj.chrome_link
risky_extension_obj.crxcavator_link = riskobj.crxcavator_link
if riskobj.id in risky_objects:
#risky_objects[risky_extension_obj.name][risky_extension_obj.version] = risky_extension_obj
risky_objects[riskobj.id][risky_extension_obj.version] = risky_extension_obj
else:
#risky_objects[risky_extension_obj.name] = {
risky_objects[riskobj.id] = {
risky_extension_obj.version: risky_extension_obj
}
logger.info('Reworking chrome extensions to combine devices and users of the same extension and same version')
risky_objects, no_results_for_extension_name_version, paid_extension_per_id, removed_from_chrome_store_per_id = combine_chrome_extensions(chrome_extensions_per_device_per_user, risky_objects, latest_version_extension, paid_extensions, not_in_store)
del chrome_extensions_per_device_per_user
# GOOGLE DRIVE STORAGE OPTION
# YOU CAN CHANGE THE BELOW PART TO DOWNLOAD TO THE LOCATION OF YOUR CHOICE
drive.drive_access_tokens[gsuite_service_account] = {}
access_token, expiry = drive.generate_drive_api_access_token(gsuite_service_account)
if access_token is not None and expiry is not None:
drive.drive_access_tokens[gsuite_service_account]['access_token'] = access_token
drive.drive_access_tokens[gsuite_service_account]['expiry'] = expiry
drive_id = drive.find_drive(team_drive, gsuite_service_account)
sheet.sheet_access_tokens[gsuite_service_account] = {}
access_token, expiry = sheet.generate_sheet_api_access_token(gsuite_service_account)
if access_token is not None and expiry is not None:
sheet.sheet_access_tokens[gsuite_service_account]['access_token'] = access_token
sheet.sheet_access_tokens[gsuite_service_account]['expiry'] = expiry
if drive_id:
logger.info('Finding folder "Chrome Extension Scanner" in %s drive' % team_drive)
folder_id = drive.find_item("Chrome Extension Scanner", gsuite_service_account, drive_id, 'folder')
if not folder_id:
logger.info('Creating folder Chrome Extension Scanner in %s drive' % team_drive)
folder_id = drive.create_file("Chrome Extension Scanner", gsuite_service_account, drive_id, 'folder')
file_name = 'Chrome_Extension_Scan_(Crxcavator)_%s' % datetime.now().isoformat()
file_id = drive.create_file(file_name, gsuite_service_account, drive_id, 'sheet', folder_id)
if file_id and risky_objects:
page = sheet.new_sheet(file_name, file_id, gsuite_service_account, ['Risky Chrome Extensions'])
if 'Risky Chrome Extensions' in page and page['Risky Chrome Extensions']:
create_report_risky_extensions(risky_objects, file_name, file_id, page['Risky Chrome Extensions'])
if file_id and paid_extension_per_id:
page = sheet.new_sheet(file_name, file_id, gsuite_service_account, ['Unscanned Paid Chrome Extensions'])
if 'Unscanned Paid Chrome Extensions' in page and page['Unscanned Paid Chrome Extensions']:
create_report_unscanned_paid_extensions(paid_extension_per_id, file_name, file_id, page['Unscanned Paid Chrome Extensions'])
if file_id and removed_from_chrome_store_per_id:
page = sheet.new_sheet(file_name, file_id, gsuite_service_account, ['Chrome Extensions Removed from Store'])
if 'Chrome Extensions Removed from Store' in page and page['Chrome Extensions Removed from Store']:
create_report_removed_from_store(removed_from_chrome_store_per_id, file_name, file_id, page['Chrome Extensions Removed from Store'])
if file_id and no_results_for_extension_name_version:
page = sheet.new_sheet(file_name, file_id, gsuite_service_account, ['Version Unavailable in Crxcavator'])
if 'Version Unavailable in Crxcavator' in page and page['Version Unavailable in Crxcavator']:
create_report_no_version_extension(no_results_for_extension_name_version, file_name, file_id, page['Version Unavailable in Crxcavator'])
if file_id and new_extensions_to_scan_again:
page = sheet.new_sheet(file_name, file_id, gsuite_service_account, ['Unscanned in Crxcavator'])
if 'Unscanned in Crxcavator' in page and page['Unscanned in Crxcavator']:
create_report_unscanned_extensions(new_extensions_to_scan_again, extension_ids_per_name, file_name, file_id, page['Unscanned in Crxcavator'])
def generate_general_report(report_name: str, vulnerabilities: List,
folder_id: str, drive_id: str) -> None:
rows = []
sheet_id = drive.create_file(folder_id, report_name,
'application/vnd.google-apps.spreadsheet',
google_user_for_service_account, drive_id)
if sheet_id is not None:
rows.append(
'Device,Criticality,Vulnerability,Solution,CVE,Info'.split(','))
criticality_of_vulnerabilities = categorize_vulnerabilities_on_criticality(
vulnerabilities)
for category in criticality_of_vulnerabilities:
solutions = {}
for vulnerability in criticality_of_vulnerabilities[category]:
if vulnerability.resolution in solutions:
solutions[vulnerability.resolution].append(vulnerability)
else:
solutions[vulnerability.resolution] = [vulnerability]
for solution in solutions:
devices = []
plugins = []
cves = []
info_links = []
for vulnerability in solutions[solution]:
devices.append(vulnerability.dns or vulnerability.ip)
plugins.append(vulnerability.plugin_name)
if vulnerability.cves:
cves.extend(vulnerability.cves)
info_links.extend(vulnerability.additional_links)
plugins = '\r'.join(list(set(plugins))).replace("'",
'').replace(
'"', '')
cves = ', '.join(list(set(cves))).replace("'",
'').replace('"', '')
info_links = '\r'.join(list(set(info_links))).replace(
"'", '').replace('"', '')
devices = list(set(devices))
# Sheet can accept a max of 50000 characters in a single cell
if len(str(devices)) > 45000:
number_in_each_block_list = int(
len(devices) / (len(str(devices)) / 45000))
device_list = [
devices[i:number_in_each_block_list] for i in range(
0, len(devices), number_in_each_block_list)
]
for each_list in device_list:
devices = '\r'.join(each_list).replace("'",
'').replace(
'"', '')
rows.append(("%s|%s|%s|%s|%s|%s" %
(devices, category, plugins, solution,
cves, info_links)).split('|'))
else:
devices = '\r'.join(devices).replace("'",
'').replace('"', '')
rows.append(("%s|%s|%s|%s|%s|%s" %
(devices, category, plugins, solution, cves,
info_links)).split('|'))
sheet.update_sheet(report_name, rows, google_user_for_service_account,
sheet_id)
def generate_windows_package_report(report_name: str, vulnerabilities: List,
folder_id: str, drive_id: str) -> None:
package_rows = []
package_rows.append(
'Device,Criticality,Vulnerability,KB,CVE,Info'.split(','))
package_sheet_id = drive.create_file(
folder_id, 'KB_%s' % report_name,
'application/vnd.google-apps.spreadsheet',
google_user_for_service_account, drive_id)
update_rows = []
update_rows.append(
'Device,Criticality,Vulnerability,Solution,CVE,Info'.split(','))
update_sheet_id = drive.create_file(
folder_id, 'Software_Update_%s' % report_name,
'application/vnd.google-apps.spreadsheet',
google_user_for_service_account, drive_id)
if package_sheet_id is not None and update_sheet_id is not None:
criticality_of_vulnerabilities = categorize_vulnerabilities_on_criticality(
vulnerabilities)
for category in criticality_of_vulnerabilities:
kb_patch_vulnerabilities = []
update_patch_vulnerabilities = []
for vulnerability in criticality_of_vulnerabilities[category]:
if 'KB' in vulnerability.resolution[:3]:
kb_patch_vulnerabilities.append(vulnerability)
else:
update_patch_vulnerabilities.append(vulnerability)
kb_patch_devices = {}
'''
{
testhost.com: {
KB: []
Plugin Name: []
CVE: []
Link: []
Critcality: ''
}
}
'''
update_patch_device = {}
for vulnerability in kb_patch_vulnerabilities:
if 'centos' not in vulnerability.plugin_name.lower():
if vulnerability.dns:
if vulnerability.dns in kb_patch_devices:
kb_patch_devices[vulnerability.dns].append(
vulnerability)
else:
kb_patch_devices[vulnerability.dns] = [
vulnerability
]
else:
if vulnerability.ip in kb_patch_devices:
kb_patch_devices[vulnerability.ip].append(
vulnerability)
else:
kb_patch_devices[vulnerability.ip] = [
vulnerability
]
for vulnerability in update_patch_vulnerabilities:
if 'centos' not in vulnerability.plugin_name.lower():
if vulnerability.dns:
if vulnerability.dns in update_patch_device:
update_patch_device[vulnerability.dns].append(
vulnerability)
else:
update_patch_device[vulnerability.dns] = [
vulnerability
]
else:
if vulnerability.ip in update_patch_device:
update_patch_device[vulnerability.ip].append(
vulnerability)
else:
update_patch_device[vulnerability.ip] = [
vulnerability
]
logger.info('Found %d devices that marked as %s for KB patches' %
(len(kb_patch_devices), category))
logger.info('Found %d devices that marked as %s for updates' %
(len(update_patch_device), category))
for device in kb_patch_devices:
KB = []
plugins = []
cves = []
info_links = []
for vulnerability in kb_patch_devices[device]:
KB.extend(vulnerability.resolution.split(','))
plugins.append(vulnerability.plugin_name)
if vulnerability.cves:
cves.extend(vulnerability.cves)
info_links.extend(vulnerability.additional_links)
KB = '\r'.join(list(set(KB))).replace("'", '').replace('"', '')
plugins = '\r'.join(list(set(plugins))).replace("'",
'').replace(
'"', '')
cves = ', '.join(list(set(cves))).replace("'",
'').replace('"', '')
info_links = '\r'.join(list(set(info_links))).replace(
"'", '').replace('"', '')
package_rows.append(
("%s|%s|%s|%s|%s|%s" % (device, category, plugins, KB,
cves, info_links)).split('|'))
for device in update_patch_device:
solution = []
plugins = []
cves = []
info_links = []
for vulnerability in update_patch_device[device]:
solution.append(vulnerability.resolution)
plugins.append(vulnerability.plugin_name)
if vulnerability.cves:
cves.extend(vulnerability.cves)
info_links.extend(vulnerability.additional_links)
solution = '\r'.join(list(set(solution))).replace("'",
'').replace(
'"', '')
plugins = '\r'.join(list(set(plugins))).replace("'",
'').replace(
'"', '')
cves = ', '.join(list(set(cves))).replace("'",
'').replace('"', '')
info_links = '\r'.join(list(set(info_links))).replace(
"'", '').replace('"', '')
update_rows.append(("%s|%s|%s|%s|%s|%s" %
(device, category, plugins, solution, cves,
info_links)).split('|'))
sheet.update_sheet('KB_%s' % report_name, package_rows,
google_user_for_service_account, package_sheet_id)
sheet.update_sheet('Software_Update_%s' % report_name, update_rows,
google_user_for_service_account, update_sheet_id)