def copysource(self, result_directory):
# copies all the json, html, css and js files and saves them to the result directory
# create content for the source.json file
source_json = {}
# Copies all the json, css, js files to the result directory for future reference
for file in self.files:
if file['type'] == 'json' or file['type'] == 'html' or file[
'type'] == 'css' or file['type'] == 'js':
file_path = file['path']
new_path = helper.fixpath(result_directory + '/' +
file['name'] + '.src')
file_name = file['name']
if os.path.isfile(file_path):
# Checks if file present
shutil.copyfile(file_path, new_path)
core.updatelog('Copied ' + file_path + ' to ' + new_path)
# append this to source_json dict
rel_path = os.path.relpath(file_path, self.directory)
file_size = str(os.path.getsize(file_path) >> 10) + ' KB'
if file['type'] == 'js':
# Retire js scan
core.updatelog(
'Running retirejs vulnerability scan on: ' +
file_name)
try:
with open(file_path, 'r') as fc:
file_content = fc.read()
rjs_scan = retirejs.scan_file_content(
file_content)
core.updatelog('Scan complete!')
except Exception as e:
core.updatelog(
'Error {0} while running retirejs scan on {1}'.
format(str(e), file_name))
rjs_scan = []
source_json[file['id']] = ({
'id': file['id'],
'file_name': file_name,
'location': new_path,
'relative_path': rel_path,
'file_size': file_size,
'retirejs_result': rjs_scan
})
else:
source_json[file['id']] = ({
'id': file['id'],
'file_name': file_name,
'location': new_path,
'relative_path': rel_path,
'file_size': file_size
})
# write all the changes to source.json
source_file = helper.fixpath(result_directory + '/source.json')
sf = open(source_file, 'w+')
sf.write(json.dumps(source_json, indent=4, sort_keys=True))
sf.close()
core.updatelog('Saved sources to: ' + source_file)
return True
def extract_chromium_plugins(self, parent_dir):
ret_list = []
extension_dirs = os.listdir(parent_dir)
for extension_dir in extension_dirs:
extension_path = os.path.join(parent_dir, extension_dir)
if not os.path.isdir(extension_path):
core.updatelog("Invalid extension directory: " +
extension_path)
continue
extension_vers = os.listdir(extension_path)
for ver in extension_vers:
manifest_file = helper.fixpath(extension_path + "/" + ver +
"/manifest.json")
if not os.path.isfile(manifest_file):
core.updatelog("Invalid extension directory: " +
extension_path)
continue
ext_name = core.GetNameFromManifest(manifest_file)
if ext_name:
ext_version = ver.split('_')[0]
ext_name = ext_name + ' version ' + ext_version
# small hack to not let commas f**k around with the path
ext_name = ext_name.replace(",", ",")
ret_list.append(ext_name + ',' +
helper.fixpath(extension_path + "/" + ver))
return ret_list
def analyzelocalfirefoxextension(path):
if os.path.isfile(path) and path.endswith('.xpi'):
# Extract the .xpi file to a temp directory in lab directory
# Analyze the extracted directory
# delete the temp directory
extract_directory = helper.fixpath(core.lab_path + '/temp_extract_directory')
try:
core.updatelog('Unzipping ' + path + ' to: ' + extract_directory)
zip_contents = zipfile.ZipFile(path, 'r')
zip_contents.extractall(extract_directory)
zip_contents.close()
core.updatelog('Unzipping complete')
except Exception as e:
helper.fixpath('Something went wrong while unzipping ' + path + ' to ' + extract_directory)
logging.error(traceback.format_exc())
return False
analysis_status = analysis.analyze(extract_directory, 'Local Firefox Extension')
if 'error:' in analysis_status:
core.updatelog('Something went wrong while analysis... deleting temporary extract directory')
else:
core.updatelog('Scanning complete... Deleting temporary extract directory')
shutil.rmtree(extract_directory)
core.updatelog('Successfully deleted: ' + extract_directory)
return analysis_status
else:
core.updatelog('[analyzelocalfirefoxextension] Invalid local firefox extension path: ' + path)
def googlechrome(self):
# TODO: add support for mac os
chrome_directory = ""
if self.os == 'windows':
chrome_directory = helper.fixpath(
self.user_directory +
'\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Extensions'
)
elif self.os == 'linux':
chrome_directory = helper.fixpath(
self.user_directory +
'/.config/google-chrome/Default/Extensions')
elif self.os == 'osx':
chrome_directory = helper.fixpath(
self.user_directory +
'/Library/Application Support/Google/Chrome/Profile 1/Extensions'
)
if chrome_directory != "":
if os.path.isdir(chrome_directory):
core.updatelog('Found Google chrome extension directory: ' +
chrome_directory)
return self.extract_chromium_plugins(chrome_directory)
else:
core.updatelog('Could not find google chrome directory!')
return False
else:
core.updatelog('Unsupported OS')
def savereport(self):
try:
# Gen scan id
report = core.report
reportids = core.reportids
curid = 'EXA' + time.strftime("%Y%j%H%M%S", time.gmtime())
core.updatelog('Saving Analysis with ID: ' + curid)
#saveas = curid + '.json'
# create result directory
result_directory = os.path.join(core.reports_path, curid)
if not os.path.exists(result_directory):
os.makedirs(result_directory)
core.updatelog('Created Result directory: ' + result_directory)
# create the basic report json file: extanalysis_report.json
report_file = helper.fixpath(result_directory + '/extanalysis_report.json')
f = open(report_file, 'w+')
reportfinal = json.dumps(report, sort_keys=True, indent=4)
f.write(reportfinal)
f.close()
core.updatelog('Saved report file: ' + report_file)
# get file list and create the graph
graph_data_stat = self.creategraphdata()
if graph_data_stat != False:
graph_file = helper.fixpath(result_directory + '/graph.data')
graph_file_create = open(graph_file, 'w+')
graph_file_create.write(self.nodes + '\n' + self.edges)
core.updatelog('Saved graph data to: ' + graph_file)
else:
core.updatelog('Could not save graph data!')
# save the source files
# print(result_directory)
self.copysource(result_directory)
'''
make an entry in the reports index file
'''
# use <reports_path> as a variable!
relative_result_path = result_directory.replace(core.reports_path, '<reports_path>')
reportindex = {"name":report['name'], "version":report['version'], "id": curid, "report_directory":relative_result_path, "time":time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())}
reportids['reports'].append(reportindex)
indexfile = core.report_index
g = open(indexfile, 'w+')
indexfinal = json.dumps(reportids, sort_keys=True, indent=4)
g.write(indexfinal)
g.close()
core.updatelog('Updated report index')
# Clear global report value for new scan
core.report = {}
return(curid)
except Exception as e:
logging.error(traceback.format_exc())
print('Something went wrong while saving result. Error: ' + str(e))
def googlechrome(self):
# TODO: add support for mac os
chrome_directory = ""
if self.os == 'windows':
chrome_directory = helper.fixpath(
self.user_directory +
'\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\Extensions'
)
elif self.os == 'linux':
chrome_directory = helper.fixpath(
self.user_directory +
'/.config/google-chrome/Default/Extensions')
if chrome_directory != "":
if os.path.isdir(chrome_directory):
core.updatelog('Found Google chrome extension directory: ' +
chrome_directory)
extension_dirs = os.listdir(chrome_directory)
for extension_dir in extension_dirs:
# print('\n\n')
if os.path.isdir(
os.path.join(chrome_directory, extension_dir)):
# Every extension directory is like this: Extension/<id>/<version>/{contents}
extension_path = os.path.join(chrome_directory,
extension_dir)
extension_vers = os.listdir(extension_path)
for ver in extension_vers:
manifest_file = helper.fixpath(extension_path +
"/" + ver +
'/manifest.json')
if os.path.isfile(manifest_file):
ext_name = core.GetNameFromManifest(
manifest_file)
if ext_name != False and ext_name != None:
# append version with name
ext_version = ver.split('_')[0]
ext_name = ext_name + ' version ' + ext_version
self.chrome_extensions.append(
ext_name + ',' +
helper.fixpath(extension_path + "/" +
ver))
else:
core.updatelog(
'Could not determine extension name.. skipping local chrome extension'
)
else:
core.updatelog(
'Invalid extension directory: ' +
extension_path)
return self.chrome_extensions
else:
core.updatelog('Could not find google chrome directory!')
return False
else:
core.updatelog('Unsupported OS')
def home():
core.updatelog('Accessed Main page')
lic = open(helper.fixpath(core.path + '/LICENSE'), 'r')
license_text = lic.read()
cred = open(helper.fixpath(core.path + '/CREDITS'), 'r')
credits_text = cred.read()
return render_template("index.html",
report_dir=core.reports_path,
lab_dir=core.lab_path,
license_text=license_text,
credits_text=credits_text,
virustotal_api=core.virustotal_api)
def downloadFirefox(url):
if 'addons.mozilla.org' not in url:
core.updatelog('Invalid Firefox addon URL')
return False
else:
try:
test = urllib.request.Request(url)
test.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0'
)
source = urllib.request.urlopen(test)
source_code = source.read().decode('utf-8')
xpi_file = re.findall(
'<a class="Button Button--action AMInstallButton-button Button--puffy" href="(.*?).xpi?',
source_code)[0]
core.updatelog('Found link for xpi file: ' + xpi_file + '.xpi')
name = xpi_file.split('/')[-1]
xpi_file += '.xpi'
save_path = helper.fixpath(core.lab_path + '/' + name + '.xpi')
core.updatelog("Downloader says: save_path is " + save_path)
try:
urllib.request.urlretrieve(xpi_file, save_path)
core.updatelog("Extension downloaded successfully: " +
save_path)
return name
except Exception as e:
core.updatelog("Error while downloading xpi file: " + xpi_file)
print(e)
return False
except Exception:
core.updatelog(
'Something went wrong while getting download link for xpi file'
)
def get_result_info(analysis_id):
'''
GET INFO ABOUT A SPECIFIC ANALYSIS INFO
RESPONSE = [TRUE/FALSE, JSON_LOADED_RESULT/ERROR_MSG]
'''
global reportids, report_index, reports_path
if reportids == {}:
# index not loaded.. let's load it up
indexs = open(report_index, 'r')
indexs = json.loads(indexs.read())
reportids = indexs
reports = reportids['reports']
if analysis_id in str(reports):
for report in reports:
if report['id'] == analysis_id:
report['report_directory'] = helper.fixpath(
report['report_directory'].replace('<reports_path>',
reports_path).replace(
'\\', '/'))
return [True, report]
return [False, 'Analysis ID mismatch: {0}'.format(analysis_id)]
else:
return [
False,
'Analysis ID {0} not found in result index!'.format(analysis_id)
]
def sort_files(extension_dir):
try:
extract_dir = helper.fixpath(core.lab_path + '/' + extension_dir)
html_files = []
js_files = []
css_files = []
static_files = []
other_files = [] # File extensions that are not listed above
for root, dirs, files in os.walk(extract_dir):
for file in files:
filepath = os.path.join(root, file)
file = file.lower()
if file.endswith('.html'):
core.updatelog('Discovered html file: ' + file)
html_files.append(filepath)
elif file.endswith('.js'):
core.updatelog('Discovered js file: ' + file)
js_files.append(filepath)
elif file.endswith('.css'):
core.updatelog('Discovered css file: ' + file)
css_files.append(filepath)
elif file.endswith(('.png', '.jpg', '.jpeg', '.bmp', '.tiff', '.svg')):
core.updatelog('Discovered static file: ' + file)
static_files.append(filepath)
else:
other_files.append(filepath)
core.updatelog('Sorted files: ' + extension_dir)
#core.updatelog('HTML Files: {0}, JS Files: {1}, CSS Files: {2}, Static Files: {3}, Other Files: {4}'.format(str(count(html_files)), str(count(js_files)), str(count(css_files)), str(count(static_files)), str(count(other_files)))
r = {'html_files': html_files, 'js_files': js_files, 'css_files': css_files, 'static_files': static_files, 'other_files': other_files}
return r
except Exception as e:
core.updatelog('Error while sorting files: ' + str(e))
return False
def get_country(ip):
'''
Gets Country and country code from given IP.
Parameters = ip = ip address for lookup
Response = [True, {country_code}, {country_name}] or [False, {ERR_MSG}]
Needs maxminddb for fast performance
'''
core.updatelog('Getting country from IP: ' + ip)
try:
# If maxminddb module is installed we don't have to query online services to get the country code hence saving a lot of time
import maxminddb
try:
core.updatelog('Getting country from local DB')
reader = maxminddb.open_database(helper.fixpath(core.path + '/db/geoip.mmdb'))
ip_info = reader.get(ip)
iso_code = ip_info['country']['iso_code'].lower()
country = ip_info['country']['names']['en']
return [True, iso_code, country]
except Exception as e:
core.updatelog('Something went wrong while getting country from ip {0}! Error: {1}'.format(ip, str(e)))
logging.error(traceback.format_exc())
return [False, str(e)]
except:
core.updatelog('maxminddb module not installed! Using online service to get Country from IP')
core.updatelog('To save time in future analysis; install maxminddb by: pip3 install maxminddb')
import core.scans as scan
gip = scan.geoip(ip)
if gip[0]:
geoip = gip[1]
return [True, geoip['country'].lower(), geoip['country_name']]
else:
return [False, gip[1]]
def update():
'''
Updates ExtAnalysis
1. Create the updater child script and save it to temp directory
2. End self process and start the child script
'''
print("\n[i] Creating Updater file")
child_script = open(helper.fixpath(core.path + '/db/updater.py'), 'r')
child_script = child_script.read()
src = child_script.replace('<current_extanalysis_directory>',
core.path.replace('\\', '\\\\'))
src = src.replace('<github_zip_url>', core.github_zip)
print('[i] Moving updater file to temp directory')
temp_dir = tempfile.gettempdir()
updater_script = helper.fixpath(temp_dir + '/update_extanalysis.py')
f = open(updater_script, 'w+')
f.write(src)
f.close()
python_loc = sys.executable
print('[i] Starting Updater script')
if sys.platform == 'win32':
os.chdir(temp_dir)
command = [python_loc, 'update_extanalysis.py']
subprocess.Popen(command,
creationflags=subprocess.CREATE_NEW_CONSOLE,
shell=False)
print(
'[i] Killing self... Next time we meet I will be a better version of myself ;)'
)
exit()
else:
os.chdir(temp_dir)
command = ['x-terminal-emulator', '-e', python_loc, updater_script]
subprocess.Popen(command, shell=False)
print(
'[i] Killing self... Next time we meet I will be a better version of myself ;)'
)
exit()
def braveLocalExtensionsCheck(self):
brave_directory = ""
if self.os == 'windows':
brave_directory = helper.fixpath(
self.user_directory +
'\\AppData\\Local\\BraveSoftware\\Brave-Browser\\User Data\\Default\\Extensions'
)
elif self.os == 'linux':
brave_directory = helper.fixpath(
self.user_directory +
'/.config/BraveSoftware/Brave-Browser/Default/Extensions')
if brave_directory != "":
if os.path.isdir(brave_directory):
core.updatelog('Found Brave extension directory: ' +
brave_directory)
return self.extract_chromium_plugins(brave_directory)
else:
core.updatelog('Could not find Brave extension directory!')
return False
else:
core.updatelog('Unsupported OS')
def extract_urls(file_path):
updatelog('Extracting URLs From: ' + file_path)
urls = []
try:
cnt = open(helper.fixpath(file_path), 'r', encoding='utf8')
contents = cnt.read()
curls = re.findall('(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?', contents)
for url in curls:
urls.append(url[0]+'://'+url[1]+url[2])
updatelog('Found url: ' + url[0]+'://'+url[1]+url[2])
urls = list(set(urls))
return(urls)
except Exception as e:
updatelog('error: Something went wrong while reading file')
updatelog('ERROR: ' + str(e))
logging.error(traceback.format_exc())
return []
def vivaldi_local_extensions_check(self):
vivaldi_dir = ""
if self.os == 'windows':
vivaldi_dir = helper.fixpath(
self.user_directory +
'\\AppData\\Local\\Vivaldi\\User Data\\Default\\Extensions')
if vivaldi_dir == "":
core.updatelog('Unsupported OS')
return
if not os.path.isdir(vivaldi_dir):
core.updatelog("Couldn't find Vivaldi Extension directory!")
return
# --
core.updatelog('Found Vivaldi extension directory: ' + vivaldi_dir)
return self.extract_chromium_plugins(vivaldi_dir)
def download(id, name=""):
ext_id = id
if name == "":
save_name = ext_id
else:
save_name = name
save_path = helper.fixpath(core.lab_path + '/' + save_name + '.crx')
core.updatelog("Downloader says: save_path is " + save_path)
dl_url = "https://clients2.google.com/service/update2/crx?response=redirect&x=id%3D" + ext_id + "%26uc&prodversion=32"
print("Download URL: " + dl_url)
try:
urllib.request.urlretrieve(dl_url, save_path)
core.updatelog("Extension downloaded successfully: " + save_path)
return save_name
except Exception as e:
core.updatelog("Error in downloader.py")
print(e)
return False
def download(id, name=""):
ext_id = id
if name == "":
save_name = ext_id
else:
save_name = name
save_path = helper.fixpath(core.lab_path + '/' + save_name + '.crx')
core.updatelog("Downloader says: save_path is " + save_path)
# dl_url = "https://clients2.google.com/service/update2/crx?response=redirect&x=id%3D" + ext_id + "%26uc&prodversion=32"
# new download URL, issue #13
dl_url = "https://clients2.google.com/service/update2/crx?response=redirect&os=win&arch=x86-64&os_arch=x86-64&nacl_arch=x86-64&prod=chromecrx&prodchannel=unknown&prodversion=81.0.4044.138&acceptformat=crx2,crx3&x=id%3D" + ext_id + "%26uc"
print("Download URL: " + dl_url)
try:
urllib.request.urlretrieve(dl_url, save_path)
core.updatelog("Extension downloaded successfully: " + save_path)
return save_name
except Exception as e:
core.updatelog("Error in downloader.py")
print(e)
return False
def download(id, name=""): #chrome
ext_id = id
if name == "":
save_name = ext_id
else:
save_name = name
save_path = helper.fixpath(core.lab_path + '/' + save_name + '.crx')
core.updatelog("Downloader says: save_path is " + save_path)
if sys.platform == 'win32':
os = 'windows'
file = open(
' '
) #path or command to find where windows installed latest version is located (wmi or registry seems to be easiest)
version = file.readline() #grabs the chrome version
#dl_url = 'http://clients2.google.com/service/update2/crx?response=redirect&prodversion=' + version + '&acceptformat=crx2,crx3&x=id%3D' + ext_id + '%26uc'
elif sys.platform == 'darwin':
os = 'osx'
username = getpass.getuser(
) #this gets the current users username as it is in the system path on MAC
file = open(
'/Users/' + username +
'/Library/Application Support/Google/Chrome/Last Version', 'r'
) # combining the username into the macOS path so we can open the file.
version = file.readline(
) # this grabs the version of chrome out of the local file
dl_url = 'http://clients2.google.com/service/update2/crx?response=redirect&prodversion=' + version + '&acceptformat=crx2,crx3&x=id%3D' + ext_id + '%26uc'
elif sys.platform == 'linux' or sys.platform == 'linux2':
os = 'linux'
dl_url = '' #need to craft linux url specific to finding the latest version of chrome
print("Download URL: " + dl_url)
try:
urllib.request.urlretrieve(dl_url, save_path)
core.updatelog("Extension downloaded successfully: " + save_path)
return save_name
except Exception as e:
core.updatelog("Error in downloader.py")
print(e)
return False
def firefox(self):
# read the profiles.ini
# check for previous list and create new if not found [list = extanalysis.json]
# Get a list of all the xpi files
# Unzip them
# Get all their names from manifest.json
# update the list
firefox_directory = ""
default_profile_path = ""
if self.os == 'windows':
firefox_directory = helper.fixpath(
self.user_directory + '\\AppData\\Roaming\\Mozilla\\Firefox')
if os.path.isdir(firefox_directory):
# firfox installed
firefox_profile = helper.fixpath(firefox_directory +
'\\profiles.ini')
if os.path.isfile(firefox_profile):
# found firefox profiles.ini
try:
firefox_config = configparser.SafeConfigParser()
with open(firefox_profile, 'rU') as ini_source:
firefox_config.readfp(ini_source)
default_profile_path = os.path.normpath(
os.path.join(
firefox_directory,
firefox_config.get('Profile0', 'Path')))
core.updatelog('Found firefox profile path: ' +
default_profile_path)
except Exception as e:
core.updatelog(
'Something went wrong while reading firefox profiles.ini'
)
logging.error(traceback.format_exc())
return False
else:
core.updatelog(
'Could not find profiles.ini ExtAnalysis can\'t analyze local firefox extensions'
)
return False
else:
# Could not find firefox directory
core.updatelog('Firefox installation could not be detected')
return False
elif self.os == 'linux':
firefox_directory = helper.fixpath(self.user_directory +
'/.mozilla/firefox/')
if os.path.isdir(firefox_directory):
# firfox installed
firefox_profile = helper.fixpath(firefox_directory +
'/profiles.ini')
if os.path.isfile(firefox_profile):
# found firefox profiles.ini
try:
firefox_config = configparser.SafeConfigParser()
with open(firefox_profile, 'rU') as ini_source:
firefox_config.readfp(ini_source)
default_profile_path = os.path.normpath(
os.path.join(
firefox_directory,
firefox_config.get('Profile0', 'Path')))
core.updatelog('Found firefox profile path: ' +
default_profile_path)
except Exception as e:
core.updatelog(
'Something went wrong while reading firefox profiles.ini'
)
logging.error(traceback.format_exc())
return False
else:
core.updatelog(
'Could not find profiles.ini ExtAnalysis can\'t analyze local firefox extensions'
)
return False
else:
# Could not find firefox directory
core.updatelog('Firefox installation could not be detected')
return False
if default_profile_path != "":
if os.path.isdir(default_profile_path):
# profile path is valid
firefox_extension_directory = os.path.join(
default_profile_path, 'extensions')
if os.path.join(firefox_extension_directory):
unfiltered_files = os.listdir(firefox_extension_directory)
xpi_files = []
for afile in unfiltered_files:
if afile.endswith('.xpi') and os.path.isfile(
os.path.join(firefox_extension_directory,
afile)):
xpi_files.append(afile)
core.updatelog('xpi list generated')
else:
core.updatelog(
'extensions directory could not be found inside firefox default profile'
)
return False
else:
core.updatelog(
'Invalid firefox profile path... Can\'t get local firefox extensions'
)
return False
else:
core.updatelog('Could not find default profile path for firefox')
return False
if xpi_files != []:
exta_firefox_list = os.path.join(firefox_extension_directory,
'extanalysis.json')
if os.path.isfile(exta_firefox_list):
# found previous list
core.updatelog(
'Found previous analysis log.. updating with current extensions'
)
listed_extensions = []
list_file = open(exta_firefox_list, 'r')
list_files = json.loads(list_file.read())
for list_file in list_files['extensions']:
listed_extensions.append(list_file)
for xpi_file in xpi_files:
if xpi_file not in listed_extensions:
core.updatelog('Inserting ' + xpi_file + ' into list')
self.createFirefoxListing(firefox_extension_directory,
xpi_file)
# return True
else:
core.updatelog('Creating ExtAnalysis list file')
list_file = open(exta_firefox_list, 'w+')
list_file.write('{"extensions":{}}')
list_file.close()
core.updatelog('Updating list file with all xpi file infos')
for xpi_file in xpi_files:
core.updatelog('Inserting ' + xpi_file + ' into list')
self.createFirefoxListing(firefox_extension_directory,
xpi_file)
# return True
else:
core.updatelog('No installed firefox extensions found!')
return False
# Read the final list and then create return list and return it
firefox_extensions_list = []
read_list = open(exta_firefox_list, 'r')
read_list = json.loads(read_list.read())
if read_list['extensions'] != {}:
# There are some extensions
for fext in read_list['extensions']:
prepare_to_insert = read_list['extensions'][fext][
'name'] + ',' + read_list['extensions'][fext]['file']
firefox_extensions_list.append(prepare_to_insert)
return firefox_extensions_list
else:
core.updatelog(
'ExtAnalysis could not find any local firefox extensions')
def view(query, allargs):
if query == 'dlanalysis':
try:
extension_id = allargs.get('extid')
saveas = ""
try:
saveas = allargs.get('savedir')
if saveas == "" or saveas == " ":
saveas = extension_id
except Exception as e:
print('Save name not specified')
try:
download_log = download_extension.download(
extension_id, saveas)
if download_log:
aok = analysis.analyze(saveas + '.crx',
'Remote Google Chrome Extension')
return (aok)
else:
return (
'error: Something went wrong while downloading extension'
)
except Exception as e:
core.updatelog(
'Something went wrong while downloading extension: ' +
str(e))
return (
'error: Something went wrong while downloading extension, check log for more information'
)
except Exception as e:
core.updatelog('Something went wrong: ' + str(e))
return (
'error: Something went wrong while downloading extension, check log for more information'
)
elif query == 'firefoxaddon':
try:
addonurl = allargs.get('addonurl')
try:
download_log = download_extension.downloadFirefox(addonurl)
if download_log:
aok = analysis.analyze(download_log + '.xpi',
'Remote Firefox Addon')
return (aok)
else:
return (
'error: Something went wrong while downloading extension'
)
except Exception as e:
core.updatelog(
'Something went wrong while downloading extension: ' +
str(e))
return (
'error: Something went wrong while downloading extension, check log for more information'
)
except Exception as e:
core.updatelog('Something went wrong: ' + str(e))
return (
'error: Something went wrong while downloading extension, check log for more information'
)
elif query == 'results':
reportids = core.reportids
if reportids == {}:
# Result index not loaded so let's load it and show em results
core.updatelog('Reading report index and loading json')
ridfile = core.report_index
ridcnt = open(ridfile, 'r', encoding='utf8')
ridcnt = ridcnt.read()
reportids = json.loads(ridcnt)
rd = "<table class='result-table' id='result-table'><thead><tr><th>Name</th><th>Version</th><th>Date</th><th>Actions</th></tr></thead><tbody>"
for areport in reportids['reports']:
report_name = areport['name']
report_id = areport['id']
report_date = areport['time']
report_version = areport['version']
rd += '<tr><td>' + report_name + '</td><td>' + report_version + '</td><td>' + report_date + '</td><td><button class="bttn-fill bttn-xs bttn-primary" onclick=viewResult(\'' + report_id + '\')><i class="fas fa-eye"></i> View</button> <button class="bttn-fill bttn-xs bttn-danger" onclick=deleteResult(\'' + report_id + '\')><i class="fas fa-trash"></i> Delete</button></td></tr>'
return (rd + '</tbody></table><br>')
elif query == 'getlocalextensions':
try:
browser = allargs.get('browser')
if browser == 'googlechrome':
import core.localextensions as localextensions
lexts = localextensions.GetLocalExtensions()
exts = ""
exts = lexts.googlechrome()
if exts != False and exts != [] and exts != None:
return_html = "<table class='result-table' id='result-table'><thead><tr><th>Extension Name</th><th>Action</th></tr></thead><tbody>"
for ext in exts:
ext_info = ext.split(',')
return_html += '<tr><td>' + ext_info[
0] + '</td><td><button class="bttn-fill bttn-xs bttn-success" onclick="analyzeLocalExtension(\'' + ext_info[
1].replace(
'\\', '\\\\'
) + '\', \'googlechrome\')"><i class="fas fa-bolt"></i> Analyze</button></td></tr>'
return (return_html + '</tbody></table>')
else:
return (
'error: Something went wrong while getting local Google Chrome extensions! Check log for more information'
)
elif browser == 'firefox':
import core.localextensions as localextensions
lexts = localextensions.GetLocalExtensions()
exts = lexts.firefox()
if exts != False and exts != [] and exts != None:
return_html = "<table class='result-table' id='result-table'><thead><tr><th>Extension Name</th><th>Action</th></tr></thead><tbody>"
for ext in exts:
ext_info = ext.split(',')
return_html += '<tr><td>' + ext_info[
0] + '</td><td><button class="bttn-fill bttn-xs bttn-success" onclick="analyzeLocalExtension(\'' + ext_info[
1].replace(
'\\', '\\\\'
) + '\', \'firefox\')"><i class="fas fa-bolt"></i> Analyze</button></td></tr>'
return (return_html + '</tbody></table>')
else:
return (
'error: Something went wrong while getting local firefox extensions! Check log for more information'
)
else:
return ('error: Invalid Browser!')
except Exception:
logging.error(traceback.format_exc())
return ('error: Incomplete Query')
elif query == 'analyzelocalextension':
try:
browser = allargs.get('browser')
path_to_local = allargs.get('path')
path = helper.fixpath(path_to_local)
if browser == 'firefox' and os.path.isfile(path):
# valid firefox extension
import core.localextensions as localextensions
analysis_stat = localextensions.analyzelocalfirefoxextension(
path)
return (analysis_stat)
elif browser == 'googlechrome' and os.path.isdir(path):
if os.path.isfile(os.path.join(path, 'manifest.json')):
analysis_stat = analysis.analyze(
path, 'Local Google Chrome Extension')
return (analysis_stat)
else:
return ('error: Invalid Google Chrome Extension Directory')
else:
return ('error: Malformed Query')
except Exception:
logging.error(traceback.format_exc())
return ('error: Incomplete Query')
elif query == 'deleteAll':
'''
DELETES ALL RESULTS
RESPONSE = SUCCESS / ERROR
'''
import core.result as result
delete_status = result.clearAllResults()
if delete_status:
return "success"
else:
return (
'There were some errors while deleting all analysis reports... refer to log for more information'
)
elif query == 'clearLab':
'''
Deletes all the contents of lab
RESPONSE = SUCCESS / ERROR
'''
clear_lab = core.clear_lab()
if clear_lab[0]:
# Successful
return (clear_lab[1])
else:
# Unsuccessful
return ('error: ' + clear_lab[1])
elif query == 'deleteResult':
'''
DELETES A SPECIFIC RESULT
PARAMETER = resultID
RESPONSE = SUCCESS_MSG / 'error: ERROR_MSG'
'''
try:
result_id_to_delete = allargs.get('resultID')
import core.result as result
delete_status = result.clearResult(result_id_to_delete)
if delete_status:
return "success"
else:
return "Something went wrong while deleting result! Check log for more information"
except Exception:
return ('Invalid Query')
elif query == 'vtDomainReport':
try:
domain = allargs.get('domain')
analysis_id = allargs.get('analysis_id')
ranalysis = core.get_result_info(analysis_id)
if ranalysis[0]:
# if ranalysis[0] is True then ranalysis[1] contains the details
analysis_dir = ranalysis[1]['report_directory']
analysis_report = os.path.join(analysis_dir,
'extanalysis_report.json')
if os.path.isfile(analysis_report):
report = open(analysis_report, 'r')
domains = json.loads(report.read())['domains']
for adomain in domains:
if adomain['name'] == domain:
vtjson = json.dumps(adomain['virustotal'],
indent=4,
sort_keys=False)
#return_html = '<div id="vt_info"></div><script>var wrapper1 = document.getElementById("vt_info");var data = '+vtjson+' try {var data = JSON.parse(dataStr);} catch (e) {} var tree = jsonTree.create(data, wrapper1);tree.expand(function(node) { return node.childNodes.length < 2 || node.label === "phoneNumbers";});</script>'
return vtjson
return ('error: Domain info not found in analysis report!')
else:
return ('error: Analysis report for #{0} not found'.format(
analysis_id))
else:
# ranalysis[1] is the error msg when ranalysis[0] = False
return ('error: ' + ranalysis[1])
except:
logging.error(traceback.format_exc())
return ('error: Malformed api call')
elif query == 'retirejsResult':
'''
GET RETIREJS SCAN RESULTS FOR FILE
REQUIRED PARAMETER: file = FILE_ID
'''
try:
file_id = allargs.get('file')
analysis_id = allargs.get('analysis_id')
ranalysis = core.get_result_info(analysis_id)
if ranalysis[0]:
# if ranalysis[0] is True then ranalysis[1] contains the details
analysis_dir = ranalysis[1]['report_directory']
source_json = os.path.join(analysis_dir, 'source.json')
if os.path.isfile(source_json):
report = open(source_json, 'r')
files = json.loads(report.read())
for _file in files:
if _file == file_id:
retirejs_result = files[_file]['retirejs_result']
if retirejs_result == []:
ret = 'none'
else:
ret = json.dumps(retirejs_result,
indent=4,
sort_keys=False)
return ret
return ('error: File ID not found in report!')
else:
return ('error: Analysis report for #{0} not found'.format(
analysis_id))
else:
# ranalysis[1] is the error msg when ranalysis[0] = False
return ('error: ' + ranalysis[1])
except:
logging.error(traceback.format_exc())
return ('error: Malformed api call')
elif query == 'whois':
'''
GET WHOIS REPORT OF DOMAIN
REQUIRES 'python-whois' module
RESPONSE = HTML DIV WITH FORMATTED WHOIS INFO
'''
try:
domain = allargs.get('domain')
try:
import whois
except:
return (
"error: python-whois module not installed! install it using `pip3 install python-whois` or `pip3 install -r requirements.txt`"
)
whois_result = whois.whois(domain)
whois_html = '<div class="whois-data" style="overflow-y: scroll; max-height:500px; text-align: left;">'
for data in whois_result:
proper_data = data.replace('_', ' ').capitalize()
if isinstance(whois_result[data], list):
for subdata in whois_result[data]:
whois_html += '<b style="color:#89ff00;">{0} : </b>{1}<br>'.format(
proper_data, subdata)
else:
whois_html += '<b style="color:#89ff00;">{0} : </b>{1}<br>'.format(
proper_data, whois_result[data])
whois_html += '</div>'
if whois_result:
return (
'<center><h4>Whois Results For {0}</h4></center><br>{1}'.
format(domain, whois_html))
else:
return (
"error: Something went wrong while checking whois information of: "
+ domain)
except Exception:
logging.error(traceback.format_exc())
return ('error: Invalid Query')
elif query == 'geoip':
'''
GEO-IP LOOKUP OF AN IP ADDRESS
PARAMETERS -> IP = CONTAINS IP ADDRESS TO BE LOOKED UP
RETURNS A HTML TO BE SHOWN
'''
try:
ip_address = allargs.get('ip')
geo_ip = scan.geoip(ip_address)
if geo_ip[0]:
gip = geo_ip[1]
rethtml = '<div class="whois-data" style="overflow-y: scroll; max-height:500px; text-align: left;">'
for g in gip:
name = str(g).replace('_', ' ').capitalize()
val = str(gip[g])
rethtml += '<b style="color:#89ff00;">{0} : </b>{1}<br>'.format(
name, val)
rethtml += '</div>'
return (
'<center><h4>Geo-IP Lookup Results For {0}</h4></center><br>{1}'
.format(ip_address, rethtml))
else:
# in case of geo_ip[0] being false element 1 has the error msg
return ('error: ' + geo_ip[1])
except Exception as e:
logging.error(traceback.format_exc())
return ('error: Invalid Query')
elif query == 'HTTPHeaders':
'''
HTTP HEADERS OF AN URL
PARAMETERS -> URL -> BASE64 ENCODED URL
RETURNS HTML
'''
try:
url = allargs.get('url')
url = base64.b64decode(url).decode('ascii')
headers_status = scan.http_headers(url)
if headers_status[0]:
rethtml = '<div class="whois-data" style="overflow-y: scroll; max-height:500px; text-align: left;">'
headers = headers_status[1]
for header in headers:
hval = headers[header]
rethtml += '<b style="color:#89ff00;">{0} : </b>{1}<br>'.format(
header, hval)
rethtml += '</div>'
return (
'<center><h4>Showing HTTP Headers of: {0}</h4></center><br>{1}'
.format(url, rethtml))
else:
return ('error: ' + headers_status[1])
except Exception as e:
logging.error(traceback.format_exc())
return ('error: Invalid Query')
elif query == 'SourceCode':
'''
GET SOURCE CODE OF AN URL
PARAMETERS -> URL -> BASE64 ENCODED URL
RETURNS HTML
'''
try:
url = allargs.get('url')
rurl = base64.b64decode(url).decode('ascii')
headers_status = scan.source_code(rurl)
if headers_status[0]:
rethtml = '<textarea id="src_code" class="source_code" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false">'
headers = headers_status[1]
rethtml += headers
rethtml += '</textarea><br><br><center><a href="{0}" target="_blank" class="start_scan"><i class="fas fa-external-link-alt"></i> View Full Screen</a>'.format(
'/source-code/' + url)
return ('<center><h4>Source Code of: {0}</h4></center><br>{1}'.
format(rurl, rethtml))
else:
return ('error: ' + headers_status[1])
except Exception as e:
logging.error(traceback.format_exc())
return ('error: Invalid Query')
elif query == 'clearlogs':
'''
CLEARS LOG
'''
core.clearlog()
return ('Logs cleared successfully!')
elif query == 'changeReportsDir':
'''
CHANGES THE REPORT DIRECTORY
RESPONSE = SUCCESS / 'error: ERROR_MSG'
'''
try:
newpath = allargs.get('newpath')
if os.path.isdir(newpath):
# valid directory.. let's get the absolute path and set it
absolute_path = os.path.abspath(newpath)
import core.settings as settings
change = settings.changedir(absolute_path)
if change[0]:
return (change[1])
else:
return ('error: ' + change[1])
else:
return ('error: Invalid directory path!')
except:
logging.error(traceback.format_exc())
return ('error: Invalid request for directory change!')
elif query == 'changeVTapi':
'''
CHANGE VIRUSTOTAL API
RESPONSE = SUCCESS_MSG / 'error: ERROR_MSG'
'''
try:
new_api = allargs.get('api')
import core.settings as settings
change = settings.change_vt_api(new_api)
if change[0]:
return (change[1])
else:
return ('error: ' + change[1])
except:
logging.error(traceback.format_exc())
return ('error: Invalid request!')
elif query == 'changelabDir':
'''
CHANGES THE LAB DIRECTORY
RESPONSE = SUCCESS / 'error : ERROR_MSG'
'''
try:
newpath = allargs.get('newpath')
if os.path.isdir(newpath):
# valid directory.. let's get the absolute path and set it
absolute_path = os.path.abspath(newpath)
import core.settings as settings
change = settings.changelabdir(absolute_path)
if change[0]:
return (change[1])
else:
return ('error: ' + change[1])
else:
return ('error: Invalid directory path!')
except:
logging.error(traceback.format_exc())
return ('error: Invalid request for directory change!')
elif query == 'updateIntelExtraction':
'''
UPDATES INTELS TO BE EXTRACTED
RESPONSE = SUCCESS_MSG / 'error: ' + ERROR_MSG
'''
try:
# Create the dict with all values and keys
parameters = {}
parameters["extract_comments"] = str(
allargs.get('extract_comments'))
parameters["extract_btc_addresses"] = str(
allargs.get('extract_btc_addresses'))
parameters["extract_base64_strings"] = str(
allargs.get('extract_base64_strings'))
parameters["extract_email_addresses"] = str(
allargs.get('extract_email_addresses'))
parameters["extract_ipv4_addresses"] = str(
allargs.get('extract_ipv4_addresses'))
parameters["extract_ipv6_addresses"] = str(
allargs.get('extract_ipv6_addresses'))
parameters["ignore_css"] = str(allargs.get('ignore_css'))
import core.settings as settings
status_code = settings.update_settings_batch(parameters)
# 0 = failed, 1 = success, 2 = some updated some not!
if status_code == '0':
return (
'error: Settings could not be updated! Check log for more information'
)
elif status_code == '1':
return (
'Settings updated successfully... Please restart ExtAnalysis for them to take effect!'
)
elif status_code == '2':
return (
'Some settings were updated and some were not... Please restart ExtAnalysis for them to take effect!'
)
else:
return (
'error: Invalid response from "update_settings_batch". please report it here: https://github.com/Tuhinshubhra/ExtAnalysis/issues/new'
)
except:
logging.error(traceback.format_exc())
return ('error: Incomplete Request!')
else:
return ('error: Invalid Query!')
def analyze(ext_name, ext_type='local'):
core.updatelog('analyze func: ' + ext_name)
# extension_extracted = False
if ext_name.endswith('.crx') or ext_name.endswith('.xpi') or ext_name.endswith('.zip'):
'''
EXTENSION NAME / PACKED PATH
UNZIP THE EXTENSION FOR FURTHER ANALYSIS
'''
if ext_name.endswith('.crx'):
file_extension = '.crx'
elif ext_name.endswith('.xpi'):
file_extension = '.xpi'
elif ext_name.endswith('.zip'):
file_extension = '.zip'
else:
file_extension = ''
if os.path.isfile(ext_name):
'''
Full extension path sent, we unzip it to the lab directroy
Used mostly to pass local firefox extensions
'''
ext_path = ext_name
ext_name = os.path.basename(ext_name).split(file_extension)[0]
extract_dir = helper.fixpath(core.lab_path + '/' + ext_name)
else:
'''
Only the extension name is sent..
In this case we assume that the extension is located inside the lab directory.
Used while scanning an uploaded or downloaded extension
'''
ext_path = helper.fixpath(core.lab_path + '/' + ext_name)
extract_dir = helper.fixpath(core.lab_path + '/' + ext_name.split(file_extension)[0])
core.updatelog('Trying to unzip {0} to {1}'.format(ext_path, extract_dir))
try:
if os.path.exists(extract_dir):
if os.path.exists(extract_dir + '_old'):
# there is already a _old directory we have to delete so that we can rename the last directory to this name
core.updatelog('Found previously created _old directory... deleting that')
try:
shutil.rmtree(extract_dir + '_old')
except Exception as e:
core.updatelog('Error while deleting: {0} . Error: {1}'.format(extract_dir + '_old', str(e)))
logging.error(traceback.format_exc())
return ('error: Something went wrong while deleting old scan directory {0}'.format(extract_dir + '_old'))
new_name = os.path.basename(extract_dir) + '_old'
core.updatelog('Renaming old extract directory {0} as {1}'.format(extract_dir, new_name))
os.rename(extract_dir, extract_dir + '_old')
core.updatelog('Old directory successfully renamed')
zip_contents = zipfile.ZipFile(ext_path, 'r')
zip_contents.extractall(extract_dir)
zip_contents.close()
core.updatelog('Zip Extracted Successfully')
# extension_extracted = True
except Exception as e:
logging.error(traceback.format_exc())
core.updatelog('Something went wrong while unzipping extension\nError: ' + str(e))
return ('error: Something went wrong while unzipping extension. Check log for more information')
elif os.path.isdir(ext_name):
# if ext_name is a directory most likely it's a local extension
ext_path = 'Local'
extract_dir = ext_name
else:
return('error: [analyze.py] Unsupported input!')
core.updatelog('======== Analysis Begins ========')
try:
core.updatelog('Reading manifest.json')
manifest_file = helper.fixpath(extract_dir + '/manifest.json')
manifest_load = open(manifest_file, 'r')
manifest_content = manifest_load.read()
manifest_content = json.loads(manifest_content)
rinit = core.initreport(manifest_content, extract_dir, ext_type)
if not rinit:
return('error: Something went wrong while parsing manifest.json... analysis stopped')
core.report['crx'] = ext_path
core.report['extracted'] = extract_dir
#####################################################################
##### PERMISSION CHECKS AND OTHER STUFFS RELATED TO PERMISSIONS #####
#####################################################################
perm_file = helper.fixpath(core.path + '/db/permissions.json')
perms = open(perm_file, 'r')
perms = perms.read()
perms = json.loads(perms)
try:
for permission in manifest_content['permissions']:
if permission != "":
permarray = {'name':permission}
core.updatelog('Discoverd Permission: ' + helper.escape(permission))
if permission in perms:
permarray['description'] = perms[permission]['description']
permarray['badge'] = perms[permission]['badge']
permarray['risk'] = perms[permission]['risk']
if perms[permission]['warning'] != 'none':
permarray['warning'] = perms[permission]['warning']
# core.updatelog('Warning: ' + perms[permission]['warning'])
else:
permarray['warning'] = 'na'
else:
permarray['description'] = 'na'
permarray['warning'] = 'na'
permarray['risk'] = 'none'
permarray['badge'] = '<i class="fas fa-question"></i>'
core.insertpermission(permarray)
except Exception as e:
core.updatelog('No permissions found')
core.updatelog(str(e))
#####################################################################
##### GET ALL FIELS AND STORE THEM FOR FURTHER ANALYSIS ####
#####################################################################
html_files = []
js_files = []
json_files = []
css_files = []
static_files = []
other_files = [] # File extensions that are not listed above
for root, dirs, files in os.walk(extract_dir):
for file in files:
filepath = os.path.join(root, file)
relpath = os.path.relpath(filepath, extract_dir)
fname = file
file = file.lower()
if file.endswith(('.html', '.htm')):
html_files.append(filepath)
core.report['files']['html'].append({fname : relpath})
elif file.endswith('.js'):
js_files.append(filepath)
core.report['files']['js'].append({fname : relpath})
elif file.endswith('.json'):
json_files.append(filepath)
core.report['files']['json'].append({fname : relpath})
elif file.endswith('.css'):
css_files.append(filepath)
core.report['files']['css'].append({fname : relpath})
elif file.endswith(('.png', '.jpg', '.jpeg', '.bmp', '.tiff', '.svg', '.gif')):
core.report['files']['static'].append({fname : relpath})
static_files.append(filepath)
else:
core.report['files']['other'].append({fname : relpath})
other_files.append(filepath)
######################################################################
## EXTRACT INTELS FROM FILES (url, email, ip address, btc address) ##
######################################################################
urls = []
domains = []
for allfiles in (js_files, html_files, json_files, css_files):
for file in allfiles:
try:
cnt = open(file, 'r', encoding="utf8")
contents = cnt.read()
relpath = os.path.relpath(file, extract_dir)
core.updatelog('Extracting intels from: ' + file)
intels = intel.extract(contents, relpath)
## Parse the intels and add them to result
found_urls = intels['urls']
found_mail = intels['mails']
found_btcs = intels['btc']
found_ipv4 = intels['ipv4']
found_ipv6 = intels['ipv6']
found_b64s = intels['base64']
found_cmnt = intels['comments']
for u in found_urls:
urls.append(u)
for m in found_mail:
core.report['emails'].append(m)
for b in found_btcs:
core.report['bitcoin_addresses'].append(b)
for i in found_ipv4:
core.report['ipv4_addresses'].append(i)
for i in found_ipv6:
core.report['ipv6_addresses'].append(i)
for b in found_b64s:
core.report['base64_strings'].append(b)
for c in found_cmnt:
core.report['comments'].append(c)
except Exception as e:
core.updatelog('Skipped reading file: {0} -- Error: {1}'.format(file, str(e)))
logging.error(traceback.format_exc())
#urls = list(set(urls)) [NOTE TO SELF] we are tracking all urls in all files so set isn't used here
######################################################################
## APPEND URLS, DOMAINS TO REPORT AND DO VIRUSTOTAL SCAN ON DOMAINS ##
######################################################################
for url in urls:
core.updatelog('Found URL: ' + url['url'])
domain = re.findall('^(?:https?:\/\/)?(?:[^@\/\\n]+@)?(?:www\.)?([^:\/?\\n]+)', url['url'])[0]
url['domain'] = domain
core.report['urls'].append(url) # add url to the report file
domains.append(domain)
if virustotal.pub_vt == []:
# No extra virustotal apis added hence the slow scan
core.updatelog('Starting virustotal analysis of domains. [SLOW MODE]')
virustotal_scans = virustotal.domain_batch_scan(set(domains))
for domain in set(domains):
core.updatelog('getting virustotal Scan results for domain: ' + domain)
if virustotal.pub_vt != []:
# the faster scan!
virustotal_report = virustotal.scan_domain(domain)
if not virustotal_report[0]:
core.updatelog('Error getting virustotal result... Error: ' + virustotal_report[1])
domain_vt = {"error":"Either rate limited or something else went wrong while getting domain report from virustotal"}
else:
core.updatelog('Virustotal result successfully acquired!')
domain_vt = virustotal_report[1]
else:
domain_vt = virustotal_scans[domain][1]
core.updatelog('Virustotal result successfully acquired!')
try:
ip = socket.gethostbyname(domain)
except:
ip = 'unknown'
if ip != 'unknown':
ip_info = ip2country.get_country(ip)
if ip_info[0]:
country = ip_info[2]
country_code = ip_info[1]
else:
country = 'unknown'
country_code = 'unknown'
else:
country = 'unknown'
country_code = 'unknown'
domainarr = {"name":domain, "ip":ip, "country_code":country_code, "country":country, "virustotal":domain_vt}
core.report['domains'].append(domainarr)
save_result = saveresult.createresult(extract_dir)
cid = save_result.savereport()
"""
if extension_extracted:
# if the extension was extracted we delete the directory it was extracted to
# don't want any local extensions to be deleted hence extension_extracted boolean
core.updatelog('Deleting extraction directory: ' + extract_dir)
try:
shutil.rmtree(extract_dir)
core.updatelog('Extraction directory successfully deleted')
except Exception as e:
core.updatelog('Something went wrong while deleting extraction directory: ' + str(e))
logging.error(traceback.format_exc())
"""
if cid != False and cid != None:
return('Extension analyzed and report saved under ID: ' + cid)
else:
return('error:Something went wrong with the analysis!')
except Exception as e:
core.updatelog('Something went wrong while reading source of manifest.json file')
print(e)
core.updatelog(logging.error(traceback.format_exc()))
import os
import time
import traceback
import logging
import json
import zipfile
import re
import shutil
import signal
import core.helper as helper
## all the paths
path = os.path.dirname(os.path.abspath(__file__)).replace('/core', '').replace(
'\core', '')
lab_path = helper.fixpath(path + '/lab')
reports_path = helper.fixpath(path + '/reports')
# Version
with open(os.path.join(path, 'current_version')) as vf:
version = vf.read()
# All the variables
quiet = False
log = ""
raw_log = "\n\n\n"
report = {
} #'{"name":"","version":"","author":"","permissions":[{"name":"","description":"","warning":""}],"urls":"","files":{"html":"","js":"","css":"","static":"","other":""},"content-scripts":[],"background-scripts":[],"pageaction-popup":[],"browseraction-popup":[]}'
reportids = {}
virustotal_api = ''
ignore_css = True
def view(analysis_id):
# so the result ids are in this format : EXA<some digits> so we can try to replace 'EXTA' and convert the rest to int if it passes it's a valid type
try:
int(analysis_id.replace('EXA', '')) # Check
analysis_info = core.get_result_info(analysis_id)
if not analysis_info[0]:
# Could not get analysis_info
error_txt = 'Something went wrong while getting analysis info!<br>Error: ' + analysis_info[
1]
return render_template(
'error.html',
error_title="Invalid Result ID",
error_head="Invalid Result ID: {0}".format(analysis_id),
error_txt=error_txt)
else:
result_directory = analysis_info[1]['report_directory'].replace(
'<reports_path>', core.reports_path)
if os.path.isdir(result_directory):
# result directory found let's check for all necessary files
graph_file = os.path.join(result_directory, 'graph.data')
report_json = os.path.join(result_directory,
'extanalysis_report.json')
source_file = os.path.join(result_directory, 'source.json')
if all([
os.path.isfile(the_file)
for the_file in [graph_file, report_json, source_file]
]):
core.updatelog('Viewing Analysis {0}'.format(analysis_id))
graph_data = open(graph_file, 'r')
graph_data = graph_data.read()
source_data = open(source_file, 'r')
source_data = json.loads(source_data.read())
report_data = open(report_json, 'r')
report_data = json.loads(report_data.read())
# prepare data to be sent to result page
basic_info_t = [
report_data['name'], report_data['version'],
report_data['author'], report_data['description'],
analysis_info[1]['time']
]
# extension type
extension_type = report_data['type']
# URL Table
if report_data['urls'] != []:
urls_table = '<table class="result-table" id="urls-table"><thead><tr><th>URL</th><th>Domain</th><th>File</th><th>Actions</th></tr></thead><tbody>'
extjs_table = '<table class="result-table" id="extjs-table"><thead><tr><th>URL</th><th>Domain</th><th>File</th><th>Actions</th></tr></thead><tbody>'
done_urls = []
done_ejss = []
urls_count = 0
extjs_count = 0
for aurl in report_data['urls']:
if aurl['url'].endswith('.js'):
if aurl['url'] not in done_ejss:
done_ejss.append(aurl['url'])
aurl_href = '<a href="{0}" class="ext_url" target="_blank"><i class="fas fa-external-link-alt" style="font-size:12px;"></i> {0}</a>'.format(
aurl['url'])
extjs_table += '<tr><td>' + aurl_href + '</td>'
b64url = "'" + base64.b64encode(
aurl['url'].encode(
'ascii',
'ignore')).decode('ascii') + "'"
extjs_table += '<td>{0}</td><td>{1}</td>'.format(
aurl['domain'], aurl['file'])
extjs_table += '<td><button class="bttn-fill bttn-xs bttn-primary" onclick=whois(\'{1}\')><i class="fab fa-searchengin"></i> WHOIS</button> <button class="bttn-fill bttn-xs bttn-success" onclick="getSource({0})"><i class="fas fa-code"></i> Source</button> <button class="bttn-fill bttn-xs bttn-danger" onclick="getHTTPHeaders({0})"><i class="fas fa-stream"></i> HTTP Headers</button></td></tr>'.format(
b64url, aurl['url'])
extjs_count += 1
else:
if aurl['url'] not in done_urls:
done_urls.append(aurl['url'])
aurl_href = '<a href="{0}" class="ext_url" target="_blank"><i class="fas fa-external-link-alt" style="font-size:12px;"></i> {0}</a>'.format(
aurl['url'])
urls_table += '<tr><td>' + aurl_href + '</td>'
urls_table += '<td>{0}</td><td>{1}</td>'.format(
aurl['domain'], aurl['file'])
b64url = "'" + base64.b64encode(
aurl['url'].encode(
'ascii',
'ignore')).decode('ascii') + "'"
urls_table += '<td><button class="bttn-fill bttn-xs bttn-primary" onclick=whois(\'{1}\')><i class="fab fa-searchengin"></i> WHOIS</button> <button class="bttn-fill bttn-xs bttn-success" onclick="getSource({0})"><i class="fas fa-code"></i> Source</button> <button class="bttn-fill bttn-xs bttn-danger" onclick="getHTTPHeaders({0})"><i class="fas fa-stream"></i> HTTP Headers</button></td></tr>'.format(
b64url, aurl['url'])
urls_count += 1
if done_urls != []:
urls_table += '</tbody></table>'
else:
urls_table = '<h3 class="nothing"> No URLs Found </h3>'
if done_ejss != []:
extjs_table += '</tbody></table>'
else:
extjs_table = '<h3 class="nothing"> No External JavaScript Found in any files! </h3>'
else:
urls_table = '<h3 class="nothing"> No URLs Found </h3>'
extjs_table = '<h3 class="nothing"> No External JavaScript Found in any files! </h3>'
extjs_count = 0
urls_count = 0
# Domains div
if report_data['domains'] != []:
domains_table = '<table class="result-table" id="domains-table"><thead><tr><th>Country</th><th>Domain</th><th>IP Address</th><th>Virustotal Score</th><th>Actions</th></tr></thead><tbody>'
for domain in report_data['domains']:
domain_flag = helper.fixpath(
core.path + '/static/images/flags/' +
domain['country_code'] + '.png')
if os.path.isfile(domain_flag):
flag_path = url_for('static',
filename='images/flags/' +
domain['country_code'] +
'.png')
else:
flag_path = url_for(
'static',
filename='images/flags/unknown.png')
country_html = '<img src="{0}" class="country_flag"> {1}'.format(
flag_path, domain['country'])
domains_table += '<tr><td>{4}</td><td>{0}</td><td>{2}</td><td>{1}</td><td><button class="bttn-fill bttn-xs bttn-danger" onclick=whois("{0}")><i class="fab fa-searchengin"></i> WHOIS</button> <button class="bttn-fill bttn-xs bttn-primary" onclick="domainvt(\'{0}\', \'{3}\')"><i class="fas fa-hourglass-end"></i> VT Report</button> <button class="bttn-fill bttn-xs bttn-success" onclick=geoip("{2}")><i class="fas fa-globe-americas"></i> Geo-IP Lookup</button></td></tr>'.format(
domain['name'], '0/66', domain['ip'],
analysis_id, country_html)
domains_table += '</tbody></table>'
else:
domains_table = '<h3 class="nothing"> No Domains Extracted! </h3>'
unique_domains = len(report_data['domains'])
# Permissions div containing all permissions accordions
permissions_div = ""
for perm in report_data['permissions']:
#perm_html = '<div class="perm"><div class="perm-name">{0}</div> <div class="perm-desc">{1}</div> <div class="perm-warn">{2}</div></div>'.format(perm['name'], perm['description'], (perm['warning'] if perm['warning'] != 'na' else ''))
perm_html = '<div class="accordion"><div class="accordion__item"><div class="accordion__question">{0} {1} <div class="risk-pill {4}">{4}</div></div><div class="accordion__answer">{2} <div class="warning"> {3} </div></div></div></div>'.format(
perm['badge'], helper.escape(perm['name']),
perm['description'],
(perm['warning']
if perm['warning'] != 'na' else ''), perm['risk'])
permissions_div += perm_html
permissions_count = len(report_data['permissions'])
# table consisting of all the viewable source files
files_table = '<table class="result-table" id="files-table"><thead><tr><th>File Name</th><th>Path</th><th>Size</th><th>Actions</th></tr></thead><tbody>'
for file_info in source_data:
file_name = source_data[file_info]['file_name']
rel_path = source_data[file_info]['relative_path']
file_id = source_data[file_info]['id']
file_size = source_data[file_info]['file_size']
file_action = '<button class="bttn-fill bttn-xs bttn-primary" onclick="viewfile(\'' + analysis_id + '\', \'' + file_id + '\')"><i class="fas fa-code"></i> View Source</button>'
if file_name.endswith('.js'):
# Add button for viewing retirejs vulnerability scan results
file_action += ' <button class="bttn-fill bttn-xs bttn-danger" onclick="retirejsResult({0}, {1}, {2})"><i class="fas fa-spider"></i> Vulnerabilities</button>'.format(
"'" + file_id + "'", "'" + analysis_id + "'",
"'" + file_name + "'")
file_type = helper.fixpath(core.path +
'/static/images/' +
file_name.split('.')[-1] +
'1.png')
if os.path.isfile(file_type):
file_type = file_name.split('.')[-1] + '1.png'
else:
file_type = 'other1.png'
file_type = url_for('static',
filename='images/' + file_type)
file_type = '<img src="{0}" class="ft_icon">'.format(
file_type)
file_html = "<tr><td>{2} {0}</td><td>{1}</td><td>{4}</td><td>{3}</td></tr>".format(
file_name, rel_path, file_type, file_action,
file_size)
files_table += file_html
files_table += '</tbody></table>'
# table consisting of ipv6 and ipv4 addresses
if report_data['ipv4_addresses'] == [] and report_data[
'ipv6_addresses'] == []:
ips_table = '<h3 class="nothing">No IPv4 or IPv6 addresses found!</h3>'
else:
ips_table = '<table class="result-table" id="ips_table"><thead><tr><th>IP Address</th><th>Type</th><th>File</th></tr></thead><tbody>'
for ip in report_data['ipv4_addresses']:
ips_table += '<tr><td>{0}</td><td>{1}</td><td>{2}</td></tr>'.format(
ip['address'], 'IPv4', ip['file'])
for ip in report_data['ipv6_addresses']:
ips_table += '<tr><td>{0}</td><td>{1}</td><td>{2}</td></tr>'.format(
ip['address'], 'IPv6', ip['file'])
ips_table += '</tbody></table>'
# table consisting of emails
if report_data['emails'] != []:
mails_table = '<table class="result-table" id="mails_table"><thead><tr><th>Email Address</th><th>File</th></tr></thead><tbody>'
for mail in report_data['emails']:
mails_table += '<tr><td>{0}</td><td>{1}</td></tr>'.format(
mail['mail'], mail['file'])
mails_table += '</tbody></table>'
else:
mails_table = '<h3 class="nothing">No email addresses found in any of the files!</h3>'
# table containing btc addresses
if report_data['bitcoin_addresses'] != []:
btc_table = '<table class="result-table" id="btc_table"><thead><tr><th>BTC Address</th><th>File</th></tr></thead><tbody>'
for mail in report_data['bitcoin_addresses']:
btc_table += '<tr><td>{0}</td><td>{1}</td></tr>'.format(
mail['address'], mail['file'])
btc_table += '</tbody></table>'
else:
btc_table = '<h3 class="nothing">No Bitcoin Address found!</h3>'
# table containing comments
if report_data['comments'] != []:
comments_table = '<table class="result-table" id="comments_table"><thead><tr><th>Comment</th><th>File</th></tr></thead><tbody>'
for comment in report_data['comments']:
comments_table += '<tr><td>{0}</td><td>{1}</td></tr>'.format(
helper.escape(comment['comment']),
comment['file'])
comments_table += '</tbody></table>'
else:
comments_table = '<h3 class="nothing">No comments found in any js/html/css files!</h3>'
# table containing base64 encoded strings
if report_data['base64_strings'] != []:
base64_table = '<table class="result-table" id="base64_table"><thead><tr><th>Base64 Encoded String</th><th>File</th></tr></thead><tbody>'
for b64 in report_data['base64_strings']:
base64_table += '<tr><td>{0}</td><td>{1}</td></tr>'.format(
b64['string'], b64['file'])
base64_table += '</tbody></table>'
else:
base64_table = '<h3 class="nothing">No base64 encoded string found in any js/html/css files!</h3>'
manifest_content = json.dumps(report_data['manifest'])
'''
Files count
'''
js_files_count = len(report_data['files']['js'])
css_files_count = len(report_data['files']['css'])
html_files_count = len(report_data['files']['html'])
json_files_count = len(report_data['files']['json'])
other_files_count = len(report_data['files']['other'])
static_files_count = len(report_data['files']['static'])
return render_template(
"report.html",
extension_type=extension_type,
graph_data=graph_data,
basic_info=basic_info_t,
urls_table=urls_table,
permissions_div=permissions_div,
analysis_id=analysis_id,
files_table=files_table,
manifest_content=manifest_content,
domains_table=domains_table,
base64_table=base64_table,
comments_table=comments_table,
ips_table=ips_table,
btc_table=btc_table,
mails_table=mails_table,
extjs_table=extjs_table,
urls_count=urls_count,
extjs_count=extjs_count,
permissions_count=permissions_count,
unique_domains=unique_domains,
js_files_count=js_files_count,
css_files_count=css_files_count,
html_files_count=html_files_count,
json_files_count=json_files_count,
other_files_count=other_files_count,
static_files_count=static_files_count)
else:
error_txt = 'All the result files are not found.. Try scanning the extension again! and don\'t mess with the result files this time'
return render_template('error.html',
error_title="Malformed Result",
error_head="Incomplete Result",
error_txt=error_txt)
else:
error_txt = 'The result directory corresponding to result id {0} could not be found... hence ExtAnalysis has nothing to show'.format(
analysis_id)
return render_template(
'error.html',
error_title="Result Directory Not Found",
error_head="Result Directory Not Foundt",
error_txt=error_txt)
except:
logging.error(traceback.format_exc())
return render_template(
'error.html',
error_title="Invalid Result ID",
error_head="Invalid Result ID",
error_txt=
'There seems to be no result corresponding to the provided ID. Did you delete the result? or maybe you did some weird shit with the parameter?'
)
def GetDescriptionFromManifest(manifest_file):
# Get's the desc of an extension from manifest file
if os.path.isfile(manifest_file):
# Path is valid and it's a manifest file
extension_path = helper.fixpath(
manifest_file.replace('manifest.json', ''))
manifest_content = open(manifest_file, 'r', encoding="utf8")
manifest_content = manifest_content.read()
try:
# load the json data
manifest_json = json.loads(manifest_content)
try:
manifest_desc = manifest_json['description']
if '__MSG_' in manifest_desc:
# This is the whole reason i created this function...
updatelog(
'Getting manifest description from locale file: ' +
extension_path)
manifest_message = re.findall('__MSG_(.*?)__',
manifest_desc)[0]
locale_dir = helper.fixpath(extension_path + '/_locales/')
if os.path.isdir(locale_dir):
# locale directory exists let' s grab our thing
try:
# get the default locale from manifest
default_locale = manifest_json['default_locale']
en_locale_file = helper.fixpath(locale_dir + '/' +
default_locale +
'/messages.json')
updatelog('Default Locale: ' + default_locale)
except Exception as e:
# use hardcoded en
en_locale_file = helper.fixpath(
locale_dir + '/en/messages.json')
if os.path.isfile(en_locale_file):
# en locale file found let's grab the desc
en_locale_content = open(en_locale_file,
'r',
encoding="utf8")
try:
en_locale_content = json.loads(
en_locale_content.read())
string_content = str(en_locale_content)
ext_desc = ''
if manifest_message in string_content:
ext_desc = en_locale_content[
manifest_message]['message']
updatelog(
'Extension description grabbed from default locale file.. description: '
+ ext_desc)
return ext_desc
else:
updatelog('Could not find description')
return False
except Exception as e:
updatelog(
'Something went wrong while reading or parsing default locale file...'
)
logging.error(traceback.format_exc())
return False
else:
# en locale not found let's just get the first one we find and be done with it
ldirs = os.listdir(locale_dir)
for dir in ldirs:
if os.path.isfile(
helper.fixpath(locale_dir + '/' + dir +
'/messages.json')):
locale_content = open(
helper.fixpath(locale_dir + '/' + dir +
'/messages.json'),
'r',
encoding="utf8")
try:
en_locale_content = json.loads(
locale_content.read())
if manifest_message in locale_content:
ext_desc = en_locale_content[
manifest_message]['message']
updatelog(
'Extension description grabbed from '
+ dir +
' locale file.. description: '
+ ext_desc)
return ext_desc
else:
updatelog(
'Could not find description')
return False
except Exception as e:
updatelog(
'Something went wrong while reading or parsing en locale file...'
)
logging.error(traceback.format_exc())
return False
else:
# _locale dir doesn't exist let's just go...
updatelog('_locale directory doesn\'t exist.. dir: ' +
locale_dir)
return False
else:
return manifest_desc
except Exception as e:
updatelog(
'No description in on manifest.json ... maybe an invalid extension?'
)
logging.error(traceback.format_exc())
return False
except Exception as e:
updatelog(
'Something went wrong while loading manifest json [GetDescriptionFromManifest]'
)
updatelog('Error: ' + str(e))
logging.error(traceback.format_exc())
return False