def _start_single_file_analysis(self, uid, selected_analysis=None, root_uid=None):
with ConnectTo(FrontEndDbInterface, self._config) as database:
file_object = database.get_object(uid)
file_object.scheduled_analysis = request.form.getlist('analysis_systems')
with ConnectTo(InterComFrontEndBinding, self._config) as intercom:
intercom.add_single_file_task(file_object)
return redirect(url_for(self.show_analysis.__name__, uid=uid, root_uid=root_uid, selected_analysis=selected_analysis))
def _app_show_browse_compare(self):
page, per_page = extract_pagination_from_request(
request, self._config)[0:2]
try:
with ConnectTo(CompareDbInterface, self._config) as db_service:
compare_list = db_service.page_compare_results(skip=per_page *
(page - 1),
limit=per_page)
except Exception as exception:
error_message = 'Could not query database: {} {}'.format(
type(exception), str(exception))
logging.error(error_message)
return render_template('error.html', message=error_message)
with ConnectTo(CompareDbInterface, self._config) as connection:
total = connection.get_total_number_of_results()
pagination = get_pagination(page=page,
per_page=per_page,
total=total,
record_name='compare results')
return render_template('database/compare_browse.html',
compare_list=compare_list,
page=page,
per_page=per_page,
pagination=pagination)
def get(self, uid=None):
if not uid:
paging, success = get_paging(request.args)
if not success:
return error_message(paging, self.URL, request_data=request.args)
offset, limit = paging
try:
query = get_query(request.args)
except ValueError as value_error:
return error_message(str(value_error), self.URL, request_data=dict(query=request.args.get('query')))
try:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
uids = connection.rest_get_file_object_uids(offset=offset, limit=limit, query=query)
return success_message(dict(uids=uids), self.URL, dict(offset=offset, limit=limit, query=query))
except Exception:
return error_message('Unknown exception on request', self.URL, dict(offset=offset, limit=limit, query=query))
else:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
file_object = connection.get_file_object(uid)
if not file_object:
return error_message('No file object with UID {} found'.format(uid), self.URL, dict(uid=uid))
fitted_file_object = self._fit_file_object(file_object)
return success_message(dict(file_object=fitted_file_object), self.URL, request_data=dict(uid=uid))
def get(self, uid):
'''
The uid of the file_object in question has to be given in the url
The return format will be {"binary": b64_encoded_binary, "file_name": file_name}
'''
with ConnectTo(FrontEndDbInterface, self.config) as db_service:
existence = db_service.existence_quick_check(uid)
if not existence:
return error_message('No firmware with UID {} found in database'.format(uid), self.URL, request_data={'uid': uid}, return_code=404)
try:
tar_flag = get_boolean_from_request(request.args, 'tar')
except ValueError as value_error:
return error_message(str(value_error), self.URL, request_data=dict(uid=uid, tar=request.args.get('tar')))
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
if not tar_flag:
binary, file_name = intercom.get_binary_and_filename(uid)
else:
binary, file_name = intercom.get_repacked_binary_and_file_name(uid)
response = {
'binary': standard_b64encode(binary).decode(),
'file_name': file_name,
'SHA256': get_sha256(binary)
}
return success_message(response, self.URL, request_data={'uid': uid, 'tar': tar_flag})
def _download_base64_decoded_section(self, uid, section, expression_id):
with ConnectTo(FrontEndDbInterface, self._config) as sc:
file_obj = sc.get_object(uid, analysis_filter=['base64_decoder'])
span_in_binary, span_in_section = None, (None, None)
for expression in file_obj.processed_analysis['base64_decoder'][
section]:
if expression['id'] == int(expression_id):
span_in_section = expression['span_in_section']
span_in_binary = expression['span_in_binary']
break
if not span_in_binary:
return render_template(
'error.html', message='Undisclosed error in base64 decoding')
with ConnectTo(InterComFrontEndBinding, self._config) as connection:
raw_binary = connection.get_binary_and_filename(file_obj.uid)
binary, _ = remove_linebreaks_from_byte_string(
raw_binary[0][span_in_binary[0]:span_in_binary[1]])
try:
binary = binascii.a2b_base64(
binary[span_in_section[0]:span_in_section[1]])
except binascii.Error as error:
return render_template('error.html', message=str(error))
response = make_response(binary)
file_name = '{}_0x{:X}-0x{:X}_decoded'.format(
file_obj.file_name, span_in_binary[0] + span_in_section[0],
span_in_binary[1] - span_in_section[2])
response.headers[
'Content-Disposition'] = 'attachment; filename={}'.format(
file_name)
return response
def _app_upload(self):
error = {}
if request.method == 'POST':
analysis_task = create_analysis_task(request)
error = check_for_errors(analysis_task)
if not error:
fw = convert_analysis_task_to_fw_obj(analysis_task)
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
sc.add_analysis_task(fw)
return render_template('upload/upload_successful.html',
uid=analysis_task['uid'])
with ConnectTo(FrontEndDbInterface, self._config) as sc:
device_class_list = sc.get_device_class_list()
vendor_list = sc.get_vendor_list()
device_name_dict = sc.get_device_name_dict()
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
analysis_plugins = sc.get_available_analysis_plugins()
analysis_presets = [key for key in self._config['default_plugins']]
return render_template('upload/upload.html',
device_classes=device_class_list,
vendors=vendor_list,
error=error,
analysis_presets=analysis_presets,
device_names=json.dumps(device_name_dict,
sort_keys=True),
analysis_plugin_dict=analysis_plugins)
def _app_show_start_compare(self):
if 'uids_for_comparison' not in session or not isinstance(
session['uids_for_comparison'],
list) or len(session['uids_for_comparison']) < 2:
return render_template('compare/error.html',
error='No UIDs found for comparison')
compare_id = convert_uid_list_to_compare_id(
session['uids_for_comparison'])
session['uids_for_comparison'] = None
redo = True if request.args.get('force_recompare') else None
with ConnectTo(CompareDbInterface, self._config) as sc:
compare_exists = sc.compare_result_is_in_db(compare_id)
if compare_exists and not redo:
return redirect(
url_for('/compare/<compare_id>', compare_id=compare_id))
try:
with ConnectTo(CompareDbInterface, self._config) as sc:
sc.check_objects_exist(compare_id)
except FactCompareException as exception:
return render_template('compare/error.html',
error=exception.get_message())
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
sc.add_compare_task(compare_id, force=redo)
return render_template('compare/wait.html', compare_id=compare_id)
def _schedule_re_analysis_task(self, uid, analysis_task, re_do):
fw = convert_analysis_task_to_fw_obj(analysis_task)
if re_do:
with ConnectTo(AdminDbInterface, self._config) as sc:
sc.delete_firmware(uid, delete_root_file=False)
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
sc.add_re_analyze_task(fw)
def show_analysis(self, uid, selected_analysis=None, root_uid=None):
other_versions = None
with ConnectTo(CompareDbInterface, self._config) as db_service:
all_comparisons = db_service.page_compare_results()
known_comparisons = [comparison for comparison in all_comparisons if uid in comparison[0]]
analysis_filter = [selected_analysis] if selected_analysis else []
with ConnectTo(FrontEndDbInterface, self._config) as sc:
file_obj = sc.get_object(uid, analysis_filter=analysis_filter)
if not file_obj:
return render_template('uid_not_found.html', uid=uid)
if isinstance(file_obj, Firmware):
root_uid = file_obj.uid
other_versions = sc.get_other_versions_of_firmware(file_obj)
included_fo_analysis_complete = not sc.all_uids_found_in_database(list(file_obj.files_included))
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
analysis_plugins = sc.get_available_analysis_plugins()
return render_template_string(
self._get_correct_template(selected_analysis, file_obj),
uid=uid,
firmware=file_obj,
selected_analysis=selected_analysis,
all_analyzed_flag=included_fo_analysis_complete,
root_uid=none_to_none(root_uid),
analysis_plugin_dict=analysis_plugins,
other_versions=other_versions,
uids_for_comparison=get_comparison_uid_list_from_session(),
user_has_admin_clearance=user_has_privilege(current_user, privilege='delete'),
known_comparisons=known_comparisons,
available_plugins=self._get_used_and_unused_plugins(
file_obj.processed_analysis,
[x for x in analysis_plugins.keys() if x != 'unpacker']
)
)
def get(self):
'''
Request system status
Request a json document showing the system state of FACT, similar to the system health page of the GUI
'''
components = ['frontend', 'database', 'backend']
status = {}
with ConnectTo(StatisticDbViewer, self.config) as stats_db:
for component in components:
status[component] = stats_db.get_statistic(component)
with ConnectTo(InterComFrontEndBinding, self.config) as sc:
plugins = sc.get_available_analysis_plugins()
if not any(bool(status[component]) for component in components):
return error_message(
'Cannot get FACT component status: Database may be down',
self.URL,
return_code=404)
response = {
'system_status': status,
'plugins': self._condense_plugin_information(plugins),
}
return success_message(response, self.URL)
def _update_analysis_get(self, uid, re_do=False, error=None):
with ConnectTo(FrontEndDbInterface, self._config) as sc:
old_firmware = sc.get_firmware(uid=uid, analysis_filter=[])
if old_firmware is None:
return render_template('uid_not_found.html', uid=uid)
device_class_list = sc.get_device_class_list()
vendor_list = sc.get_vendor_list()
device_name_dict = sc.get_device_name_dict()
device_class_list.remove(old_firmware.device_class)
vendor_list.remove(old_firmware.vendor)
device_name_dict[old_firmware.device_class][old_firmware.vendor].remove(old_firmware.device_name)
previously_processed_plugins = list(old_firmware.processed_analysis.keys())
with ConnectTo(InterComFrontEndBinding, self._config) as intercom:
plugin_dict = self._overwrite_default_plugins(intercom.get_available_analysis_plugins(), previously_processed_plugins)
title = 're-do analysis' if re_do else 'update analysis'
return render_template(
'upload/upload.html',
device_classes=device_class_list,
vendors=vendor_list,
error=error if error is not None else {},
device_names=json.dumps(device_name_dict, sort_keys=True),
firmware=old_firmware,
analysis_plugin_dict=plugin_dict,
title=title
)
def _update_analysis(self, uid, update):
with ConnectTo(FrontEndDbInterface, self.config) as connection:
firmware = connection.get_firmware(uid)
if not firmware:
return error_message('No firmware with UID {} found'.format(uid),
self.URL, dict(uid=uid))
unpack = 'unpacker' in update
while 'unpacker' in update:
update.remove('unpacker')
firmware.scheduled_analysis = update
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
supported_plugins = intercom.get_available_analysis_plugins().keys(
)
for item in update:
if item not in supported_plugins:
return error_message(
'Unknown analysis system \'{}\''.format(item),
self.URL, dict(uid=uid, update=update))
intercom.add_re_analyze_task(firmware, unpack)
if unpack:
update.append('unpacker')
return success_message({}, self.URL, dict(uid=uid, update=update))
def put(self):
'''
Start a comparison
For this sake a list of uids of the files, which should be compared, is needed
The `uid_list` must contain uids of already analysed FileObjects or Firmware objects
'''
data = self.validate_payload_data(compare_model)
compare_id = normalize_compare_id(';'.join(data['uid_list']))
with ConnectTo(CompareDbInterface, self.config) as db_compare_service:
if db_compare_service.compare_result_is_in_db(
compare_id) and not data['redo']:
return error_message(
'Compare already exists. Use "redo" to force re-compare.',
self.URL,
request_data=request.json,
return_code=200)
try:
db_compare_service.check_objects_exist(compare_id)
except FactCompareException as exception:
return error_message(exception.get_message(),
self.URL,
request_data=request.json,
return_code=404)
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
intercom.add_compare_task(compare_id, force=data['redo'])
return success_message(
{'message': 'Compare started. Please use GET to get the results.'},
self.URL,
request_data=request.json,
return_code=202)
def get(self, uid=None):
if not uid:
paging, success = get_paging(request.args)
if not success:
return error_message(paging, self.URL, request_data=request.args)
offset, limit = paging
try:
recursive = get_recursive(request.args)
query = get_query(request.args)
except ValueError as value_error:
return error_message(str(value_error), self.URL, request_data=dict(query=request.args.get('query'), recursive=request.args.get('recursive')))
if recursive and not query:
return error_message('recursive search is only permissible with non-empty query', self.URL, request_data=dict(query=request.args.get('query'), recursive=request.args.get('recursive')))
try:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
uids = connection.rest_get_firmware_uids(offset=offset, limit=limit, query=query, recursive=recursive)
return success_message(dict(uids=uids), self.URL, dict(offset=offset, limit=limit, query=query, recursive=recursive))
except Exception:
return error_message('Unknown exception on request', self.URL, dict(offset=offset, limit=limit, query=query, recursive=recursive))
else:
summary = get_summary_flag(request.args)
if summary:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
firmware = connection.get_complete_object_including_all_summaries(uid)
else:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
firmware = connection.get_firmware(uid)
if not firmware or not isinstance(firmware, Firmware):
return error_message('No firmware with UID {} found'.format(uid), self.URL, dict(uid=uid))
fitted_firmware = self._fit_firmware(firmware)
return success_message(dict(firmware=fitted_firmware), self.URL, request_data=dict(uid=uid))
def _app_delete_firmware(self, uid):
with ConnectTo(FrontEndDbInterface, config=self._config) as sc:
is_firmware = sc.is_firmware(uid)
if not is_firmware:
return render_template('error.html', message='Firmware not found in database: {}'.format(uid))
with ConnectTo(AdminDbInterface, config=self._config) as sc:
deleted_virtual_path_entries, deleted_files = sc.delete_firmware(uid)
return render_template('delete_firmware.html', deleted_vps=deleted_virtual_path_entries, deleted_files=deleted_files, uid=uid)
def _start_single_file_analysis(self, uid):
if user_has_privilege(current_user, privilege='submit_analysis'):
with ConnectTo(FrontEndDbInterface, self._config) as database:
file_object = database.get_object(uid)
file_object.scheduled_analysis = request.form.getlist('analysis_systems')
with ConnectTo(InterComFrontEndBinding, self._config) as intercom:
intercom.add_single_file_task(file_object)
else:
flash('You have insufficient rights to add additional analyses')
def _app_home(self):
stats = StatisticUpdater(config=self._config)
with ConnectTo(FrontEndDbInterface, config=self._config) as sc:
latest_firmware_submissions = sc.get_last_added_firmwares(int(self._config['database'].get('number_of_latest_firmwares_to_display', '10')))
latest_comments = sc.get_latest_comments(int(self._config['database'].get('number_of_latest_firmwares_to_display', '10')))
with ConnectTo(CompareDbInterface, config=self._config) as sc:
latest_comparison_results = sc.page_compare_results(limit=10)
general_stats = stats.get_general_stats()
stats.shutdown()
return render_template('home.html', general_stats=general_stats, latest_firmware_submissions=latest_firmware_submissions,
latest_comments=latest_comments, latest_comparison_results=latest_comparison_results)
def _show_system_health(self):
components = ['frontend', 'database', 'backend']
status = []
with ConnectTo(StatisticDbViewer, self._config) as stats_db:
for component in components:
status.append(stats_db.get_statistic(component))
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
plugin_dict = sc.get_available_analysis_plugins()
return render_template('system_health.html', status=status, analysis_plugin_info=plugin_dict)
def _get_data_for_file_diff(self, uid: str,
root_uid: Optional[str]) -> FileDiffData:
with ConnectTo(InterComFrontEndBinding, self._config) as db:
content, _ = db.get_binary_and_filename(uid)
with ConnectTo(FrontEndDbInterface, self._config) as db:
fo = db.get_object(uid)
if root_uid in [None, 'None']:
root_uid = fo.get_root_uid()
fw_hid = db.get_object(root_uid).get_hid()
mime = fo.processed_analysis.get('file_type', {}).get('mime')
return FileDiffData(uid, content.decode(errors='replace'),
fo.file_name, mime, fw_hid)
def _app_add_comment(self, uid):
error = False
if request.method == 'POST':
comment = request.form['comment']
author = request.form['author']
with ConnectTo(FrontendEditingDbInterface, config=self._config) as sc:
sc.add_comment_to_object(uid, comment, author, round(time()))
return redirect(url_for('analysis/<uid>', uid=uid))
with ConnectTo(FrontEndDbInterface, config=self._config) as sc:
if not sc.existence_quick_check(uid):
error = True
return render_template('add_comment.html', uid=uid, error=error)
def _schedule_re_analysis_task(self, uid, analysis_task, re_do, force_reanalysis=False):
if re_do:
base_fw = None
with ConnectTo(AdminDbInterface, self._config) as sc:
sc.delete_firmware(uid, delete_root_file=False)
else:
with ConnectTo(FrontEndDbInterface, self._config) as db:
base_fw = db.get_firmware(uid)
base_fw.force_update = force_reanalysis
fw = convert_analysis_task_to_fw_obj(analysis_task, base_fw=base_fw)
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
sc.add_re_analyze_task(fw, unpack=re_do)
def get_upload(self, error=None):
error = error or {}
with ConnectTo(FrontEndDbInterface, self._config) as sc:
device_class_list = sc.get_device_class_list()
vendor_list = sc.get_vendor_list()
device_name_dict = sc.get_device_name_dict()
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
analysis_plugins = sc.get_available_analysis_plugins()
return render_template(
'upload/upload.html',
device_classes=device_class_list, vendors=vendor_list, error=error,
analysis_presets=list(self._config['default_plugins']),
device_names=json.dumps(device_name_dict, sort_keys=True), analysis_plugin_dict=analysis_plugins
)
def put(self):
'''
The request data should have the form
{"uid_list": uid_list, "<optional>redo": True}
return value: the result dict from the compare
'''
try:
data = convert_rest_request(request.data)
except TypeError as type_error:
return error_message(str(type_error),
self.URL,
request_data=request.data)
try:
uid_string = ';'.join(data['uid_list'])
compare_id = normalize_compare_id(uid_string)
redo = data.get('redo', False)
except (AttributeError, TypeError, KeyError):
return error_message(
'Request should be of the form {"uid_list": uid_list, "redo": boolean}',
self.URL,
request_data=data)
with ConnectTo(CompareDbInterface, self.config) as db_compare_service:
if not db_compare_service.compare_result_is_in_db(
compare_id) or redo:
try:
db_compare_service.check_objects_exist(compare_id)
except FactCompareException as exception:
return error_message(exception.get_message(),
self.URL,
request_data=data,
return_code=404)
with ConnectTo(InterComFrontEndBinding,
self.config) as intercom:
intercom.add_compare_task(compare_id, force=redo)
return success_message(
{
'message':
'Compare started. Please use GET to get the results.'
},
self.URL,
request_data=data,
return_code=202)
return error_message(
'Compare already exists. Use "redo" to force re-compare.',
self.URL,
request_data=data,
return_code=200)
def _app_get_binary_search_results(self):
firmware_dict, error, yara_rules = None, None, None
if request.args.get('request_id'):
request_id = request.args.get('request_id')
with ConnectTo(InterComFrontEndBinding,
self._config) as connection:
result, yara_rules = connection.get_binary_search_result(
request_id)
if isinstance(result, str):
error = result
elif result is not None:
yara_rules = make_unicode_string(yara_rules[0])
joined_results = self._join_results(result)
query_uid = self._store_binary_search_query(
joined_results, yara_rules)
return redirect(
url_for('database/browse',
query=query_uid,
only_firmwares=request.args.get('only_firmware')))
else:
error = 'No request ID found'
request_id = None
return render_template('database/database_binary_search_results.html',
result=firmware_dict,
error=error,
request_id=request_id,
yara_rules=yara_rules)
def _sync_view(self, plugin_path):
if plugin_path:
view_source = self._get_view_file_path(plugin_path)
if view_source is not None:
view = get_binary_from_file(view_source)
with ConnectTo(ViewUpdater, self.config) as connection:
connection.update_view(self.NAME, view)
def _search_database(self,
query,
skip=0,
limit=0,
only_firmwares=False,
inverted=False):
sorted_meta_list = list()
with ConnectTo(FrontEndDbInterface, self._config) as connection:
result = connection.generic_search(
query,
skip,
limit,
only_fo_parent_firmware=only_firmwares,
inverted=inverted)
if not isinstance(result, list):
raise Exception(result)
if query not in ('{}', {}):
firmware_list = [
connection.firmwares.find_one(uid)
or connection.file_objects.find_one(uid) for uid in result
]
else: # if search query is empty: get only firmware objects
firmware_list = [
connection.firmwares.find_one(uid) for uid in result
]
sorted_meta_list = sorted(connection.get_meta_list(firmware_list),
key=lambda x: x[1].lower())
return sorted_meta_list
def _get_search_parameters(self, query, only_firmware, inverted):
'''
This function prepares the requested search by parsing all necessary parameters.
In case of a binary search, indicated by the query being an uid instead of a dict, the cached search result is
retrieved.
'''
search_parameters = dict()
if request.args.get('query'):
query = request.args.get('query')
if is_uid(query):
with ConnectTo(FrontEndDbInterface,
self._config) as connection:
cached_query = connection.get_query_from_cache(query)
query = cached_query['search_query']
search_parameters['query_title'] = cached_query[
'query_title']
search_parameters['only_firmware'] = request.args.get(
'only_firmwares') == 'True' if request.args.get(
'only_firmwares') else only_firmware
search_parameters['inverted'] = request.args.get(
'inverted') == 'True' if request.args.get('inverted') else inverted
search_parameters['query'] = apply_filters_to_query(request, query)
if 'query_title' not in search_parameters.keys():
search_parameters['query_title'] = search_parameters['query']
if request.args.get('date'):
search_parameters['query'] = self._add_date_to_query(
search_parameters['query'], request.args.get('date'))
return search_parameters
def _ajax_get_tree_root(self, uid, root_uid):
root = list()
with ConnectTo(FrontEndDbInterface, self._config) as sc:
for node in sc.generate_file_tree_level(uid, root_uid): # only a single item in this 'iterable'
root = [self._generate_jstree_node(node)]
root = remove_virtual_path_from_root(root)
return jsonify(root)
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
def show_logs(self):
with ConnectTo(InterComFrontEndBinding, self._config) as sc:
backend_logs = '\n'.join(sc.get_backend_logs())
frontend_logs = '\n'.join(self._get_frontend_logs())
return render_template('logs.html',
backend_logs=backend_logs,
frontend_logs=frontend_logs)