def test_success_message_bad_type():
with pytest.raises(TypeError):
success_message(None, '/any/url')
with pytest.raises(TypeError):
success_message('Done some stuff. Didn\'t look for standards.',
'/any/url')
def test_success_message_succeeds():
message, code = success_message({'my': 'response'}, '/any/url')
assert message['my'] == 'response'
assert message['status'] == 0
assert code == 200
_, code = success_message({'my': 'response'}, '/any/url', return_code=202)
assert code == 202
def _get_without_uid(self):
try:
query, recursive, inverted, offset, limit = self._get_parameters_from_request(
request.args)
except ValueError as value_error:
request_data = {
k: request.args.get(k)
for k in ['query', 'limit', 'offset', 'recursive', 'inverted']
}
return error_message(str(value_error),
self.URL,
request_data=request_data)
parameters = dict(offset=offset,
limit=limit,
query=query,
recursive=recursive,
inverted=inverted)
try:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
uids = connection.rest_get_firmware_uids(**parameters)
return success_message(dict(uids=uids), self.URL, parameters)
except PyMongoError:
return error_message('Unknown exception on request', self.URL,
parameters)
def _update_analysis(self, uid, update):
with ConnectTo(FrontEndDbInterface, self.config) as connection:
firmware = connection.get_firmware(uid)
if not firmware:
return error_message('No firmware with UID {} found'.format(uid),
self.URL, dict(uid=uid))
unpack = 'unpacker' in update
while 'unpacker' in update:
update.remove('unpacker')
firmware.scheduled_analysis = update
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
supported_plugins = intercom.get_available_analysis_plugins().keys(
)
for item in update:
if item not in supported_plugins:
return error_message(
'Unknown analysis system \'{}\''.format(item),
self.URL, dict(uid=uid, update=update))
intercom.add_re_analyze_task(firmware, unpack)
if unpack:
update.append('unpacker')
return success_message({}, self.URL, dict(uid=uid, update=update))
def get(self, compare_id):
'''
Request results from a comparisons
The result can be requested by providing a semicolon separated list of uids as compare_id
The response will contain a json_document with the comparison result, along with the fields status, timestamp,
request_resource and request as meta data
'''
try:
self._validate_compare_id(compare_id)
compare_id = normalize_compare_id(compare_id)
except (TypeError, ValueError) as error:
return error_message(
f'Compare ID must be of the form uid1;uid2(;uid3..): {error}',
self.URL,
request_data={'compare_id': compare_id})
with ConnectTo(CompareDbInterface, self.config) as db_compare_service:
result = None
with suppress(FactCompareException):
if db_compare_service.compare_result_is_in_db(compare_id):
result = db_compare_service.get_compare_result(compare_id)
if result:
return success_message(result,
self.URL,
request_data={'compare_id': compare_id},
return_code=202)
return error_message(
'Compare not found in database. Please use /rest/start_compare to start the compare.',
self.URL,
request_data={'compare_id': compare_id},
return_code=404)
def get(self, compare_id=None):
'''
The request data should have the form
{"uid_list": uid_list, "<optional>redo": True}
return value: the result dict from the compare
'''
try:
compare_id = normalize_compare_id(compare_id)
except (AttributeError, TypeError):
return error_message(
'Compare ID must be of the form uid1;uid2(;uid3..)',
self.URL,
request_data={'compare_id': compare_id})
with ConnectTo(CompareDbInterface, self.config) as db_compare_service:
result = None
with suppress(FactCompareException):
if db_compare_service.compare_result_is_in_db(compare_id):
result = db_compare_service.get_compare_result(compare_id)
if result:
return success_message(result,
self.URL,
request_data={'compare_id': compare_id},
return_code=202)
return error_message(
'Compare not found in database. Please use /rest/start_compare to start the compare.',
self.URL,
request_data={'compare_id': compare_id},
return_code=404)
def get(self, uid):
'''
The uid of the file_object in question has to be given in the url
The return format will be {"binary": b64_encoded_binary, "file_name": file_name}
'''
with ConnectTo(FrontEndDbInterface, self.config) as db_service:
existence = db_service.existence_quick_check(uid)
if not existence:
return error_message('No firmware with UID {} found in database'.format(uid), self.URL, request_data={'uid': uid}, return_code=404)
try:
tar_flag = get_boolean_from_request(request.args, 'tar')
except ValueError as value_error:
return error_message(str(value_error), self.URL, request_data=dict(uid=uid, tar=request.args.get('tar')))
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
if not tar_flag:
binary, file_name = intercom.get_binary_and_filename(uid)
else:
binary, file_name = intercom.get_repacked_binary_and_file_name(uid)
response = {
'binary': standard_b64encode(binary).decode(),
'file_name': file_name,
'SHA256': get_sha256(binary)
}
return success_message(response, self.URL, request_data={'uid': uid, 'tar': tar_flag})
def post(self):
'''
The request data should have the form
{"rule_file": rule_string, 'uid': firmware_uid}
The uid parameter is optional and can be specified if the user want's to search in the files of a single firmware.
rule_string can be something like "rule rule_name {strings: $a = \"foobar\" condition: $a}"
'''
try:
data = convert_rest_request(request.data)
yara_rules = self._get_yara_rules(data)
uid = self._get_firmware_uid(data)
except TypeError as type_error:
return error_message(str(type_error),
self.URL,
request_data=request.data)
except RestBinarySearchException as exception:
return error_message(exception.get_message(),
self.URL,
request_data=request.data)
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
search_id = intercom.add_binary_search_request(yara_rules, uid)
return success_message(
{
'message':
'Started binary search. Please use GET and the search_id to get the results'
},
self.URL,
request_data={'search_id': search_id})
def post(self):
'''
Start a binary search
The parameter `uid` is optional and can be specified if the user wants to search the files of a single firmware
`rule_file` can be something like `rule rule_name {strings: $a = \"foobar\" condition: $a}`
'''
payload_data = self.validate_payload_data(binary_search_model)
if not is_valid_yara_rule_file(payload_data["rule_file"]):
return error_message('Error in YARA rule file',
self.URL,
request_data=request.data)
if payload_data["uid"] and not self._is_firmware(payload_data["uid"]):
return error_message(
f'Firmware with UID {payload_data["uid"]} not found in database',
self.URL,
request_data=request.data)
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
search_id = intercom.add_binary_search_request(
payload_data["rule_file"].encode(), payload_data["uid"])
return success_message(
{
'message':
'Started binary search. Please use GET and the search_id to get the results'
},
self.URL,
request_data={'search_id': search_id})
def put(self):
'''
Start a comparison
For this sake a list of uids of the files, which should be compared, is needed
The `uid_list` must contain uids of already analysed FileObjects or Firmware objects
'''
data = self.validate_payload_data(compare_model)
compare_id = normalize_compare_id(';'.join(data['uid_list']))
with ConnectTo(CompareDbInterface, self.config) as db_compare_service:
if db_compare_service.compare_result_is_in_db(
compare_id) and not data['redo']:
return error_message(
'Compare already exists. Use "redo" to force re-compare.',
self.URL,
request_data=request.json,
return_code=200)
try:
db_compare_service.check_objects_exist(compare_id)
except FactCompareException as exception:
return error_message(exception.get_message(),
self.URL,
request_data=request.json,
return_code=404)
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
intercom.add_compare_task(compare_id, force=data['redo'])
return success_message(
{'message': 'Compare started. Please use GET to get the results.'},
self.URL,
request_data=request.json,
return_code=202)
def get(self):
'''
Request system status
Request a json document showing the system state of FACT, similar to the system health page of the GUI
'''
components = ['frontend', 'database', 'backend']
status = {}
with ConnectTo(StatisticDbViewer, self.config) as stats_db:
for component in components:
status[component] = stats_db.get_statistic(component)
with ConnectTo(InterComFrontEndBinding, self.config) as sc:
plugins = sc.get_available_analysis_plugins()
if not any(bool(status[component]) for component in components):
return error_message(
'Cannot get FACT component status: Database may be down',
self.URL,
return_code=404)
response = {
'system_status': status,
'plugins': self._condense_plugin_information(plugins),
}
return success_message(response, self.URL)
def get(self):
with ConnectTo(FrontEndDbInterface, self.config) as db:
missing_analyses_data = {
'missing_files': self._make_json_serializable(db.find_missing_files()),
'missing_analyses': self._make_json_serializable(db.find_missing_analyses()),
'failed_analyses': db.find_failed_analyses(),
'orphaned_objects': db.find_orphaned_objects(),
}
return success_message(missing_analyses_data, self.URL)
def get(self, uid):
results = get_analysis_results_for_included_uid(uid, self.config)
endpoint = self.ENDPOINTS[0][0]
if not results:
error_message('no results found for uid {}'.format(uid),
endpoint,
request_data={'uid': uid})
return success_message({AnalysisPlugin.NAME: results},
endpoint,
request_data={'uid': uid})
def get(self, uid):
with ConnectTo(FsMetadataRoutesDbInterface, self.config) as db:
results = db.get_analysis_results_for_included_uid(uid)
endpoint = self.ENDPOINTS[0][0]
if not results:
error_message('no results found for uid {}'.format(uid),
endpoint,
request_data={'uid': uid})
return success_message({AnalysisPlugin.NAME: results},
endpoint,
request_data={'uid': uid})
def test_messages_with_request_data():
request_data = {'for_example': 'some uids'}
message, _ = success_message({'my': 'data'},
'/any/url',
request_data=request_data)
assert message['request'] == request_data
message, _ = error_message('my_error',
'/any/url',
request_data=request_data)
assert message['request'] == request_data
def _get_with_uid(self, uid):
with ConnectTo(FrontEndDbInterface, self.config) as connection:
file_object = connection.get_file_object(uid)
if not file_object:
return error_message(
'No file object with UID {} found'.format(uid), self.URL,
dict(uid=uid))
fitted_file_object = self._fit_file_object(file_object)
return success_message(dict(file_object=fitted_file_object),
self.URL,
request_data=dict(uid=uid))
def get(self, search_id=None):
'''
Get the results of a previously initiated binary search
The `search_id` is needed to fetch the corresponding search result
The result of the search request can only be fetched once
After this the search needs to be started again.
'''
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
result, _ = intercom.get_binary_search_result(search_id)
if result is None:
return error_message('The result is not ready yet or it has already been fetched', self.URL)
return success_message({'binary_search_results': result}, self.URL)
def start_compare(self, db_compare_service, compare_id, data, redo):
try:
db_compare_service.check_objects_exist(compare_id)
except FactCompareException as exception:
return error_message(exception.get_message(),
self.URL,
request_data=data,
return_code=404)
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
intercom.add_compare_task(compare_id, force=redo)
return success_message(
{'message': 'Compare started. Please use GET to get the results.'},
self.URL,
request_data=data,
return_code=202)
def get(self, uid):
'''
Request a specific file
Get the analysis results of a specific file by providing the corresponding uid
'''
with ConnectTo(FrontEndDbInterface, self.config) as connection:
file_object = connection.get_file_object(uid)
if not file_object:
return error_message(
'No file object with UID {} found'.format(uid), self.URL,
dict(uid=uid))
fitted_file_object = self._fit_file_object(file_object)
return success_message(dict(file_object=fitted_file_object),
self.URL,
request_data=dict(uid=uid))
def _get_with_uid(self, uid):
summary = get_boolean_from_request(request.args, 'summary')
if summary:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
firmware = connection.get_complete_object_including_all_summaries(
uid)
else:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
firmware = connection.get_firmware(uid)
if not firmware or not isinstance(firmware, Firmware):
return error_message('No firmware with UID {} found'.format(uid),
self.URL, dict(uid=uid))
fitted_firmware = self._fit_firmware(firmware)
return success_message(dict(firmware=fitted_firmware),
self.URL,
request_data=dict(uid=uid))
def _put_without_uid(self):
try:
data = convert_rest_request(request.data)
except TypeError as type_error:
return error_message(str(type_error),
self.URL,
request_data=request.data)
result = self._process_data(data)
if 'error_message' in result:
logging.warning(
'Submission not according to API guidelines! (data could not be parsed)'
)
return error_message(result['error_message'],
self.URL,
request_data=data)
logging.debug('Upload Successful!')
return success_message(result, self.URL, request_data=data)
def get(self):
components = ["frontend", "database", "backend"]
status = {}
with ConnectTo(StatisticDbViewer, self.config) as stats_db:
for component in components:
status[component] = stats_db.get_statistic(component)
with ConnectTo(InterComFrontEndBinding, self.config) as sc:
plugins = sc.get_available_analysis_plugins()
if not status:
return error_message('Unknown Issue. Cannot Stat FACT.',
self.URL,
return_code=404)
response = {
'system_status': status,
'plugins': self._condense_plugin_information(plugins),
}
return success_message(response, self.URL)
def _get_without_uid(self):
try:
query = get_query(request.args)
offset, limit = get_paging(request.args)
except ValueError as value_error:
request_data = {
k: request.args.get(k)
for k in ['query', 'limit', 'offset']
}
return error_message(str(value_error),
self.URL,
request_data=request_data)
parameters = dict(offset=offset, limit=limit, query=query)
try:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
uids = connection.rest_get_file_object_uids(**parameters)
return success_message(dict(uids=uids), self.URL, parameters)
except PyMongoError:
return error_message('Unknown exception on request', self.URL,
parameters)
def get(self, uid):
'''
Request a specific firmware
Get the analysis results of a specific firmware by providing the corresponding uid
'''
summary = get_boolean_from_request(request.args, 'summary')
if summary:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
firmware = connection.get_complete_object_including_all_summaries(
uid)
else:
with ConnectTo(FrontEndDbInterface, self.config) as connection:
firmware = connection.get_firmware(uid)
if not firmware or not isinstance(firmware, Firmware):
return error_message(f'No firmware with UID {uid} found', self.URL,
dict(uid=uid))
fitted_firmware = self._fit_firmware(firmware)
return success_message(dict(firmware=fitted_firmware),
self.URL,
request_data=dict(uid=uid))
def put(self):
'''
Upload a firmware
The HTTP body must contain a json document of the structure shown below
Important: The binary has to be a base64 string representing the raw binary you want to submit
'''
try:
data = self.validate_payload_data(firmware_model)
except MarshallingError as error:
logging.error(f'REST|firmware|PUT: Error in payload data: {error}')
return error_message(str(error), self.URL)
result = self._process_data(data)
if 'error_message' in result:
logging.warning(
'Submission not according to API guidelines! (data could not be parsed)'
)
return error_message(result['error_message'],
self.URL,
request_data=data)
logging.debug('Upload Successful!')
return success_message(result, self.URL, request_data=data)
def get(self, search_id=None):
'''
The search_id is needed to fetch the corresponding search result.
The result of the search request can only be fetched once. After this the search needs to be started again.
The results have the form:
{'binary_search_results': {'<rule_name_1>': ['<matching_uid_1>', ...], '<rule_name_2>': [...], ...}
'''
if search_id is None:
return error_message(
'The request is missing a search_id (.../binary_search/<search_id>).',
self.URL)
with ConnectTo(InterComFrontEndBinding, self.config) as intercom:
result, _ = intercom.get_binary_search_result(search_id)
if result is None:
return error_message(
'The result is not ready yet or it has already been fetched',
self.URL)
return success_message({'binary_search_results': result}, self.URL)
def get(self):
with ConnectTo(FrontEndDbInterface, self.config) as db:
missing_files = self._make_json_serializable(db.find_missing_files())
missing_analyses = self._make_json_serializable(db.find_missing_analyses())
return success_message(dict(missing_files=missing_files, missing_analyses=missing_analyses), self.URL)
def get(self):
return success_message({'dummy': 'rest'}, self.ENDPOINTS[0][0])
