def resubmit(uuid):
try:
root = RootAnalysis(storage_dir=storage_dir_from_uuid(uuid))
root.load()
root.reset()
root.schedule()
return json_result({'result': 'success'})
except Exception as e:
return json_result({'result': 'failed', 'error': str(e)})
def submit():
if KEY_ANALYSIS not in request.values:
abort(
Response(
"missing {} field (see documentation)".format(KEY_ANALYSIS),
400))
r = json.loads(request.values[KEY_ANALYSIS])
# the specified company needs to match the company of this node
# TODO eventually we'll have a single node that serves API to all configured companies
if KEY_COMPANY_NAME in r and r[KEY_COMPANY_NAME] != saq.CONFIG['global'][
'company_name']:
abort(
Response(
"wrong company {} (are you sending to the correct system?)".
format(r[KEY_COMPANY_NAME]), 400))
if KEY_DESCRIPTION not in r:
abort(
Response("missing {} field in submission".format(KEY_DESCRIPTION),
400))
root = RootAnalysis()
root.uuid = str(uuid.uuid4())
# does the engine use a different drive for the workload?
analysis_mode = r[
KEY_ANALYSIS_MODE] if KEY_ANALYSIS_MODE in r else saq.CONFIG['engine'][
'default_analysis_mode']
if analysis_mode != ANALYSIS_MODE_CORRELATION:
root.storage_dir = workload_storage_dir(root.uuid)
else:
root.storage_dir = storage_dir_from_uuid(root.uuid)
root.initialize_storage()
try:
root.analysis_mode = r[
KEY_ANALYSIS_MODE] if KEY_ANALYSIS_MODE in r else saq.CONFIG[
'engine']['default_analysis_mode']
root.company_id = saq.CONFIG['global'].getint('company_id')
root.tool = r[KEY_TOOL] if KEY_TOOL in r else 'api'
root.tool_instance = r[
KEY_TOOL_INSTANCE] if KEY_TOOL_INSTANCE in r else 'api({})'.format(
request.remote_addr)
root.alert_type = r[KEY_TYPE] if KEY_TYPE in r else saq.CONFIG['api'][
'default_alert_type']
root.description = r[KEY_DESCRIPTION]
root.event_time = LOCAL_TIMEZONE.localize(datetime.datetime.now())
if KEY_EVENT_TIME in r:
try:
root.event_time = parse_event_time(r[KEY_EVENT_TIME])
except ValueError as e:
abort(
Response(
"invalid event time format for {} (use {} format)".
format(r[KEY_EVENT_TIME],
event_time_format_json_tz), 400))
root.details = r[KEY_DETAILS] if KEY_DETAILS in r else {}
# go ahead and allocate storage
# XXX use temp dir instead...
if KEY_TAGS in r:
for tag in r[KEY_TAGS]:
root.add_tag(tag)
# add the observables
if KEY_OBSERVABLES in r:
for o in r[KEY_OBSERVABLES]:
# check for required fields
for field in [KEY_O_TYPE, KEY_O_VALUE]:
if field not in o:
abort(
Response(
"an observable is missing the {} field".format(
field), 400))
o_type = o[KEY_O_TYPE]
o_value = o[KEY_O_VALUE]
o_time = None
if KEY_O_TIME in o:
try:
o_time = parse_event_time(o[KEY_O_TIME])
except ValueError:
abort(
Response(
"an observable has an invalid time format {} (use {} format)"
.format(o[KEY_O_TIME],
event_time_format_json_tz), 400))
observable = root.add_observable(o_type,
o_value,
o_time=o_time)
if KEY_O_TAGS in o:
for tag in o[KEY_O_TAGS]:
observable.add_tag(tag)
if KEY_O_DIRECTIVES in o:
for directive in o[KEY_O_DIRECTIVES]:
# is this a valid directive?
if directive not in VALID_DIRECTIVES:
abort(
Response(
"observable {} has invalid directive {} (choose from {})"
.format('{}:{}'.format(o_type,
o_value), directive,
','.join(VALID_DIRECTIVES)), 400))
observable.add_directive(directive)
if KEY_O_LIMITED_ANALYSIS in o:
for module_name in o[KEY_O_LIMITED_ANALYSIS]:
observable.limit_analysis(module_name)
# save the files to disk and add them as observables of type file
for f in request.files.getlist('file'):
logging.debug("recording file {}".format(f.filename))
#temp_dir = tempfile.mkdtemp(dir=saq.CONFIG.get('api', 'incoming_dir'))
#_path = os.path.join(temp_dir, secure_filename(f.filename))
try:
#if os.path.exists(_path):
#logging.error("duplicate file name {}".format(_path))
#abort(400)
#logging.debug("saving file to {}".format(_path))
#try:
#f.save(_path)
#except Exception as e:
#logging.error("unable to save file to {}: {}".format(_path, e))
#abort(400)
full_path = os.path.join(root.storage_dir, f.filename)
try:
dest_dir = os.path.dirname(full_path)
if not os.path.isdir(dest_dir):
try:
os.makedirs(dest_dir)
except Exception as e:
logging.error(
"unable to create directory {}: {}".format(
dest_dir, e))
abort(400)
logging.debug("saving file {}".format(full_path))
f.save(full_path)
# add this as a F_FILE type observable
root.add_observable(
F_FILE,
os.path.relpath(full_path, start=root.storage_dir))
except Exception as e:
logging.error(
"unable to copy file from {} to {} for root {}: {}".
format(_path, full_path, root, e))
abort(400)
except Exception as e:
logging.error("unable to deal with file {}: {}".format(f, e))
report_exception()
abort(400)
#finally:
#try:
#shutil.rmtree(temp_dir)
#except Exception as e:
#logging.error("unable to delete temp dir {}: {}".format(temp_dir, e))
try:
if not root.save():
logging.error("unable to save analysis")
abort(
Response(
"an error occured trying to save the alert - review the logs",
400))
# if we received a submission for correlation mode then we go ahead and add it to the database
if root.analysis_mode == ANALYSIS_MODE_CORRELATION:
ALERT(root)
# add this analysis to the workload
root.schedule()
except Exception as e:
logging.error("unable to sync to database: {}".format(e))
report_exception()
abort(
Response(
"an error occured trying to save the alert - review the logs",
400))
return json_result({'result': {'uuid': root.uuid}})
except Exception as e:
logging.error("error processing submit: {}".format(e))
report_exception()
try:
if os.path.isdir(root.storage_dir):
logging.info("removing failed submit dir {}".format(
root.storage_dir))
shutil.rmtree(root.storage_dir)
except Exception as e2:
logging.error("unable to delete failed submit dir {}: {}".format(
root.storage_dir, e))
raise e
def _create_analysis(url, reprocess, details, db, c):
assert isinstance(url, str)
assert isinstance(reprocess, bool)
assert isinstance(details, dict)
sha256_url = hash_url(url)
if reprocess:
# if we're reprocessing the url then we clear any existing analysis
# IF the current analysis has completed
# it's OK if we delete nothing here
execute_with_retry("""DELETE FROM cloudphish_analysis_results
WHERE sha256_url = UNHEX(%s) AND status = 'ANALYZED'""",
(sha256_url,), commit=True)
# if we're at this point it means that when we asked the database for an entry from cloudphish_analysis_results
# it was empty, OR, we cleared existing analysis
# however, we could have multiple requests coming in at the same time for the same url
# so we need to take that into account here
# first we'll generate our analysis uuid we're going to use
_uuid = str(uuid.uuid4())
# so first we try to insert it
try:
execute_with_retry(db, c, ["""INSERT INTO cloudphish_analysis_results ( sha256_url, uuid, insert_date )
VALUES ( UNHEX(%s), %s, NOW() )""",
"""INSERT INTO cloudphish_url_lookup ( sha256_url, url )
VALUES ( UNHEX(%s), %s )"""],
[(sha256_url, _uuid),
(sha256_url, url)], commit=True)
except pymysql.err.IntegrityError as e:
# (<class 'pymysql.err.IntegrityError'>--(1062, "Duplicate entry
# if we get a duplicate key entry here then it means that an entry was created between when we asked
# and now
if e.args[0] != 1062:
raise e
# so just return that one that was already created
return get_cached_analysis(url)
# at this point we've inserted an entry into cloudphish_analysis_results for this url
# now at it's processing to the workload
root = RootAnalysis()
root.uuid = _uuid
root.storage_dir = storage_dir_from_uuid(root.uuid)
root.initialize_storage()
root.analysis_mode = ANALYSIS_MODE_CLOUDPHISH
# this is kind of a kludge but,
# the company_id initially starts out as whatever the default is for this node
# later, should the analysis turn into an alert, the company_id changes to whatever
# is stored as the "d" field in the KEY_DETAILS_CONTEXT
root.company_id = saq.COMPANY_ID
root.tool = 'ACE - Cloudphish'
root.tool_instance = saq.SAQ_NODE
root.alert_type = ANALYSIS_TYPE_CLOUDPHISH
root.description = 'ACE Cloudphish Detection - {}'.format(url)
root.event_time = datetime.datetime.now()
root.details = {
KEY_DETAILS_URL: url,
KEY_DETAILS_SHA256_URL: sha256_url,
# this used to be configurable but it's always true now
KEY_DETAILS_ALERTABLE: True,
KEY_DETAILS_CONTEXT: details, # <-- optionally contains the source company_id
}
url_observable = root.add_observable(F_URL, url)
if url_observable:
url_observable.add_directive(DIRECTIVE_CRAWL)
root.save()
root.schedule()
return get_cached_analysis(url)
def upload(uuid):
validate_uuid(uuid)
if KEY_UPLOAD_MODIFIERS not in request.values:
abort(Response("missing key {} in request".format(KEY_UPLOAD_MODIFIERS), 400))
if KEY_ARCHIVE not in request.files:
abort(Response("missing files key {}".format(KEY_ARCHIVE), 400))
upload_modifiers = json.loads(request.values[KEY_UPLOAD_MODIFIERS])
if not isinstance(upload_modifiers, dict):
abort(Response("{} should be a dict".format(KEY_UPLOAD_MODIFIERS), 400))
overwrite = False
if KEY_OVERWRITE in upload_modifiers:
overwrite = upload_modifiers[KEY_OVERWRITE]
if not isinstance(overwrite, bool):
abort(Response("{} should be a boolean".format(KEY_OVERWRITE), 400))
sync = False
if KEY_SYNC in upload_modifiers:
sync = upload_modifiers[KEY_SYNC]
if not isinstance(sync, bool):
abort(Response("{} should be a boolean".format(KEY_SYNC), 400))
logging.info("requested upload for {}".format(uuid))
# does the target directory already exist?
target_dir = storage_dir_from_uuid(uuid)
if os.path.exists(target_dir):
# are we over-writing it?
if not overwrite:
abort(Response("{} already exists (specify overwrite modifier to replace the data)".format(target_dir), 400))
# if we are overwriting the entry then we need to completely clear the
# TODO implement this
try:
os.makedirs(target_dir)
except Exception as e:
logging.error("unable to create directory {}: {}".format(target_dir, e))
report_exception()
abort(Response("unable to create directory {}: {}".format(target_dir, e), 400))
logging.debug("target directory for {} is {}".format(uuid, target_dir))
# save the tar file so we can extract it
fp, tar_path = tempfile.mkstemp(suffix='.tar', prefix='upload_{}'.format(uuid), dir=saq.TEMP_DIR)
os.close(fp)
try:
request.files[KEY_ARCHIVE].save(tar_path)
t = tarfile.open(tar_path, 'r|')
t.extractall(path=target_dir)
logging.debug("extracted {} to {}".format(uuid, target_dir))
# update the root analysis to indicate it's new location
root = RootAnalysis(storage_dir=target_dir)
root.load()
root.location = saq.SAQ_NODE
root.company_id = saq.COMPANY_ID
root.company_name = saq.COMPANY_NAME
root.save()
if sync:
root.schedule()
# looks like it worked
return json_result({'result': True})
except Exception as e:
logging.error("unable to upload {}: {}".format(uuid, e))
report_exception()
abort(Response("unable to upload {}: {}".format(uuid, e)))
finally:
try:
os.remove(tar_path)
except Exception as e:
logging.error("unable to remove {}: {}".format(tar_path,e ))