def upload(uuid):
validate_uuid(uuid)
if KEY_UPLOAD_MODIFIERS not in request.values:
abort(Response("missing key {} in request".format(KEY_UPLOAD_MODIFIERS), 400))
if KEY_ARCHIVE not in request.files:
abort(Response("missing files key {}".format(KEY_ARCHIVE), 400))
upload_modifiers = json.loads(request.values[KEY_UPLOAD_MODIFIERS])
if not isinstance(upload_modifiers, dict):
abort(Response("{} should be a dict".format(KEY_UPLOAD_MODIFIERS), 400))
overwrite = False
if KEY_OVERWRITE in upload_modifiers:
overwrite = upload_modifiers[KEY_OVERWRITE]
if not isinstance(overwrite, bool):
abort(Response("{} should be a boolean".format(KEY_OVERWRITE), 400))
sync = False
if KEY_SYNC in upload_modifiers:
sync = upload_modifiers[KEY_SYNC]
if not isinstance(sync, bool):
abort(Response("{} should be a boolean".format(KEY_SYNC), 400))
logging.info("requested upload for {}".format(uuid))
# does the target directory already exist?
target_dir = storage_dir_from_uuid(uuid)
if os.path.exists(target_dir):
# are we over-writing it?
if not overwrite:
abort(Response("{} already exists (specify overwrite modifier to replace the data)".format(target_dir), 400))
# if we are overwriting the entry then we need to completely clear the
# TODO implement this
try:
os.makedirs(target_dir)
except Exception as e:
logging.error("unable to create directory {}: {}".format(target_dir, e))
report_exception()
abort(Response("unable to create directory {}: {}".format(target_dir, e), 400))
logging.debug("target directory for {} is {}".format(uuid, target_dir))
# save the tar file so we can extract it
fp, tar_path = tempfile.mkstemp(suffix='.tar', prefix='upload_{}'.format(uuid), dir=saq.TEMP_DIR)
os.close(fp)
try:
request.files[KEY_ARCHIVE].save(tar_path)
t = tarfile.open(tar_path, 'r|')
t.extractall(path=target_dir)
logging.debug("extracted {} to {}".format(uuid, target_dir))
# update the root analysis to indicate it's new location
root = RootAnalysis(storage_dir=target_dir)
root.load()
root.location = saq.SAQ_NODE
root.company_id = saq.COMPANY_ID
root.company_name = saq.COMPANY_NAME
root.save()
if sync:
root.schedule()
# looks like it worked
return json_result({'result': True})
except Exception as e:
logging.error("unable to upload {}: {}".format(uuid, e))
report_exception()
abort(Response("unable to upload {}: {}".format(uuid, e)))
finally:
try:
os.remove(tar_path)
except Exception as e:
logging.error("unable to remove {}: {}".format(tar_path,e ))
def post_http_analysis(self, root):
from saq.modules.http import BrotexHTTPPackageAnalysis, \
KEY_TIME, \
KEY_SRC_IP, \
KEY_SRC_PORT, \
KEY_DEST_IP, \
KEY_DEST_PORT, \
KEY_METHOD, \
KEY_HOST, \
KEY_URI, \
KEY_REFERRER, \
KEY_USER_AGENT, \
KEY_STATUS_CODE, \
KEY_FILES
# get the paths to the http scanning system
#http_scanner_dir = saq.CONFIG['engine_http_scanner']['collection_dir']
http_scanner_dir = self.collection_dir
analysis = None
for a in root.all_analysis:
if isinstance(a, BrotexHTTPPackageAnalysis) and a.requests:
analysis = a
break
# this can happen if the request was whitelisted
if analysis:
for request in analysis.requests:
subroot = RootAnalysis()
subroot.company_name = root.company_name
subroot.tool = root.tool
subroot.tool_instance = root.tool_instance
subroot.alert_type = root.alert_type
subroot.description = "Brotex HTTP Stream Detection - "
if request[KEY_HOST]:
subroot.description += " {} ".format(request[KEY_HOST])
if request[KEY_DEST_IP]:
subroot.description += " ({}) ".format(
request[KEY_DEST_IP])
if request[KEY_URI]:
# don't want to show all the fragments and query params
try:
parts = urlparse(request[KEY_URI])
subroot.description += parts.path
except Exception as e:
logging.warning("unable to parse {}: {}".format(
request[KEY_URI], e))
subroot.description += request[KEY_URI]
subroot.event_time = root.event_time
subroot.details = request
subroot.uuid = str(uuid.uuid4())
# we use a temporary directory while we process the file
subroot.storage_dir = os.path.join(http_scanner_dir,
subroot.uuid[0:3],
subroot.uuid)
subroot.initialize_storage()
if request[KEY_SRC_IP]:
subroot.add_observable(F_IPV4, request[KEY_SRC_IP])
if request[KEY_DEST_IP]:
subroot.add_observable(F_IPV4, request[KEY_DEST_IP])
if request[KEY_SRC_IP] and request[KEY_DEST_IP]:
subroot.add_observable(
F_IPV4_CONVERSATION,
create_ipv4_conversation(request[KEY_SRC_IP],
request[KEY_DEST_IP]))
if request[KEY_HOST]:
subroot.add_observable(F_FQDN, request[KEY_HOST])
if request[KEY_URI]:
subroot.add_observable(F_URL, request[KEY_URI])
if request[KEY_REFERRER]:
subroot.add_observable(F_URL, request[KEY_REFERRER])
for file_path in request[KEY_FILES]:
src_path = os.path.join(root.storage_dir, file_path)
dest_path = os.path.join(subroot.storage_dir,
os.path.basename(file_path))
try:
shutil.copy(src_path, dest_path)
except Exception as e:
logging.error("unable to copy {} to {}: {}".format(
src_path, dest_path, e))
report_exception()
subroot.add_observable(
F_FILE,
os.path.basename(file_path)) # already relative
try:
subroot.save()
except Exception as e:
logging.error("unable to save {}: {}".format(alert, e))
report_exception()
continue
# submit the path to the database of the email scanner for analysis
try:
submit_sql_work_item(
'HTTP', subroot.storage_dir) # XXX hard coded constant
except:
# failure is already logged inside the call
continue
def post_smtp_analysis(self, root):
from saq.modules.email import EmailAnalysis, SMTPStreamAnalysis, \
BrotexSMTPPackageAnalysis, \
KEY_ENVELOPES_MAIL_FROM, KEY_ENVELOPES_RCPT_TO
# get the paths to the email scanning system
#email_scanner_dir = saq.CONFIG['engine_email_scanner']['collection_dir']
email_scanner_dir = self.collection_dir
# create a new analysis root for each email analysis we found
for analysis in root.all_analysis:
if not isinstance(analysis, EmailAnalysis) or not analysis.email:
continue
env_mail_from = None
env_rcpt_to = None
connection_id = None
# the observable for this EmailAnalysis will be a file
email_file = analysis.observable
if email_file.type != F_FILE:
logging.warning(
"the observable for {} should be F_FILE but it is {}".
format(analysis, email_file.type))
else:
# this will be either an rfc822 file generated by the SMTPStreamAnalysis module
# (which will have the envelope information)
# OR it is a "broken stream" file, which does not
stream_analysis = [
a for a in root.all_analysis
if isinstance(a, SMTPStreamAnalysis)
and email_file in a.observables
]
if len(stream_analysis) > 1:
logging.error("there should not be more than one of these")
elif len(stream_analysis) == 1:
stream_analysis = stream_analysis[0]
logging.debug(
"detected stream analysis for {}".format(email_file))
# get the MAIL FROM and RCPT TO from this
if not analysis.env_mail_from:
if email_file.value in stream_analysis.envelopes:
analysis.env_mail_from = stream_analysis.envelopes[
email_file.value][KEY_ENVELOPES_MAIL_FROM]
if not analysis.env_rcpt_to:
if email_file.value in stream_analysis.envelopes:
analysis.env_rcpt_to = stream_analysis.envelopes[
email_file.value][KEY_ENVELOPES_RCPT_TO]
# get the original brotex package file that the stream came from
stream_package = stream_analysis.observable
# get the BrotexSMTPPackageAnalysis for this stream package so we can get the connection id
package_analysis = [
a for a in root.all_analysis
if isinstance(a, BrotexSMTPPackageAnalysis)
and stream_package in a.observables
]
if len(package_analysis) > 1:
logging.error(
"there should not be more than one of these!")
elif len(package_analysis) == 1:
package_analysis = package_analysis[0]
connection_id = package_analysis.connection_id
# if we could not find the stream, we will want to find the brotex smtp package so we can have the connection id
package_analysis = [
a for a in root.all_analysis
if isinstance(a, BrotexSMTPPackageAnalysis)
and email_file in a.observables
]
if len(package_analysis) > 1:
logging.error(
"there should not be more than one of these!")
elif len(package_analysis) == 1:
package_analysis = package_analysis[0]
connection_id = package_analysis.connection_id
subroot = RootAnalysis()
subroot.company_name = root.company_name
subroot.tool = root.tool
subroot.tool_instance = root.tool_instance
subroot.alert_type = root.alert_type
subroot.description = 'Brotex SMTP Stream Detection - '
if analysis.decoded_subject:
subroot.description += '{} '.format(analysis.decoded_subject)
elif analysis.subject:
subroot.description += '{} '.format(analysis.subject)
else:
subroot.description += '(no subject) '
if analysis.env_mail_from:
subroot.description += 'From {} '.format(
normalize_email_address(analysis.env_mail_from))
elif analysis.mail_from:
subroot.description += 'From {} '.format(
normalize_email_address(analysis.mail_from))
if analysis.env_rcpt_to:
if len(analysis.env_rcpt_to) == 1:
subroot.description += 'To {} '.format(
analysis.env_rcpt_to[0])
else:
subroot.description += 'To ({} recipients) '.format(
len(analysis.env_rcpt_to))
elif analysis.mail_to:
if isinstance(analysis.mail_to,
list): # XXX I think this *has* to be a list
if len(analysis.mail_to) == 1:
subroot.description += 'To {} '.format(
analysis.mail_to[0])
else:
subroot.description += 'To ({} recipients) '.format(
len(analysis.mail_to))
else:
subroot.description += 'To {} '.format(
analysis.mail_to)
subroot.event_time = root.event_time
subroot.details = analysis.details
subroot.details['connection_id'] = connection_id
subroot.uuid = str(uuid.uuid4())
# we use a temporary directory while we process the file
subroot.storage_dir = os.path.join(email_scanner_dir,
subroot.uuid[0:3], subroot.uuid)
subroot.initialize_storage()
# copy the original file
src_path = os.path.join(root.storage_dir,
analysis.observable.value)
dest_path = os.path.join(subroot.storage_dir,
analysis.observable.value)
subroot.add_observable(
F_FILE, os.path.relpath(dest_path, start=subroot.storage_dir))
# so the EmailAnalysis that will trigger on the RFC822 file (or whatever you have)
# will *not* have the envelope headers
# so we do that here in the main alert
env_mail_from = None
if analysis.env_mail_from:
# this is to handle this: <*****@*****.**> SIZE=80280
# XXX assuming there can be no spaces in an email address
env_mail_from = analysis.env_mail_from.split(' ', 1)
env_mail_from = env_mail_from[0]
# is this not the empty indicator?
if env_mail_from != '<>':
env_mail_from = normalize_email_address(env_mail_from)
subroot.add_observable(F_EMAIL_ADDRESS, env_mail_from)
if analysis.env_rcpt_to:
for address in analysis.env_rcpt_to:
address = normalize_email_address(address)
if address:
subroot.add_observable(F_EMAIL_ADDRESS, address)
if env_mail_from:
subroot.add_observable(
F_EMAIL_CONVERSATION,
create_email_conversation(
env_mail_from, address))
try:
subroot.save()
except Exception as e:
logging.error("unable to save {}: {}".format(alert, e))
report_exception()
continue
# TODO also add the stream and update any envelopment headers and stuff
try:
logging.debug("copying {} to {}".format(src_path, dest_path))
shutil.copy(src_path, dest_path)
except Exception as e:
logging.error("unable to copy {} to {}: {}".format(
src_path, dest_path, e))
report_exception()
continue
# submit the path to the database of the email scanner for analysis
try:
submit_sql_work_item('EMAIL', subroot.storage_dir)
except Exception as e:
logging.error("unable to add work item: {}".format(e))
report_exception()
continue
def process(self, work_item):
url, alertable, details = work_item
# any other result means we should process it
logging.info("processing url {} (alertable {})".format(url, alertable))
#logging.debug("details = {}".format(details))
sha256_url = hash_url(url)
# create or update our analysis entry
with get_db_connection('cloudphish') as db:
c = db.cursor()
c.execute(
"""UPDATE analysis_results SET status = %s WHERE sha256_url = UNHEX(%s)""",
(STATUS_ANALYZING, sha256_url))
db.commit()
root = RootAnalysis()
# create a temporary storage directory for this work
root.tool = 'ACE - Cloudphish'
root.tool_instance = self.location
root.alert_type = 'cloudphish'
root.description = 'ACE Cloudphish Detection - {}'.format(url)
root.event_time = datetime.datetime.now()
root.uuid = str(uuid.uuid4())
root.storage_dir = os.path.join(self.work_dir, root.uuid[0:2],
root.uuid)
root.initialize_storage()
if 'i' in details:
root.company_name = details['i']
if 'd' in details:
root.company_id = details['d']
root.details = {
KEY_DETAILS_URL: url,
KEY_DETAILS_SHA256_URL: sha256_url,
KEY_DETAILS_ALERTABLE: alertable,
KEY_DETAILS_CONTEXT: details,
}
url_observable = root.add_observable(F_URL, url)
if url_observable is None:
logging.error("request for invalid url received: {}".format(url))
return
url_observable.add_directive(DIRECTIVE_CRAWL)
# the "details context" can also contain observables
for key in root.details[KEY_DETAILS_CONTEXT].keys():
if key in VALID_OBSERVABLE_TYPES:
root.add_observable(key,
root.details[KEY_DETAILS_CONTEXT][key])
try:
self.analyze(root)
except Exception as e:
logging.error("analysis failed for {}: {}".format(url, e))
report_exception()
with get_db_connection('cloudphish') as db:
c = db.cursor()
c.execute(
"""UPDATE analysis_results SET
result = %s,
status = %s,
http_result_code = NULL,
http_message = NULL,
sha256_content = NULL
WHERE sha256_url = UNHEX(%s)""",
(SCAN_RESULT_ERROR, STATUS_ANALYZED, sha256_url))
db.commit()
return
