def upload(uuid): validate_uuid(uuid) if KEY_UPLOAD_MODIFIERS not in request.values: abort(Response("missing key {} in request".format(KEY_UPLOAD_MODIFIERS), 400)) if KEY_ARCHIVE not in request.files: abort(Response("missing files key {}".format(KEY_ARCHIVE), 400)) upload_modifiers = json.loads(request.values[KEY_UPLOAD_MODIFIERS]) if not isinstance(upload_modifiers, dict): abort(Response("{} should be a dict".format(KEY_UPLOAD_MODIFIERS), 400)) overwrite = False if KEY_OVERWRITE in upload_modifiers: overwrite = upload_modifiers[KEY_OVERWRITE] if not isinstance(overwrite, bool): abort(Response("{} should be a boolean".format(KEY_OVERWRITE), 400)) sync = False if KEY_SYNC in upload_modifiers: sync = upload_modifiers[KEY_SYNC] if not isinstance(sync, bool): abort(Response("{} should be a boolean".format(KEY_SYNC), 400)) logging.info("requested upload for {}".format(uuid)) # does the target directory already exist? target_dir = storage_dir_from_uuid(uuid) if os.path.exists(target_dir): # are we over-writing it? if not overwrite: abort(Response("{} already exists (specify overwrite modifier to replace the data)".format(target_dir), 400)) # if we are overwriting the entry then we need to completely clear the # TODO implement this try: os.makedirs(target_dir) except Exception as e: logging.error("unable to create directory {}: {}".format(target_dir, e)) report_exception() abort(Response("unable to create directory {}: {}".format(target_dir, e), 400)) logging.debug("target directory for {} is {}".format(uuid, target_dir)) # save the tar file so we can extract it fp, tar_path = tempfile.mkstemp(suffix='.tar', prefix='upload_{}'.format(uuid), dir=saq.TEMP_DIR) os.close(fp) try: request.files[KEY_ARCHIVE].save(tar_path) t = tarfile.open(tar_path, 'r|') t.extractall(path=target_dir) logging.debug("extracted {} to {}".format(uuid, target_dir)) # update the root analysis to indicate it's new location root = RootAnalysis(storage_dir=target_dir) root.load() root.location = saq.SAQ_NODE root.company_id = saq.COMPANY_ID root.company_name = saq.COMPANY_NAME root.save() if sync: root.schedule() # looks like it worked return json_result({'result': True}) except Exception as e: logging.error("unable to upload {}: {}".format(uuid, e)) report_exception() abort(Response("unable to upload {}: {}".format(uuid, e))) finally: try: os.remove(tar_path) except Exception as e: logging.error("unable to remove {}: {}".format(tar_path,e ))
def post_http_analysis(self, root): from saq.modules.http import BrotexHTTPPackageAnalysis, \ KEY_TIME, \ KEY_SRC_IP, \ KEY_SRC_PORT, \ KEY_DEST_IP, \ KEY_DEST_PORT, \ KEY_METHOD, \ KEY_HOST, \ KEY_URI, \ KEY_REFERRER, \ KEY_USER_AGENT, \ KEY_STATUS_CODE, \ KEY_FILES # get the paths to the http scanning system #http_scanner_dir = saq.CONFIG['engine_http_scanner']['collection_dir'] http_scanner_dir = self.collection_dir analysis = None for a in root.all_analysis: if isinstance(a, BrotexHTTPPackageAnalysis) and a.requests: analysis = a break # this can happen if the request was whitelisted if analysis: for request in analysis.requests: subroot = RootAnalysis() subroot.company_name = root.company_name subroot.tool = root.tool subroot.tool_instance = root.tool_instance subroot.alert_type = root.alert_type subroot.description = "Brotex HTTP Stream Detection - " if request[KEY_HOST]: subroot.description += " {} ".format(request[KEY_HOST]) if request[KEY_DEST_IP]: subroot.description += " ({}) ".format( request[KEY_DEST_IP]) if request[KEY_URI]: # don't want to show all the fragments and query params try: parts = urlparse(request[KEY_URI]) subroot.description += parts.path except Exception as e: logging.warning("unable to parse {}: {}".format( request[KEY_URI], e)) subroot.description += request[KEY_URI] subroot.event_time = root.event_time subroot.details = request subroot.uuid = str(uuid.uuid4()) # we use a temporary directory while we process the file subroot.storage_dir = os.path.join(http_scanner_dir, subroot.uuid[0:3], subroot.uuid) subroot.initialize_storage() if request[KEY_SRC_IP]: subroot.add_observable(F_IPV4, request[KEY_SRC_IP]) if request[KEY_DEST_IP]: subroot.add_observable(F_IPV4, request[KEY_DEST_IP]) if request[KEY_SRC_IP] and request[KEY_DEST_IP]: subroot.add_observable( F_IPV4_CONVERSATION, create_ipv4_conversation(request[KEY_SRC_IP], request[KEY_DEST_IP])) if request[KEY_HOST]: subroot.add_observable(F_FQDN, request[KEY_HOST]) if request[KEY_URI]: subroot.add_observable(F_URL, request[KEY_URI]) if request[KEY_REFERRER]: subroot.add_observable(F_URL, request[KEY_REFERRER]) for file_path in request[KEY_FILES]: src_path = os.path.join(root.storage_dir, file_path) dest_path = os.path.join(subroot.storage_dir, os.path.basename(file_path)) try: shutil.copy(src_path, dest_path) except Exception as e: logging.error("unable to copy {} to {}: {}".format( src_path, dest_path, e)) report_exception() subroot.add_observable( F_FILE, os.path.basename(file_path)) # already relative try: subroot.save() except Exception as e: logging.error("unable to save {}: {}".format(alert, e)) report_exception() continue # submit the path to the database of the email scanner for analysis try: submit_sql_work_item( 'HTTP', subroot.storage_dir) # XXX hard coded constant except: # failure is already logged inside the call continue
def post_smtp_analysis(self, root): from saq.modules.email import EmailAnalysis, SMTPStreamAnalysis, \ BrotexSMTPPackageAnalysis, \ KEY_ENVELOPES_MAIL_FROM, KEY_ENVELOPES_RCPT_TO # get the paths to the email scanning system #email_scanner_dir = saq.CONFIG['engine_email_scanner']['collection_dir'] email_scanner_dir = self.collection_dir # create a new analysis root for each email analysis we found for analysis in root.all_analysis: if not isinstance(analysis, EmailAnalysis) or not analysis.email: continue env_mail_from = None env_rcpt_to = None connection_id = None # the observable for this EmailAnalysis will be a file email_file = analysis.observable if email_file.type != F_FILE: logging.warning( "the observable for {} should be F_FILE but it is {}". format(analysis, email_file.type)) else: # this will be either an rfc822 file generated by the SMTPStreamAnalysis module # (which will have the envelope information) # OR it is a "broken stream" file, which does not stream_analysis = [ a for a in root.all_analysis if isinstance(a, SMTPStreamAnalysis) and email_file in a.observables ] if len(stream_analysis) > 1: logging.error("there should not be more than one of these") elif len(stream_analysis) == 1: stream_analysis = stream_analysis[0] logging.debug( "detected stream analysis for {}".format(email_file)) # get the MAIL FROM and RCPT TO from this if not analysis.env_mail_from: if email_file.value in stream_analysis.envelopes: analysis.env_mail_from = stream_analysis.envelopes[ email_file.value][KEY_ENVELOPES_MAIL_FROM] if not analysis.env_rcpt_to: if email_file.value in stream_analysis.envelopes: analysis.env_rcpt_to = stream_analysis.envelopes[ email_file.value][KEY_ENVELOPES_RCPT_TO] # get the original brotex package file that the stream came from stream_package = stream_analysis.observable # get the BrotexSMTPPackageAnalysis for this stream package so we can get the connection id package_analysis = [ a for a in root.all_analysis if isinstance(a, BrotexSMTPPackageAnalysis) and stream_package in a.observables ] if len(package_analysis) > 1: logging.error( "there should not be more than one of these!") elif len(package_analysis) == 1: package_analysis = package_analysis[0] connection_id = package_analysis.connection_id # if we could not find the stream, we will want to find the brotex smtp package so we can have the connection id package_analysis = [ a for a in root.all_analysis if isinstance(a, BrotexSMTPPackageAnalysis) and email_file in a.observables ] if len(package_analysis) > 1: logging.error( "there should not be more than one of these!") elif len(package_analysis) == 1: package_analysis = package_analysis[0] connection_id = package_analysis.connection_id subroot = RootAnalysis() subroot.company_name = root.company_name subroot.tool = root.tool subroot.tool_instance = root.tool_instance subroot.alert_type = root.alert_type subroot.description = 'Brotex SMTP Stream Detection - ' if analysis.decoded_subject: subroot.description += '{} '.format(analysis.decoded_subject) elif analysis.subject: subroot.description += '{} '.format(analysis.subject) else: subroot.description += '(no subject) ' if analysis.env_mail_from: subroot.description += 'From {} '.format( normalize_email_address(analysis.env_mail_from)) elif analysis.mail_from: subroot.description += 'From {} '.format( normalize_email_address(analysis.mail_from)) if analysis.env_rcpt_to: if len(analysis.env_rcpt_to) == 1: subroot.description += 'To {} '.format( analysis.env_rcpt_to[0]) else: subroot.description += 'To ({} recipients) '.format( len(analysis.env_rcpt_to)) elif analysis.mail_to: if isinstance(analysis.mail_to, list): # XXX I think this *has* to be a list if len(analysis.mail_to) == 1: subroot.description += 'To {} '.format( analysis.mail_to[0]) else: subroot.description += 'To ({} recipients) '.format( len(analysis.mail_to)) else: subroot.description += 'To {} '.format( analysis.mail_to) subroot.event_time = root.event_time subroot.details = analysis.details subroot.details['connection_id'] = connection_id subroot.uuid = str(uuid.uuid4()) # we use a temporary directory while we process the file subroot.storage_dir = os.path.join(email_scanner_dir, subroot.uuid[0:3], subroot.uuid) subroot.initialize_storage() # copy the original file src_path = os.path.join(root.storage_dir, analysis.observable.value) dest_path = os.path.join(subroot.storage_dir, analysis.observable.value) subroot.add_observable( F_FILE, os.path.relpath(dest_path, start=subroot.storage_dir)) # so the EmailAnalysis that will trigger on the RFC822 file (or whatever you have) # will *not* have the envelope headers # so we do that here in the main alert env_mail_from = None if analysis.env_mail_from: # this is to handle this: <*****@*****.**> SIZE=80280 # XXX assuming there can be no spaces in an email address env_mail_from = analysis.env_mail_from.split(' ', 1) env_mail_from = env_mail_from[0] # is this not the empty indicator? if env_mail_from != '<>': env_mail_from = normalize_email_address(env_mail_from) subroot.add_observable(F_EMAIL_ADDRESS, env_mail_from) if analysis.env_rcpt_to: for address in analysis.env_rcpt_to: address = normalize_email_address(address) if address: subroot.add_observable(F_EMAIL_ADDRESS, address) if env_mail_from: subroot.add_observable( F_EMAIL_CONVERSATION, create_email_conversation( env_mail_from, address)) try: subroot.save() except Exception as e: logging.error("unable to save {}: {}".format(alert, e)) report_exception() continue # TODO also add the stream and update any envelopment headers and stuff try: logging.debug("copying {} to {}".format(src_path, dest_path)) shutil.copy(src_path, dest_path) except Exception as e: logging.error("unable to copy {} to {}: {}".format( src_path, dest_path, e)) report_exception() continue # submit the path to the database of the email scanner for analysis try: submit_sql_work_item('EMAIL', subroot.storage_dir) except Exception as e: logging.error("unable to add work item: {}".format(e)) report_exception() continue
def process(self, work_item): url, alertable, details = work_item # any other result means we should process it logging.info("processing url {} (alertable {})".format(url, alertable)) #logging.debug("details = {}".format(details)) sha256_url = hash_url(url) # create or update our analysis entry with get_db_connection('cloudphish') as db: c = db.cursor() c.execute( """UPDATE analysis_results SET status = %s WHERE sha256_url = UNHEX(%s)""", (STATUS_ANALYZING, sha256_url)) db.commit() root = RootAnalysis() # create a temporary storage directory for this work root.tool = 'ACE - Cloudphish' root.tool_instance = self.location root.alert_type = 'cloudphish' root.description = 'ACE Cloudphish Detection - {}'.format(url) root.event_time = datetime.datetime.now() root.uuid = str(uuid.uuid4()) root.storage_dir = os.path.join(self.work_dir, root.uuid[0:2], root.uuid) root.initialize_storage() if 'i' in details: root.company_name = details['i'] if 'd' in details: root.company_id = details['d'] root.details = { KEY_DETAILS_URL: url, KEY_DETAILS_SHA256_URL: sha256_url, KEY_DETAILS_ALERTABLE: alertable, KEY_DETAILS_CONTEXT: details, } url_observable = root.add_observable(F_URL, url) if url_observable is None: logging.error("request for invalid url received: {}".format(url)) return url_observable.add_directive(DIRECTIVE_CRAWL) # the "details context" can also contain observables for key in root.details[KEY_DETAILS_CONTEXT].keys(): if key in VALID_OBSERVABLE_TYPES: root.add_observable(key, root.details[KEY_DETAILS_CONTEXT][key]) try: self.analyze(root) except Exception as e: logging.error("analysis failed for {}: {}".format(url, e)) report_exception() with get_db_connection('cloudphish') as db: c = db.cursor() c.execute( """UPDATE analysis_results SET result = %s, status = %s, http_result_code = NULL, http_message = NULL, sha256_content = NULL WHERE sha256_url = UNHEX(%s)""", (SCAN_RESULT_ERROR, STATUS_ANALYZED, sha256_url)) db.commit() return