def test_util_000_date_parsing(self): default_format = '2018-10-19 14:06:34 +0000' old_default_format = '2018-10-19 14:06:34' json_format = '2018-10-19T18:08:08.346118-05:00' old_json_format = '2018-10-19T18:08:08.346118' splunk_format = '2015-02-19T09:50:49.000-05:00' result = parse_event_time(default_format) self.assertEquals(result.year, 2018) self.assertEquals(result.month, 10) self.assertEquals(result.day, 19) self.assertEquals(result.hour, 14) self.assertEquals(result.minute, 6) self.assertEquals(result.second, 34) self.assertIsNotNone(result.tzinfo) self.assertEquals(int(result.tzinfo.utcoffset(None).total_seconds()), 0) result = parse_event_time(old_default_format) self.assertEquals(result.year, 2018) self.assertEquals(result.month, 10) self.assertEquals(result.day, 19) self.assertEquals(result.hour, 14) self.assertEquals(result.minute, 6) self.assertEquals(result.second, 34) self.assertIsNotNone(result.tzinfo) self.assertEquals(saq.LOCAL_TIMEZONE.tzname, result.tzinfo.tzname) result = parse_event_time(json_format) self.assertEquals(result.year, 2018) self.assertEquals(result.month, 10) self.assertEquals(result.day, 19) self.assertEquals(result.hour, 18) self.assertEquals(result.minute, 8) self.assertEquals(result.second, 8) self.assertIsNotNone(result.tzinfo) self.assertEquals(int(result.tzinfo.utcoffset(None).total_seconds()), -(5 * 60 * 60)) result = parse_event_time(old_json_format) self.assertEquals(result.year, 2018) self.assertEquals(result.month, 10) self.assertEquals(result.day, 19) self.assertEquals(result.hour, 18) self.assertEquals(result.minute, 8) self.assertEquals(result.second, 8) self.assertIsNotNone(result.tzinfo) self.assertEquals(saq.LOCAL_TIMEZONE.tzname, result.tzinfo.tzname) result = parse_event_time(splunk_format) self.assertEquals(result.year, 2015) self.assertEquals(result.month, 2) self.assertEquals(result.day, 19) self.assertEquals(result.hour, 9) self.assertEquals(result.minute, 50) self.assertEquals(result.second, 49) self.assertIsNotNone(result.tzinfo) self.assertEquals(int(result.tzinfo.utcoffset(None).total_seconds()), -(5 * 60 * 60))
def execute_analysis(self, ipv4_fc): target_time = ipv4_fc.time if ipv4_fc.time else self.root.event_time # source -> dest (dest_port) source_dest_json = None self.splunk_query(f"""index=carbonblack event_type=netconn local_ip={ipv4_fc.source} remote_ip={ipv4_fc.dest} remote_port={ipv4_fc.dest_port} | fields *""", target_time) if self.search_results is not None: source_dest_json = self.json() # dest -> source (src_port) dest_source_json = None self.splunk_query(f"""index=carbonblack event_type=netconn local_ip={ipv4_fc.dest} remote_ip={ipv4_fc.source} remote_port={ipv4_fc.source_port} | fields *""", target_time) if self.search_results is not None: dest_source_json = self.json() if source_dest_json is None and dest_source_json is None: return False analysis = self.create_analysis(ipv4_fc) if source_dest_json is not None: analysis.details.extend(source_dest_json) if dest_source_json is not None: analysis.details.extend(dest_source_json) procs = [(p['process_guid'], parse_event_time(p['_time'])) for p in analysis.details if 'process_guid' in p] for process_guid, event_time in procs[:self.process_guid_limit]: process_guid = analysis.add_observable(F_PROCESS_GUID, process_guid, event_time) return True
def submit(): if KEY_ANALYSIS not in request.values: abort( Response( "missing {} field (see documentation)".format(KEY_ANALYSIS), 400)) r = json.loads(request.values[KEY_ANALYSIS]) # the specified company needs to match the company of this node # TODO eventually we'll have a single node that serves API to all configured companies if KEY_COMPANY_NAME in r and r[KEY_COMPANY_NAME] != saq.CONFIG['global'][ 'company_name']: abort( Response( "wrong company {} (are you sending to the correct system?)". format(r[KEY_COMPANY_NAME]), 400)) if KEY_DESCRIPTION not in r: abort( Response("missing {} field in submission".format(KEY_DESCRIPTION), 400)) root = RootAnalysis() root.uuid = str(uuid.uuid4()) # does the engine use a different drive for the workload? analysis_mode = r[ KEY_ANALYSIS_MODE] if KEY_ANALYSIS_MODE in r else saq.CONFIG['engine'][ 'default_analysis_mode'] if analysis_mode != ANALYSIS_MODE_CORRELATION: root.storage_dir = workload_storage_dir(root.uuid) else: root.storage_dir = storage_dir_from_uuid(root.uuid) root.initialize_storage() try: root.analysis_mode = r[ KEY_ANALYSIS_MODE] if KEY_ANALYSIS_MODE in r else saq.CONFIG[ 'engine']['default_analysis_mode'] root.company_id = saq.CONFIG['global'].getint('company_id') root.tool = r[KEY_TOOL] if KEY_TOOL in r else 'api' root.tool_instance = r[ KEY_TOOL_INSTANCE] if KEY_TOOL_INSTANCE in r else 'api({})'.format( request.remote_addr) root.alert_type = r[KEY_TYPE] if KEY_TYPE in r else saq.CONFIG['api'][ 'default_alert_type'] root.description = r[KEY_DESCRIPTION] root.event_time = LOCAL_TIMEZONE.localize(datetime.datetime.now()) if KEY_EVENT_TIME in r: try: root.event_time = parse_event_time(r[KEY_EVENT_TIME]) except ValueError as e: abort( Response( "invalid event time format for {} (use {} format)". format(r[KEY_EVENT_TIME], event_time_format_json_tz), 400)) root.details = r[KEY_DETAILS] if KEY_DETAILS in r else {} # go ahead and allocate storage # XXX use temp dir instead... if KEY_TAGS in r: for tag in r[KEY_TAGS]: root.add_tag(tag) # add the observables if KEY_OBSERVABLES in r: for o in r[KEY_OBSERVABLES]: # check for required fields for field in [KEY_O_TYPE, KEY_O_VALUE]: if field not in o: abort( Response( "an observable is missing the {} field".format( field), 400)) o_type = o[KEY_O_TYPE] o_value = o[KEY_O_VALUE] o_time = None if KEY_O_TIME in o: try: o_time = parse_event_time(o[KEY_O_TIME]) except ValueError: abort( Response( "an observable has an invalid time format {} (use {} format)" .format(o[KEY_O_TIME], event_time_format_json_tz), 400)) observable = root.add_observable(o_type, o_value, o_time=o_time) if KEY_O_TAGS in o: for tag in o[KEY_O_TAGS]: observable.add_tag(tag) if KEY_O_DIRECTIVES in o: for directive in o[KEY_O_DIRECTIVES]: # is this a valid directive? if directive not in VALID_DIRECTIVES: abort( Response( "observable {} has invalid directive {} (choose from {})" .format('{}:{}'.format(o_type, o_value), directive, ','.join(VALID_DIRECTIVES)), 400)) observable.add_directive(directive) if KEY_O_LIMITED_ANALYSIS in o: for module_name in o[KEY_O_LIMITED_ANALYSIS]: observable.limit_analysis(module_name) # save the files to disk and add them as observables of type file for f in request.files.getlist('file'): logging.debug("recording file {}".format(f.filename)) #temp_dir = tempfile.mkdtemp(dir=saq.CONFIG.get('api', 'incoming_dir')) #_path = os.path.join(temp_dir, secure_filename(f.filename)) try: #if os.path.exists(_path): #logging.error("duplicate file name {}".format(_path)) #abort(400) #logging.debug("saving file to {}".format(_path)) #try: #f.save(_path) #except Exception as e: #logging.error("unable to save file to {}: {}".format(_path, e)) #abort(400) full_path = os.path.join(root.storage_dir, f.filename) try: dest_dir = os.path.dirname(full_path) if not os.path.isdir(dest_dir): try: os.makedirs(dest_dir) except Exception as e: logging.error( "unable to create directory {}: {}".format( dest_dir, e)) abort(400) logging.debug("saving file {}".format(full_path)) f.save(full_path) # add this as a F_FILE type observable root.add_observable( F_FILE, os.path.relpath(full_path, start=root.storage_dir)) except Exception as e: logging.error( "unable to copy file from {} to {} for root {}: {}". format(_path, full_path, root, e)) abort(400) except Exception as e: logging.error("unable to deal with file {}: {}".format(f, e)) report_exception() abort(400) #finally: #try: #shutil.rmtree(temp_dir) #except Exception as e: #logging.error("unable to delete temp dir {}: {}".format(temp_dir, e)) try: if not root.save(): logging.error("unable to save analysis") abort( Response( "an error occured trying to save the alert - review the logs", 400)) # if we received a submission for correlation mode then we go ahead and add it to the database if root.analysis_mode == ANALYSIS_MODE_CORRELATION: ALERT(root) # add this analysis to the workload root.schedule() except Exception as e: logging.error("unable to sync to database: {}".format(e)) report_exception() abort( Response( "an error occured trying to save the alert - review the logs", 400)) return json_result({'result': {'uuid': root.uuid}}) except Exception as e: logging.error("error processing submit: {}".format(e)) report_exception() try: if os.path.isdir(root.storage_dir): logging.info("removing failed submit dir {}".format( root.storage_dir)) shutil.rmtree(root.storage_dir) except Exception as e2: logging.error("unable to delete failed submit dir {}: {}".format( root.storage_dir, e)) raise e
def splunktime_to_saqtime(splunk_time): """Convert a splunk time in 2015-02-19T09:50:49.000-05:00 format to SAQ time format YYYY-MM-DD HH:MM:SS.""" assert isinstance(splunk_time, str) return parse_event_time(splunk_time).strftime(event_time_format_json_tz)
def splunktime_to_datetime(splunk_time): """Convert a splunk time in 2015-02-19T09:50:49.000-05:00 format to a datetime object.""" assert isinstance(splunk_time, str) #return datetime.datetime.strptime(splunk_time.split('.')[0], '%Y-%m-%dT%H:%M:%S') return parse_event_time(splunk_time)
def test_api_analysis_submit(self, db, c): t = saq.LOCAL_TIMEZONE.localize( datetime.datetime(2017, 11, 11, hour=7, minute=36, second=1, microsecond=1)).astimezone( pytz.UTC).strftime(event_time_format_json_tz) result = self.client.post( url_for('analysis.submit'), data={ 'analysis': json.dumps( { 'analysis_mode': 'analysis', 'tool': 'unittest', 'tool_instance': 'unittest_instance', 'type': 'unittest', 'description': 'testing', 'event_time': t, 'details': { 'hello': 'world' }, 'observables': [ { 'type': F_IPV4, 'value': '1.2.3.4', 'time': t, 'tags': ['tag_1', 'tag_2'], 'directives': [DIRECTIVE_NO_SCAN], 'limited_analysis': ['basic_test'] }, { 'type': F_USER, 'value': 'test_user', 'time': t }, ], 'tags': ['alert_tag_1', 'alert_tag_2'], }, cls=_JSONEncoder), 'file': (io.BytesIO(b'Hello, world!'), 'sample.dat'), }, content_type='multipart/form-data') result = result.get_json() self.assertIsNotNone(result) self.assertTrue('result' in result) result = result['result'] self.assertIsNotNone(result['uuid']) #self.assertIsNotNone(result['id']) uuid = result['uuid'] #_id = result['id'] result = self.client.get(url_for('analysis.get_analysis', uuid=uuid)) result = result.get_json() self.assertIsNotNone(result) self.assertTrue('result' in result) result = result['result'] self.assertEquals(result['analysis_mode'], 'analysis') self.assertEquals(result['tool'], 'unittest') self.assertEquals(result['tool_instance'], 'unittest_instance') self.assertEquals(result['type'], 'unittest') self.assertEquals(result['description'], 'testing') self.assertEquals(result['event_time'], '2017-11-11T07:36:01.000001+0000') self.assertEquals(result['tags'][0], 'alert_tag_1') self.assertEquals(result['tags'][1], 'alert_tag_2') self.assertEquals(len(result['observable_store']), 3) file_uuid = None for o_uuid in result['observable_store']: o = result['observable_store'][o_uuid] if o['type'] == F_IPV4: self.assertEquals(o['type'], F_IPV4) self.assertEquals(o['value'], '1.2.3.4') self.assertEquals(o['time'], '2017-11-11T07:36:01.000001+0000') self.assertEquals(o['tags'][0], 'tag_1') self.assertEquals(o['tags'][1], 'tag_2') self.assertEquals(o['directives'][0], DIRECTIVE_NO_SCAN) self.assertEquals(o['limited_analysis'][0], 'basic_test') elif o['type'] == F_USER: self.assertEquals(o['type'], F_USER) self.assertEquals(o['value'], 'test_user') self.assertEquals(o['time'], '2017-11-11T07:36:01.000001+0000') elif o['type'] == F_FILE: self.assertEquals(o['type'], F_FILE) self.assertEquals(o['value'], 'sample.dat') self.assertIsNone(o['time']) self.assertIsNotNone(o['id']) file_uuid = o['id'] # we should see a single workload entry c.execute( "SELECT id, uuid, node_id, analysis_mode FROM workload WHERE uuid = %s", (uuid, )) row = c.fetchone() self.assertIsNotNone(row) self.assertIsNotNone(row[0]) self.assertEquals(row[1], uuid) self.assertEquals(row[2], saq.SAQ_NODE_ID) self.assertEquals(row[3], 'analysis') result = self.client.get( url_for('analysis.get_details', uuid=uuid, name=result['details']['file_path'])) result = result.get_json() self.assertIsNotNone(result) result = result['result'] self.assertTrue('hello' in result) self.assertEquals(result['hello'], 'world') result = self.client.get( url_for('analysis.get_file', uuid=uuid, file_uuid_or_name=file_uuid)) self.assertEquals(result.status_code, 200) self.assertEquals(result.data, b'Hello, world!') result = self.client.get( url_for('analysis.get_file', uuid=uuid, file_uuid_or_name='sample.dat')) self.assertEquals(result.status_code, 200) self.assertEquals(result.data, b'Hello, world!') result = self.client.get(url_for('analysis.get_status', uuid=uuid)) self.assertEquals(result.status_code, 200) result = result.get_json() self.assertIsNotNone(result) result = result['result'] self.assertTrue('workload' in result) self.assertTrue('delayed_analysis' in result) self.assertTrue('locks' in result) self.assertEquals(result['delayed_analysis'], []) self.assertIsNone(result['locks']) self.assertTrue(isinstance(result['workload']['id'], int)) self.assertEquals(result['workload']['uuid'], uuid) self.assertEquals(result['workload']['node_id'], saq.SAQ_NODE_ID) self.assertEquals(result['workload']['analysis_mode'], 'analysis') self.assertTrue( isinstance(parse_event_time(result['workload']['insert_date']), datetime.datetime))