def observe_observables(): http_client = get_chronicle_http_client(get_jwt()) chronicle_client = ChronicleClient(current_app.config['API_URL'], http_client) observables = get_observables() limit = current_app.config['CTR_ENTITIES_LIMIT'] time_delta = current_app.config[ 'DEFAULT_NUMBER_OF_DAYS_FOR_CHRONICLE_TIME_FILTER'] g.sightings = [] g.indicators = [] g.relationships = [] for x in observables: mapping = Mapping.for_(x) if mapping: assets_data = chronicle_client.list_assets(x, time_delta, limit) ioc_details = chronicle_client.list_ioc_details(x) x_sightings = mapping.extract_sightings(assets_data, limit) x_indicators = mapping.extract_indicators(ioc_details, limit) g.sightings.extend(x_sightings) g.indicators.extend(x_indicators) g.relationships.extend( mapping.create_relationships(x_sightings, x_indicators)) return jsonify_result()
def observe_observables(): key = get_key() observables = get_observables() client = SecurityTrailsClient(current_app.config['API_URL'], key, current_app.config['USER_AGENT'], current_app.config['NUMBER_OF_PAGES'], current_app.config['GET_ALL_PAGES']) g.sightings = [] try: for observable in observables: mapping = Mapping.for_(observable) if mapping: client_data = client.get_data(observable) for record in client_data: refer_link = client.refer_link( current_app.config['UI_URL'], observable ) sighting = mapping.extract_sighting(record, refer_link) if sighting: g.sightings.append(sighting) except KeyError: g.errors = [{ 'type': 'fatal', 'code': 'key error', 'message': 'The data structure of SecurityTrails ' 'has changed. The module is broken.' }] return jsonify_result()
def observe_observables(): client = Auth0SignalsClient(get_jwt()) observables = get_observables() g.verdicts = [] g.judgements = [] g.sightings = [] g.indicators = [] g.relationships = [] for observable in observables: if observable['type'] == 'ip': response_data = client.get_auth0_response(observable) if response_data: g.verdicts.append(extract_verdict(response_data, observable)) g.judgements.extend( extract_judgements(response_data, observable)) details = client.get_full_details(response_data) sightings = extract_sightings(observable, details) g.sightings.extend(sightings) indicators = extract_indicators(details) g.indicators.extend(indicators) g.relationships.extend( extract_relationships(sightings, indicators)) return jsonify_result()
def deliberate_observables(): client = Auth0SignalsClient(get_jwt()) observables = get_observables() g.verdicts = [] for observable in observables: if observable['type'] == 'ip': response_data = client.get_auth0_response(observable) if response_data: g.verdicts.append(extract_verdict(response_data, observable)) return jsonify_result()
def observe_observables(): credentials = get_credentials() observables = get_observables() client = XForceClient(current_app.config['API_URL'], credentials, current_app.config['USER_AGENT']) ui_url = current_app.config['UI_URL'] number_of_days_verdict_valid = int( current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID']) number_of_days_judgement_valid = int( current_app.config['NUMBER_OF_DAYS_JUDGEMENT_IS_VALID']) number_of_days_indicator_valid = int( current_app.config['NUMBER_OF_DAYS_INDICATOR_IS_VALID']) limit = current_app.config['CTR_ENTITIES_LIMIT'] g.bundle = Bundle() try: for observable in observables: refer_link = client.refer_link(ui_url, observable) mapping = Mapping.for_(observable, source_uri=refer_link) if mapping: if limit > 0: report = client.report(observable) if report: report_bundle = mapping.process_report_data( report, number_of_days_verdict_valid, number_of_days_judgement_valid, number_of_days_indicator_valid, limit) limit -= len(report_bundle.get(SIGHTING)) g.bundle.merge(report_bundle) if limit > 0 and isinstance(mapping, DNSInformationMapping): resolutions_bundle = mapping.process_resolutions( client.resolve(observable)) limit -= len(resolutions_bundle.get(SIGHTING)) g.bundle.merge(resolutions_bundle) if limit > 0: api_linkage = client.api_linkage(observable) if api_linkage: api_linkage_bundle = mapping.process_api_linkage( api_linkage, ui_url, number_of_days_indicator_valid, limit) g.bundle.merge(api_linkage_bundle) except KeyError: add_error(XForceKeyError()) return jsonify_result()
def observe_observables(): misp = create_misp_instance() observables = filter_observables(get_observables()) g.verdicts = [] g.judgements = [] g.sightings = [] g.indicators = [] g.relationships = [] for observable in observables: mapping = Mapping(observable) # This check was added due to the fact that MISP # returns all data in case of searching for spaces. if not observable['value'].strip(): continue events = misp.search(value=observable['value'], metadata=False, limit=current_app.config['CTR_ENTITIES_LIMIT']) events.sort(key=lambda elem: elem['Event']['threat_level_id']) judgements_for_observable = [] for event in events: judgement = mapping.extract_judgement(event['Event']) judgements_for_observable.append(judgement) sighting = mapping.extract_sighting(event['Event']) g.sightings.append(sighting) indicator = mapping.extract_indicator(event['Event']) g.indicators.append(indicator) g.relationships.append( mapping.extract_relationship( sighting['id'], indicator['id'], current_app.config['MEMBER_OF_RELATION'])) g.relationships.append( mapping.extract_relationship( judgement['id'], indicator['id'], current_app.config['ELEMENT_OF_RELATION'])) if judgements_for_observable: g.judgements.extend(judgements_for_observable) verdict = mapping.extract_verdict(events[0]['Event']) verdict['judgement_id'] = judgements_for_observable[0]['id'] g.verdicts.append(verdict) return jsonify_result()
def observe_observables(): def query_qradar(obs): query_with_observables = arial_query.build( observable=obs, names=names, time=get_time_intervals(), limit=current_app.config['CTR_ENTITIES_LIMIT']) if not query_with_observables: return jsonify_data({}) search_id, status = api_client.create_search(query_with_observables) start_time = time() while status != 'COMPLETED': if (time() - start_time < current_app.config['SEARCH_TIMOUT_IN_SEC']): status = api_client.get_search(search_id)['status'] else: raise QRadarTimeoutError return api_client.get_search_results(search_id)['events'] relay_input = get_observables() observables = filter_observables(relay_input) if not observables: return jsonify_data({}) api_client = ArielSearchClient(get_credentials(), current_app.config) arial_query = ArielQuery() params = { 'fields': 'columns (name)', 'filter': 'object_value_type = "Host"' } names = api_client.get_metadata(params) g.sightings = [] for observable in observables: response = query_qradar(observable) for event in response: mapping = Mapping() g.sightings.append(mapping.sighting(observable, event)) return jsonify_result()
def observe_observables(): payload = get_jwt() client = APIVoidClient(payload) observables = get_observables() g.sightings = [] g.indicators = [] g.relationships = [] for observable in observables: output = client.get_data(observable) if output: for engine in get_engines(output): sighting = extract_sighting(observable, engine) g.sightings.append(sighting) indicator = extract_indicator(engine) g.indicators.append(indicator) g.relationships.append( extract_relationship(sighting['id'], indicator['id'])) return jsonify_result()
def respond_trigger(): add_status('failure') params = get_action_form_params() client = ReferenceDataClient(get_credentials(), current_app.config) action_map = { ADD_ACTION_ID: client.add_to_reference_set, REMOVE_ACTION_ID: client.remove_from_reference_set, ADD_AND_CRETE_ACTION_ID: client.add_and_create_reference_set } action = action_map.get(params['action-id']) if not action: raise InvalidArgumentError('Unsupported action.') action(params['reference_set_name'], params['observable_value']) add_status('success') return jsonify_result()
def respond_trigger(): add_status('failure') params = get_action_form_params() credentials = get_jwt() client = AkamaiClient(credentials, current_app.config['USER_AGENT']) action_map = { ADD_ACTION_ID: client.add_to_network_list, REMOVE_ACTION_ID: client.remove_from_network_list } action = action_map.get(params['action-id']) if not action: raise InvalidArgumentError("Unsupported action.") action(params['network_list_id'], params['observable_value']) add_status('success') return jsonify_result()
def deliberate_observables(): misp = create_misp_instance() observables = filter_observables(get_observables()) g.verdicts = [] for observable in observables: mapping = Mapping(observable) events = misp.search(value=observable['value'], metadata=False) events.sort(key=lambda elem: elem['Event']['threat_level_id']) # We sort events in order to create a single Verdict for a set # of events based on the fact that High threat level has # priority over all others, then Medium threat level, # and so on down to Undefined. if events: g.verdicts.append(mapping.extract_verdict(events[0]['Event'])) return jsonify_result()
def observe_observables(): key = get_key() observables = get_observables() client = FarsightClient(current_app.config['API_URL'], key, current_app.config['USER_AGENT']) g.sightings = [] limit = current_app.config['CTR_ENTITIES_LIMIT'] aggr = current_app.config['AGGREGATE'] time_delta = (current_app.config['NUMBER_OF_DAYS_FOR_FARSIGHT_TIME_FILTER'] if aggr else None) url_template = current_app.config['UI_SEARCH_URL'] try: for x in observables: mapping = Mapping.for_(x) if mapping: lookup_data = client.lookup(x, time_delta) if lookup_data: refer_link = url_template.format(query=x['value']) g.sightings.extend( mapping.extract_sightings( lookup_data, refer_link, limit, aggr ) ) except KeyError: g.errors = [{ 'type': 'fatal', 'code': 'key error', 'message': 'The data structure of Farsight DNSDB ' 'has changed. The module is broken.' }] return jsonify_result()
def observe_observables(): key = get_jwt().get('key', '') client = C1fAppClient(key) observables = get_observables() g.sightings = [] g.indicators = [] g.relationships = [] limit = current_app.config['CTR_ENTITIES_LIMIT'] for observable in observables: mapping = Mapping.for_(observable) if mapping: response_data = client.get_c1fapp_response(observable['value']) response_data.sort(key=lambda x: x['reportime'], reverse=True) response_data = response_data[:limit] g.sightings.extend(mapping.extract_sightings(response_data)) g.indicators.extend(mapping.extract_indicators(response_data)) g.relationships.extend(mapping.extract_relationships()) return jsonify_result()
def deliberate_observables(): def deliberate(observable): mapping = Mapping.for_(observable) client_data = client.report(observable) if client_data: return mapping.extract_verdict(client_data, number_of_days_verdict_valid) credentials = get_credentials() observables = get_observables() observables = [ ob for ob in observables if ob['type'] in XFORCE_OBSERVABLE_TYPES ] client = XForceClient(current_app.config['API_URL'], credentials, current_app.config['USER_AGENT']) number_of_days_verdict_valid = int( current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID']) g.bundle = Bundle() try: with ThreadPoolExecutor( max_workers=min(len(observables) or 1, (cpu_count() or 1) * 5)) as executor: iterator = executor.map(deliberate, observables) g.bundle = Bundle( *[verdict for verdict in iterator if verdict is not None]) except KeyError: add_error(XForceKeyError()) return jsonify_result()
def handle_tr_formatted_error(error): add_error(error) app.logger.error(error.json) return jsonify_result()
def handle_tr_formatted_error(error): app.logger.error(error.json) g.errors = [error.json] return jsonify_result()
def handle_tr_formatted_error(error): app.logger.error(traceback.format_exc()) add_error(error) return jsonify_result()
def handle_tr_formatted_error(error): g.errors = [error.json] return jsonify_result()