def refer_observables(): observables, error = get_observables() if error: return jsonify_errors(error) observable_types = current_app.config['GTI_OBSERVABLE_TYPES'] def type_of(observable): return observable_types[observable['type']] url_template = current_app.config['GTI_UI_SEARCH_URL'] data = [ { 'id': 'ref-gti-search-{type}-{value}'.format(**observable), 'title': f'Search for this {type_of(observable)}', 'description': ( f'Lookup this {type_of(observable)} ' 'on Gigamon ThreatINSIGHT' ), 'url': url_template.format(query=observable['value']), 'categories': ['Search', 'Gigamon ThreatINSIGHT'], } for observable in observables if observable['type'] in observable_types ] return jsonify_data(data)
def observe_observables(): observables, error = get_observables() if error: return jsonify_errors(error) observable_types = current_app.config['GTI_OBSERVABLE_TYPES'] observables = [ observable for observable in observables if observable['type'] in observable_types ] key = get_key() bundle = Bundle() for observable in observables: events, error = get_events_for_observable(key, observable) if error: # Make sure not to lose any data processed so far. return jsonify_errors(error, data=bundle.json()) indicator_by_rule_uuid = {} for event in events: sighting = Sighting.map(event) bundle.add(sighting) if 'detection' in event: rule = event['detection']['rule'] indicator = indicator_by_rule_uuid.get(rule['uuid']) if indicator is None: indicator = Indicator.map(rule) indicator_by_rule_uuid[rule['uuid']] = indicator bundle.add(indicator) relationship = Relationship.map(sighting, indicator) bundle.add(relationship) data = bundle.json() return jsonify_data(data)
def health(): url = url_for('threatLists') if url is None: # Mimic the GSB API error response payload. error = { 'code': HTTPStatus.FORBIDDEN, 'message': 'The request is missing a valid API key.', 'status': 'PERMISSION_DENIED', } return jsonify_errors(error) _, error = execute(requests.get, url, headers=headers()) if error: return jsonify_errors(error) else: return jsonify_data({'status': 'ok'})
def observe_observables(): observables, error = get_observables() if error: return jsonify_errors(error) emails = [ observable['value'] for observable in observables if observable['type'] == 'email' ] key = get_key() bundle = Bundle() limit = current_app.config['CTR_ENTITIES_LIMIT'] for email in emails: breaches, error = fetch_breaches(key, email) if error: return jsonify_errors(error, data=bundle.json()) breaches.sort(key=itemgetter('BreachDate'), reverse=True) breaches = breaches[:limit] source_uri = current_app.config['HIBP_UI_URL'].format( email=quote(email, safe='')) for breach in breaches: indicator = Indicator.map(breach) sighting = Sighting.map(breach, email, source_uri) relationship = Relationship.map(indicator, sighting) bundle.add(indicator) bundle.add(sighting) bundle.add(relationship) data = bundle.json() return jsonify_data(data)
def health(): key = get_key() # Use some supported entity just to check that the GTI API key is valid. observable = current_app.config['GTI_TEST_ENTITY'] _, error = get_events(key, observable) if error: return jsonify_errors(error) else: return jsonify_data({'status': 'ok'})
def health(): key = get_key() # Use some breached email just to check that the HIBP API key is valid. email = current_app.config['HIBP_TEST_EMAIL'] _, error = fetch_breaches(key, email, truncate=True) if error: return jsonify_errors(error) else: return jsonify_data({'status': 'ok'})
def observe_observables(): relay_input, error = validate_relay_input() if error: return jsonify_errors(error) observables = group_observables(relay_input) if not observables: # Optimize a bit by not sending empty requests to the GSB API. return jsonify_data({}) bundle = Bundle() start_time = datetime.utcnow() # Split the data into chunks and make multiple requests to the GSB API. size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST'] for observables in map(dict, chunks(observables.items(), size)): gsb_input = build_gsb_input(observables) gsb_output, error = fetch_gsb_output(gsb_input) if error: return jsonify_errors(error, data=bundle.json()) matches = group_matches(gsb_output) # Extract judgements first in order to label each match with some # "judgement_id", so that it can be extracted for each verdict later. judgements = extract_judgements(observables, matches, start_time) verdicts = extract_verdicts(observables, matches, start_time) for entity in chain(judgements, verdicts): bundle.add(entity) relay_output = bundle.json() return jsonify_data(relay_output)
def observe_observables(): observables, error = get_observables() if error: return jsonify_errors(error) observables = group_observables(observables) if not observables: return jsonify_data({}) data = {} g.verdicts = [] g.judgements = [] g.errors = [] observables = build_input_api(observables) try: for observable in observables: o_value = observable['value'] o_type = observable['type'].lower() response = call_api(o_value) disposition_tuple = get_disposition(response['message']) if not disposition_tuple: continue start_time = datetime.utcnow() end_time = start_time + timedelta(weeks=1) valid_time = { 'start_time': start_time.isoformat() + 'Z', 'end_time': end_time.isoformat() + 'Z', } g.verdicts.append( get_verdict(o_value, o_type, disposition_tuple, valid_time)) g.judgements.append( get_judgement(o_value, o_type, disposition_tuple, valid_time)) except KeyError: g.errors.append(key_error()) if g.verdicts: data['verdicts'] = format_docs(g.verdicts) if g.judgements: data['judgements'] = format_docs(g.judgements) result = {'data': data} if g.errors: result['errors'] = g.errors return jsonify(result)
def deliberate_observables(): relay_input, error = validate_relay_input() if error: return jsonify_errors(error) observables = group_observables(relay_input) if not observables: # Optimize a bit by not sending empty requests to the GSB API. return jsonify_data({}) bundle = Bundle() start_time = datetime.utcnow() # Split the data into chunks and make multiple requests to the GSB API. size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST'] for observables in map(dict, chunks(observables.items(), size)): gsb_input = build_gsb_input(observables) gsb_output, error = fetch_gsb_output(gsb_input) if error: return jsonify_errors(error, data=bundle.json()) matches = group_matches(gsb_output) verdicts = extract_verdicts(observables, matches, start_time) for entity in verdicts: bundle.add(entity) relay_output = bundle.json() return jsonify_data(relay_output)
def health(): try: response = requests.get(current_app.config['API_URL'], headers=current_app.config['CTR_HEADERS']) except requests.exceptions.SSLError as ex: raise CybercrimeSSLError(ex) if response.ok: return jsonify_data({'status': 'ok'}) error = { 'code': 'invalid_health_check', 'message': 'Invalid 3rd party API connect.', } return jsonify_errors(error)
def refer_observables(): observables, error = get_observables() if error: return jsonify_errors(error) observables = group_observables(observables) data = [] for observable in observables: o_type = observable['type'] o_value = observable['value'] if o_type in ('sha1', 'sha256'): entity = 'files' elif o_type in ('ip', 'ipv6'): entity = 'ips' elif o_type == 'domain': entity = 'urls' else: continue title = 'Search for this {o_type}'.format( o_type=current_app.config["MD_ATP_OBSERVABLE_TYPES"][o_type]) description = \ 'Lookup this {o_type} on Microsoft Defender for Endpoint'.format( o_type=current_app.config['MD_ATP_OBSERVABLE_TYPES'][o_type] ) url = '{host}/{entity}/{o_value}'.format( host=current_app.config['SECURITY_CENTER_URL'], entity=entity, o_value=o_value) data.append({ 'id': f'ref-md-search-{o_type}-{o_value}', 'title': title, 'description': description, 'url': url, 'categories': ['Search', 'Microsoft Defender for Endpoint'] }) return jsonify_data(data)
def refer_observables(): observables, error = get_observables() if error: return jsonify_errors(error) emails = [ observable['value'] for observable in observables if observable['type'] == 'email' ] data = [{ 'id': f'ref-hibp-search-email-{email}', 'title': 'Search for this email', 'description': 'Check this email status with Have I Been Pwned', 'url': current_app.config['HIBP_UI_URL'].format(email=email), 'categories': ['Search', 'Have I Been Pwned'], } for email in map(lambda email: quote(email, safe=''), emails)] return jsonify_data(data)
def refer_observables(): relay_input, error = validate_relay_input() if error: return jsonify_errors(error) observables = group_observables(relay_input) relay_output = [{ 'id': f'ref-gsb-search-{type}-{quote(value, safe="")}', 'title': ('Search for this ' f'{current_app.config["GSB_OBSERVABLE_TYPES"][type]}'), 'description': ('Check this ' f'{current_app.config["GSB_OBSERVABLE_TYPES"][type]} ' 'status with Google Safe Browsing'), 'url': (current_app.config['GSB_TRANSPARENCY_REPORT_URL'].format( url=quote(value, safe=':'))), 'categories': ['Search', 'Google Safe Browsing'], } for value, types in observables.items() for type in types] return jsonify_data(relay_output)
def respond_observables(): observables, error = get_observables() if error: return jsonify_errors(error) observables = group_observables(observables, 'respond') if not observables: return jsonify_data([]) g.actions = [] credentials = get_jwt() client = Client(credentials) client.open_session() for observable in observables: query_params = { 'observable_type': observable['type'], 'observable_value': observable['value'] } if observable['type'] == 'ms_machine_id': _actions = get_supported_actions(client, observable['value']) _actions = get_reverse_actions(client, observable['value'], _actions) actions = [] for item in _actions: action = {} action['id'] = \ f'microsoft-defender-{_TR_SUPPORTED_ACTIONS[item]}' action['title'] = item action['description'] = item action['categories'] = [ 'Microsoft Defender for Endpoint', 'Machine Actions' ] action['query-params'] = query_params actions.append(action) else: params = "$filter=indicatorValue+eq+'{value}'&$top=1".format( value=observable['value']).encode('utf-8') response, error = client.call_api( current_app.config['INDICATOR_URL'], params=params) if response and response.get('value'): obj = response['value'][0] human_action = 'Alert and Block' \ if obj['action'] == 'AlertAndBlock' else obj['action'] query_params['indicator_id'] = obj['id'] actions = [ { 'id': 'microsoft-defender-remove-indicator', 'title': 'Remove indicator: {action} - {title}'.format( action=human_action, title=obj['title']), 'description': f'Remove indicator with {human_action} ' f'action for {observable["value"]}', 'categories': [ 'Microsoft Defender for Endpoint', 'Remove Indicator' ], 'query-params': query_params }, ] else: actions = [{ 'id': 'microsoft-defender-add-indicator-alert', 'title': 'Add indicator: Alert', 'description': f'Add indicator with Alert action ' f'for {observable["value"]}', 'categories': ['Microsoft Defender for Endpoint', 'Add Indicator'], 'query-params': query_params }, { 'id': 'microsoft-defender-' 'add-indicator-alert-and-block', 'title': 'Add indicator: Alert and Block', 'description': f'Add indicator with ' f'Alert and Block action' f' for {observable["value"]}', 'categories': ['Microsoft Defender for Endpoint', 'Add Indicator'], 'query-params': query_params }, { 'id': 'microsoft-defender-add-indicator-allowed', 'title': 'Add indicator: Allow', 'description': f'Add indicator with Allow action ' f'for {observable["value"]}', 'categories': ['Microsoft Defender for Endpoint', 'Add Indicator'], 'query-params': query_params }] g.actions.extend(actions) client.close_session() return jsonify_data(g.actions)
def handle_tr_formatted_error(error): app.logger.error(error.json) return jsonify_errors(error.json)
def handle_tr_error(exception): app.logger.error(traceback.format_exc()) return jsonify_errors(exception.json)
def handle_relay_error(error): app.logger.error(traceback.format_exc()) return jsonify_errors(error)
def handle_tr_formatted_error(error): app.logger.error(traceback.format_exc()) return jsonify_errors(error.json)
def handle_relay_error(error): return jsonify_errors(error)
def observe_observables(): observables, error = get_observables() if error: return jsonify_errors(error) observables = group_observables(observables) if not observables: return jsonify_data({}) data = {} g.sightings = [] credentials = get_jwt() client = Client(credentials) for observable in observables: client.open_session() response = get_alert(client, observable) if not response or not response.get('value'): alerts = [] else: alerts = response['value'] alerts.sort(key=lambda x: x['alertCreationTime'], reverse=True) count = len(alerts) if count >= current_app.config['CTR_ENTITIES_LIMIT']: alerts = alerts[:current_app.config['CTR_ENTITIES_LIMIT']] events = [] else: events = call_advanced_hunting( client, observable['value'], observable['type'], current_app.config['CTR_ENTITIES_LIMIT'] - count) count = count + len(events) events.sort(key=lambda x: x['Timestamp'], reverse=True) mapping = Mapping(client, observable, count) if alerts: with ThreadPoolExecutor(max_workers=min(len(alerts), cpu_count() or 1) * 5) as executor: alerts = executor.map(mapping.build_sighting_from_alert, alerts) [g.sightings.append(alert) for alert in alerts if alert] if events: with ThreadPoolExecutor(max_workers=min(len(events), cpu_count() or 1) * 5) as executor: events = executor.map(mapping.build_sighting_from_ah, events) [g.sightings.append(event) for event in events if event] client.close_session() if g.sightings: data['sightings'] = format_docs(g.sightings) return jsonify_data(data)
def handle_tr_formatted_error(exception): app.logger.error(exception) return jsonify_errors(exception.json)
def respond_trigger(): mapping_by_type = { 'sha1': 'FileSha1', 'sha256': 'FileSha256', 'ip': 'IpAddress', 'ipv6': 'IpAddress', 'domain': 'DomainName' } data, error = get_action_form_params() if error: return jsonify_errors(error) title = 'From SecureX Threat Response' description = 'This indicator was added via SecureX Threat Response ' \ 'by the UI or API response actions' if data['observable_type'] == 'ms_machine_id': comment = 'Performed via SecureX Threat Response' actions = { 'microsoft-defender-FullIsolation': { 'url': '{base_url}/machines/{machine_id}/isolate'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment, 'IsolationType': 'Full' } }, 'microsoft-defender-SelectiveIsolation': { 'url': '{base_url}/machines/{machine_id}/isolate'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment, 'IsolationType': 'Selective' } }, 'microsoft-defender-Unisolate': { 'url': '{base_url}/machines/{machine_id}/unisolate'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment } }, 'microsoft-defender-RestrictCodeExecution': { 'url': '{base_url}/machines/{machine_id}' '/restrictCodeExecution'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment } }, 'microsoft-defender-UnrestrictCodeExecution': { 'url': '{base_url}/machines/{machine_id}' '/unrestrictCodeExecution'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment } }, 'microsoft-defender-RunAntiVirusScanQuick': { 'url': '{base_url}/machines/{machine_id}' '/runAntiVirusScan'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment, 'ScanType': 'Quick' } }, 'microsoft-defender-RunAntiVirusScanFull': { 'url': '{base_url}/machines/{machine_id}' '/runAntiVirusScan'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment, 'ScanType': 'Full' } }, 'microsoft-defender-CollectInvestigationPackage': { 'url': '{base_url}/machines/{machine_id}' '/collectInvestigationPackage'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment } }, 'microsoft-defender-InitiateInvestigation': { 'url': '{base_url}/machines/{machine_id}' '/startInvestigation'.format( base_url=current_app.config['BASE_URL'], machine_id=data['observable_value']), 'method': 'POST', 'data': { 'Comment': comment } } } else: actions = { 'microsoft-defender-add-indicator-alert': { 'url': current_app.config['INDICATOR_URL'], 'method': 'POST', 'data': { 'indicatorValue': data['observable_value'], 'indicatorType': mapping_by_type.get(data['observable_type']), 'action': 'Alert', 'title': title, 'description': description, 'severity': 'High' } }, 'microsoft-defender-add-indicator-alert-and-block': { 'url': current_app.config['INDICATOR_URL'], 'method': 'POST', 'data': { 'indicatorValue': data['observable_value'], 'indicatorType': mapping_by_type.get(data['observable_type']), 'action': 'AlertAndBlock', 'title': title, 'description': description, 'severity': 'High' } }, 'microsoft-defender-add-indicator-allowed': { 'url': current_app.config['INDICATOR_URL'], 'method': 'POST', 'data': { 'indicatorValue': data['observable_value'], 'indicatorType': mapping_by_type.get(data['observable_type']), 'action': 'Allowed', 'title': title, 'description': description } }, 'microsoft-defender' '-remove-indicator': { 'url': current_app.config['INDICATOR_URL'] + '/' + str(data.get('indicator_id', '')), 'method': 'DELETE', 'data': {} } } result = {'data': {'status': 'success'}} item = actions.get(data['action-id']) if not item: result['data']['status'] = 'failure' result['errors'] = [ CTRBadRequestError("Unsupported action.").json, ] return jsonify(result) if data['observable_type'] != 'ms_machine_id' and \ not item['data']['indicatorType']: raise UnsupportedTypeError(data['observable_type']) if item['data']: action = json.dumps(item['data']).encode('utf-8') else: action = None credentials = get_jwt() client = Client(credentials) client.open_session() response, error = client.call_api(item['url'], item['method'], data=action) client.close_session() if error is not None: result['data']['status'] = 'failure' result['errors'] = [ CTRBadRequestError(error).json, ] return jsonify(result)