def observe_observables():
key = get_key()
observables = get_observables()
client = SecurityTrailsClient(current_app.config['API_URL'],
key,
current_app.config['USER_AGENT'],
current_app.config['NUMBER_OF_PAGES'],
current_app.config['GET_ALL_PAGES'])
g.sightings = []
try:
for observable in observables:
mapping = Mapping.for_(observable)
if mapping:
client_data = client.get_data(observable)
for record in client_data:
refer_link = client.refer_link(
current_app.config['UI_URL'], observable
)
sighting = mapping.extract_sighting(record, refer_link)
if sighting:
g.sightings.append(sighting)
except KeyError:
g.errors = [{
'type': 'fatal',
'code': 'key error',
'message': 'The data structure of SecurityTrails '
'has changed. The module is broken.'
}]
return jsonify_result()
def observe_observables():
http_client = get_chronicle_http_client(get_jwt())
chronicle_client = ChronicleClient(current_app.config['API_URL'],
http_client)
observables = get_observables()
limit = current_app.config['CTR_ENTITIES_LIMIT']
time_delta = current_app.config[
'DEFAULT_NUMBER_OF_DAYS_FOR_CHRONICLE_TIME_FILTER']
g.sightings = []
g.indicators = []
g.relationships = []
for x in observables:
mapping = Mapping.for_(x)
if mapping:
assets_data = chronicle_client.list_assets(x, time_delta, limit)
ioc_details = chronicle_client.list_ioc_details(x)
x_sightings = mapping.extract_sightings(assets_data, limit)
x_indicators = mapping.extract_indicators(ioc_details, limit)
g.sightings.extend(x_sightings)
g.indicators.extend(x_indicators)
g.relationships.extend(
mapping.create_relationships(x_sightings, x_indicators))
return jsonify_result()
def test_mapping_of():
assert isinstance(Mapping.of('domain'), Domain)
assert isinstance(Mapping.of('file_name'), FileName)
assert isinstance(Mapping.of('file_path'), FilePath)
assert isinstance(Mapping.of('sha256'), SHA256)
assert isinstance(Mapping.of('ip'), IP)
assert isinstance(Mapping.of('url'), URL)
assert isinstance(Mapping.of('hostname'), Hostname)
assert Mapping.of('whatever') is None
def test_mapping_for_():
assert isinstance(Mapping.for_({'type': 'domain'}), Domain)
assert isinstance(Mapping.for_({'type': 'url'}), URL)
assert isinstance(Mapping.for_({'type': 'ip'}), IP)
assert isinstance(Mapping.for_({'type': 'ipv6'}), IPV6)
assert isinstance(Mapping.for_({'type': 'md5'}), MD5)
assert isinstance(Mapping.for_({'type': 'sha1'}), SHA1)
assert isinstance(Mapping.for_({'type': 'sha256'}), SHA256)
assert Mapping.for_({'type': 'whatever'}) is None
def observe_observables():
credentials = get_credentials()
observables = get_observables()
client = XForceClient(current_app.config['API_URL'], credentials,
current_app.config['USER_AGENT'])
ui_url = current_app.config['UI_URL']
number_of_days_verdict_valid = int(
current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID'])
number_of_days_judgement_valid = int(
current_app.config['NUMBER_OF_DAYS_JUDGEMENT_IS_VALID'])
number_of_days_indicator_valid = int(
current_app.config['NUMBER_OF_DAYS_INDICATOR_IS_VALID'])
limit = current_app.config['CTR_ENTITIES_LIMIT']
g.bundle = Bundle()
try:
for observable in observables:
refer_link = client.refer_link(ui_url, observable)
mapping = Mapping.for_(observable, source_uri=refer_link)
if mapping:
if limit > 0:
report = client.report(observable)
if report:
report_bundle = mapping.process_report_data(
report, number_of_days_verdict_valid,
number_of_days_judgement_valid,
number_of_days_indicator_valid, limit)
limit -= len(report_bundle.get(SIGHTING))
g.bundle.merge(report_bundle)
if limit > 0 and isinstance(mapping, DNSInformationMapping):
resolutions_bundle = mapping.process_resolutions(
client.resolve(observable))
limit -= len(resolutions_bundle.get(SIGHTING))
g.bundle.merge(resolutions_bundle)
if limit > 0:
api_linkage = client.api_linkage(observable)
if api_linkage:
api_linkage_bundle = mapping.process_api_linkage(
api_linkage, ui_url,
number_of_days_indicator_valid, limit)
g.bundle.merge(api_linkage_bundle)
except KeyError:
add_error(XForceKeyError())
return jsonify_result()
def test_create_relationships():
sightings = [{'id': 's1'}, {'id': 's2'}, {'id': 's3'}]
indicators = []
relationships = Mapping.create_relationships(sightings, indicators)
assert relationships == []
sightings = []
indicators = [{'id': 'i1'}, {'id': 'i2'}]
relationships = Mapping.create_relationships(sightings, indicators)
assert relationships == []
sightings = [{'id': 's1'}, {'id': 's2'}, {'id': 's3'}]
indicators = [{'id': 'i1'}, {'id': 'i2'}]
relationships = Mapping.create_relationships(sightings, indicators)
for relationship in relationships:
assert relationship.pop('id').startswith('transient:relationship-')
assert relationships == [
{'schema_version': '1.0.17',
'type': 'relationship', 'relationship_type': 'sighting-of',
'source_ref': 's1', 'target_ref': 'i1'},
{'schema_version': '1.0.17',
'type': 'relationship', 'relationship_type': 'sighting-of',
'source_ref': 's2', 'target_ref': 'i1'},
{'schema_version': '1.0.17',
'type': 'relationship', 'relationship_type': 'sighting-of',
'source_ref': 's3', 'target_ref': 'i1'},
{'schema_version': '1.0.17',
'type': 'relationship', 'relationship_type': 'sighting-of',
'source_ref': 's1', 'target_ref': 'i2'},
{'schema_version': '1.0.17',
'type': 'relationship', 'relationship_type': 'sighting-of',
'source_ref': 's2', 'target_ref': 'i2'},
{'schema_version': '1.0.17',
'type': 'relationship', 'relationship_type': 'sighting-of',
'source_ref': 's3', 'target_ref': 'i2'}
]
def observe_observables():
key = get_key()
observables = get_observables()
client = FarsightClient(current_app.config['API_URL'],
key,
current_app.config['USER_AGENT'])
g.sightings = []
limit = current_app.config['CTR_ENTITIES_LIMIT']
aggr = current_app.config['AGGREGATE']
time_delta = (current_app.config['NUMBER_OF_DAYS_FOR_FARSIGHT_TIME_FILTER']
if aggr else None)
url_template = current_app.config['UI_SEARCH_URL']
try:
for x in observables:
mapping = Mapping.for_(x)
if mapping:
lookup_data = client.lookup(x, time_delta)
if lookup_data:
refer_link = url_template.format(query=x['value'])
g.sightings.extend(
mapping.extract_sightings(
lookup_data, refer_link, limit, aggr
)
)
except KeyError:
g.errors = [{
'type': 'fatal',
'code': 'key error',
'message': 'The data structure of Farsight DNSDB '
'has changed. The module is broken.'
}]
return jsonify_result()
def observe_observables():
key = get_jwt().get('key', '')
client = C1fAppClient(key)
observables = get_observables()
g.sightings = []
g.indicators = []
g.relationships = []
limit = current_app.config['CTR_ENTITIES_LIMIT']
for observable in observables:
mapping = Mapping.for_(observable)
if mapping:
response_data = client.get_c1fapp_response(observable['value'])
response_data.sort(key=lambda x: x['reportime'], reverse=True)
response_data = response_data[:limit]
g.sightings.extend(mapping.extract_sightings(response_data))
g.indicators.extend(mapping.extract_indicators(response_data))
g.relationships.extend(mapping.extract_relationships())
return jsonify_result()
def test_mapping_for_():
assert isinstance(Mapping.for_({'type': 'domain'}), Domain)
assert isinstance(Mapping.for_({'type': 'ip'}), IP)
assert isinstance(Mapping.for_({'type': 'ipv6'}), IPV6)
assert Mapping.for_({'type': 'whatever'}) is None
def deliberate(observable):
mapping = Mapping.for_(observable)
client_data = client.report(observable)
if client_data:
return mapping.extract_verdict(client_data,
number_of_days_verdict_valid)