def observe_observables():
credentials = get_credentials()
observables = get_observables()
client = XForceClient(current_app.config['API_URL'], credentials,
current_app.config['USER_AGENT'])
ui_url = current_app.config['UI_URL']
number_of_days_verdict_valid = int(
current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID'])
number_of_days_judgement_valid = int(
current_app.config['NUMBER_OF_DAYS_JUDGEMENT_IS_VALID'])
number_of_days_indicator_valid = int(
current_app.config['NUMBER_OF_DAYS_INDICATOR_IS_VALID'])
limit = current_app.config['CTR_ENTITIES_LIMIT']
g.bundle = Bundle()
try:
for observable in observables:
refer_link = client.refer_link(ui_url, observable)
mapping = Mapping.for_(observable, source_uri=refer_link)
if mapping:
if limit > 0:
report = client.report(observable)
if report:
report_bundle = mapping.process_report_data(
report, number_of_days_verdict_valid,
number_of_days_judgement_valid,
number_of_days_indicator_valid, limit)
limit -= len(report_bundle.get(SIGHTING))
g.bundle.merge(report_bundle)
if limit > 0 and isinstance(mapping, DNSInformationMapping):
resolutions_bundle = mapping.process_resolutions(
client.resolve(observable))
limit -= len(resolutions_bundle.get(SIGHTING))
g.bundle.merge(resolutions_bundle)
if limit > 0:
api_linkage = client.api_linkage(observable)
if api_linkage:
api_linkage_bundle = mapping.process_api_linkage(
api_linkage, ui_url,
number_of_days_indicator_valid, limit)
g.bundle.merge(api_linkage_bundle)
except KeyError:
add_error(XForceKeyError())
return jsonify_result()
def _get_pages(self, observable, endpoint, *args, **kwargs):
def extract(response, meta_field):
return (response.get('pages')
or response.get('meta', {}).get(meta_field) or 0)
def get_page(p):
return add_pause(endpoint, observable, *args, page=p, **kwargs)
data = get_page(1)
if data and data.get('records') and self.number_of_pages > 1:
rate_limit = self._rate_limit()
max_page = extract(data, 'max_page')
total_pages = extract(data, 'total_pages')
pages_left = 0
if self.number_of_pages < max_page:
max_page = self.number_of_pages
elif max_page < total_pages:
pages_left = total_pages - max_page
with ThreadPoolExecutor(
max_workers=min(rate_limit, (cpu_count() or 1) *
5)) as executor:
iterator = executor.map(
get_page, [page for page in range(2, max_page + 1)])
for r in iterator:
if r and r.get('records'):
data['records'].extend(r['records'])
else:
break
if pages_left:
add_error(
UnprocessedPagesWarning(observable['value'], pages_left))
return data
def deliberate_observables():
def deliberate(observable):
mapping = Mapping.for_(observable)
client_data = client.report(observable)
if client_data:
return mapping.extract_verdict(client_data,
number_of_days_verdict_valid)
credentials = get_credentials()
observables = get_observables()
observables = [
ob for ob in observables if ob['type'] in XFORCE_OBSERVABLE_TYPES
]
client = XForceClient(current_app.config['API_URL'], credentials,
current_app.config['USER_AGENT'])
number_of_days_verdict_valid = int(
current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID'])
g.bundle = Bundle()
try:
with ThreadPoolExecutor(
max_workers=min(len(observables) or 1, (cpu_count() or 1) *
5)) as executor:
iterator = executor.map(deliberate, observables)
g.bundle = Bundle(
*[verdict for verdict in iterator if verdict is not None])
except KeyError:
add_error(XForceKeyError())
return jsonify_result()
def handle_tr_formatted_error(error):
app.logger.error(traceback.format_exc())
add_error(error)
return jsonify_result()
def handle_tr_formatted_error(error):
app.logger.error(error.json)
add_error(error)
return jsonify_result()