def spawn_engagers(self, providers, entity_type, entity_json_string, force, force_save):
self.logger.info('Started spawning engagers flows.')
# Iterate over all required providers and launch enrichment
result_futures = {}
for provider_name in providers:
try:
result_futures[provider_name] = self.launcher.launch.call_async(provider_name, entity_type, entity_json_string, force)
self.logger.info('Launched (async) engagement with %s', provider_name)
except EngagementException as ee:
self.logger.error('Failed to engage via %s on entity <to specify...> (exception: %s)', provider_name, ee)
self.logger.info('Completed spawning all engagers.')
engagement_results = {}
for provider_name in result_futures.keys():
try:
res = result_futures[provider_name].result()
engagement_results[provider_name] = EngagementResult.from_json_string(res)
self.logger.info('Done collecting results of provider %s', provider_name)
except Exception as ee:
self.logger.info('Exception raised getting future result of %s: %s. Ignoring this result.', provider_name, ee)
self.logger.info('Completed collecting results from all engagers')
# Recreate entity
new_entity = self._get_entity_by_type(entity_type)
entity = new_entity.from_json_string(entity_json_string)
self.logger.info('Done recreating entity from json - %s', entity)
# Merge all results into entity
changed = False
redigest_properties = {}
for provider_name, engagement_result in engagement_results.items():
if engagement_result.status != EngagementResult.SKIPPED and engagement_result.status != EngagementResult.NOCHANGE:
enrich_key = engagement_result.properties_changed['enrich_key']
for k, v in engagement_result.properties_changed.items():
property_changed = entity.set_data(provider_name, enrich_key, k, v)
if property_changed and k in LISTS.TRIGGERING_PROPERTIES:
redigest_properties[k] = v
changed |= property_changed
self.logger.info('Done merging properties of %s. Changed = %s', provider_name, changed)
pass
if changed or force_save:
entity.last_update = datetime.datetime.now()
# Digest and store
entity.digest()
Store.set_entity(entity_type, entity)
self.logger.info('Stored in Store! (changed=%s, force_save=%s)', changed, force_save)
# Redigest other entities
self.redigest(redigest_properties)
else:
self.logger.info('Not stored. No change detected')
pass
def spawn_engagers_sequentially(self, providers, entity_type, entity, enrichment_behavior, enriched=False):
org_entity = copy.deepcopy(entity)
# Iterate over all required providers and run enrichment
engagement_results = {}
el = EngagerLauncher()
for provider_name in providers:
try:
res = el.launch(provider_name, entity_type, entity, enrichment_behavior.force)
engagement_results[provider_name] = EngagementResult.from_json_string(res)
except EngagementException as ee:
self.logger.error('Failed to engage via %s on entity %s (exception: %s)', provider_name, entity.aid, ee)
# Recreate entity
new_entity = org_entity
# Merge all results into entity
#changed = False
changed = enriched
redigest_properties = {}
for provider_name, engagement_result in engagement_results.items():
if engagement_result.status != EngagementResult.SKIPPED and engagement_result.status != EngagementResult.NOCHANGE:
enrich_key = engagement_result.properties_changed['enrich_key']
for k, v in engagement_result.properties_changed.items():
property_changed = new_entity.set_data(provider_name, enrich_key, k, v)
if property_changed and k in LISTS.TRIGGERING_PROPERTIES:
redigest_properties[k] = v
changed |= property_changed
self.logger.info('Done merging properties of %s. Changed = %s', provider_name, changed)
pass
if changed or enrichment_behavior.force_save:
new_entity.last_update = datetime.datetime.now()
new_entity.digest()
Store.set_entity(entity_type, new_entity)
msg = 'Stored in Store! (changed=%s, force_save=%s)' % (changed, enrichment_behavior.force_save)
# Redigest other entities
self.redigest(redigest_properties)
else:
msg = 'Not stored. No change detected'
self.logger.info(msg)
# Prepare information to send to webhook
if enrichment_behavior.webhook:
payload = {'status_message': msg, 'status_code': 200, 'ts': time.time(), 'aid': new_entity.aid}
r = AcureRateUtils.announce(enrichment_behavior.webhook, payload)
self.logger.info('Done merging enrichment result into entity. Changed = %s', changed)
def redigest(self, redigest_properties):
if True:
return
self.logger.info('Started redigest according to these properties: %s', redigest_properties)
# TODO: fix the below to work also with companies - be agnostic to the entity type!
# Run a query to get all those entities who have the value in the field "unrecognized_<prop>":
for k, v in redigest_properties.items():
if k == 'name': # TODO: temp code. Need to align Mongo properties and all is well...
k = 'companies'
q = {'deduced.unrecognized_%s' % k: v}
cursor = Store.get_entities('people', q, mongo_query=True)
for entity in cursor:
entity.digest()
Store.set_entity('people', entity)
self.logger.info('Done redigesting entities of property %s', k)
self.logger.info('Done redigesting all entities.')
pass