def spawn_engagers(self, providers, entity_type, entity_json_string, force, force_save): self.logger.info('Started spawning engagers flows.') # Iterate over all required providers and launch enrichment result_futures = {} for provider_name in providers: try: result_futures[provider_name] = self.launcher.launch.call_async(provider_name, entity_type, entity_json_string, force) self.logger.info('Launched (async) engagement with %s', provider_name) except EngagementException as ee: self.logger.error('Failed to engage via %s on entity <to specify...> (exception: %s)', provider_name, ee) self.logger.info('Completed spawning all engagers.') engagement_results = {} for provider_name in result_futures.keys(): try: res = result_futures[provider_name].result() engagement_results[provider_name] = EngagementResult.from_json_string(res) self.logger.info('Done collecting results of provider %s', provider_name) except Exception as ee: self.logger.info('Exception raised getting future result of %s: %s. Ignoring this result.', provider_name, ee) self.logger.info('Completed collecting results from all engagers') # Recreate entity new_entity = self._get_entity_by_type(entity_type) entity = new_entity.from_json_string(entity_json_string) self.logger.info('Done recreating entity from json - %s', entity) # Merge all results into entity changed = False redigest_properties = {} for provider_name, engagement_result in engagement_results.items(): if engagement_result.status != EngagementResult.SKIPPED and engagement_result.status != EngagementResult.NOCHANGE: enrich_key = engagement_result.properties_changed['enrich_key'] for k, v in engagement_result.properties_changed.items(): property_changed = entity.set_data(provider_name, enrich_key, k, v) if property_changed and k in LISTS.TRIGGERING_PROPERTIES: redigest_properties[k] = v changed |= property_changed self.logger.info('Done merging properties of %s. Changed = %s', provider_name, changed) pass if changed or force_save: entity.last_update = datetime.datetime.now() # Digest and store entity.digest() Store.set_entity(entity_type, entity) self.logger.info('Stored in Store! (changed=%s, force_save=%s)', changed, force_save) # Redigest other entities self.redigest(redigest_properties) else: self.logger.info('Not stored. No change detected') pass
def spawn_engagers_sequentially(self, providers, entity_type, entity, enrichment_behavior, enriched=False): org_entity = copy.deepcopy(entity) # Iterate over all required providers and run enrichment engagement_results = {} el = EngagerLauncher() for provider_name in providers: try: res = el.launch(provider_name, entity_type, entity, enrichment_behavior.force) engagement_results[provider_name] = EngagementResult.from_json_string(res) except EngagementException as ee: self.logger.error('Failed to engage via %s on entity %s (exception: %s)', provider_name, entity.aid, ee) # Recreate entity new_entity = org_entity # Merge all results into entity #changed = False changed = enriched redigest_properties = {} for provider_name, engagement_result in engagement_results.items(): if engagement_result.status != EngagementResult.SKIPPED and engagement_result.status != EngagementResult.NOCHANGE: enrich_key = engagement_result.properties_changed['enrich_key'] for k, v in engagement_result.properties_changed.items(): property_changed = new_entity.set_data(provider_name, enrich_key, k, v) if property_changed and k in LISTS.TRIGGERING_PROPERTIES: redigest_properties[k] = v changed |= property_changed self.logger.info('Done merging properties of %s. Changed = %s', provider_name, changed) pass if changed or enrichment_behavior.force_save: new_entity.last_update = datetime.datetime.now() new_entity.digest() Store.set_entity(entity_type, new_entity) msg = 'Stored in Store! (changed=%s, force_save=%s)' % (changed, enrichment_behavior.force_save) # Redigest other entities self.redigest(redigest_properties) else: msg = 'Not stored. No change detected' self.logger.info(msg) # Prepare information to send to webhook if enrichment_behavior.webhook: payload = {'status_message': msg, 'status_code': 200, 'ts': time.time(), 'aid': new_entity.aid} r = AcureRateUtils.announce(enrichment_behavior.webhook, payload) self.logger.info('Done merging enrichment result into entity. Changed = %s', changed)
def redigest(self, redigest_properties): if True: return self.logger.info('Started redigest according to these properties: %s', redigest_properties) # TODO: fix the below to work also with companies - be agnostic to the entity type! # Run a query to get all those entities who have the value in the field "unrecognized_<prop>": for k, v in redigest_properties.items(): if k == 'name': # TODO: temp code. Need to align Mongo properties and all is well... k = 'companies' q = {'deduced.unrecognized_%s' % k: v} cursor = Store.get_entities('people', q, mongo_query=True) for entity in cursor: entity.digest() Store.set_entity('people', entity) self.logger.info('Done redigesting entities of property %s', k) self.logger.info('Done redigesting all entities.') pass