def spawn_engagers_sequentially(self, providers, entity_type, entity, enrichment_behavior, enriched=False):
org_entity = copy.deepcopy(entity)
# Iterate over all required providers and run enrichment
engagement_results = {}
el = EngagerLauncher()
for provider_name in providers:
try:
res = el.launch(provider_name, entity_type, entity, enrichment_behavior.force)
engagement_results[provider_name] = EngagementResult.from_json_string(res)
except EngagementException as ee:
self.logger.error('Failed to engage via %s on entity %s (exception: %s)', provider_name, entity.aid, ee)
# Recreate entity
new_entity = org_entity
# Merge all results into entity
#changed = False
changed = enriched
redigest_properties = {}
for provider_name, engagement_result in engagement_results.items():
if engagement_result.status != EngagementResult.SKIPPED and engagement_result.status != EngagementResult.NOCHANGE:
enrich_key = engagement_result.properties_changed['enrich_key']
for k, v in engagement_result.properties_changed.items():
property_changed = new_entity.set_data(provider_name, enrich_key, k, v)
if property_changed and k in LISTS.TRIGGERING_PROPERTIES:
redigest_properties[k] = v
changed |= property_changed
self.logger.info('Done merging properties of %s. Changed = %s', provider_name, changed)
pass
if changed or enrichment_behavior.force_save:
new_entity.last_update = datetime.datetime.now()
new_entity.digest()
Store.set_entity(entity_type, new_entity)
msg = 'Stored in Store! (changed=%s, force_save=%s)' % (changed, enrichment_behavior.force_save)
# Redigest other entities
self.redigest(redigest_properties)
else:
msg = 'Not stored. No change detected'
self.logger.info(msg)
# Prepare information to send to webhook
if enrichment_behavior.webhook:
payload = {'status_message': msg, 'status_code': 200, 'ts': time.time(), 'aid': new_entity.aid}
r = AcureRateUtils.announce(enrichment_behavior.webhook, payload)
self.logger.info('Done merging enrichment result into entity. Changed = %s', changed)
def _enrich_entity(self,
entity_type,
enrichment_key,
enrichment_behavior,
enrichment_data=None,
enrichment_source=None):
"""
Enrich a person - either with provided data or external enrichment (or both)
:param enrichment_key: the search key to be used to retrieve the object
:param enrichment_behavior: object determining external enrichment, dates, force, new, etc.
;param enrichment_data: an EnrichmentData object or Array of objects including data rows to add
;param enrichment_source: an EnrichmentSource object specifying the source of the added data
:return: the person entity after the enrichment process
"""
status_code = EnrichmentException.ALL_OK
status_message = "Enrichment completed succesfully (behavior: %s)" % str(
enrichment_behavior)
# Validate parameters
if enrichment_data and not enrichment_source:
raise EnrichmentException(
"Cannot enrich with additional data without enrichment source.",
EnrichmentException.BAD_REQUEST)
# Decide which external providers are to be used (all, selective list or empty list)
providers = self._decide_providers(enrichment_behavior)
try:
updated_entities = []
changed = False
# Get person from the Store
# TODO: in case too many results are returned - they are in-memory - need to limit
entities = Store.get_entities(
entity_type,
enrichment_key,
single_result=False,
mongo_query=enrichment_behavior.mongo_query)
if len(entities) == 0:
if enrichment_behavior.create_new:
self.logger.info(
'Enriching on %s. Could not locate entities in %s collection, creating a new entity.',
enrichment_key, entity_type)
if entity_type == 'people':
entities = [AcureRatePerson()]
elif entity_type == 'company':
entities = [AcureRateCompany()]
# If no provider, add a Dummy engager, so the system digests and stores the data
if not providers:
providers = ['System']
elif 'System' not in providers:
providers.insert(0, 'System')
else:
msg = 'Attempting enrichment on key %s. Could not locate entities matching key (Behavior::create_new = False)' % enrichment_key
raise EnrichmentException(
msg, EnrichmentException.CONTACT_NOT_FOUND)
elif len(entities) > 1 and not enrichment_behavior.enrich_multiple:
msg = 'Enrichment data %s returns %d entities but enrich_multiple=False. Not enriching' % (
enrichment_key, len(entities))
raise EnrichmentException(
msg, EnrichmentException.MULTIPLE_CONTACTS)
# Go over all entities retrieved from store (per given key)
#with ClusterRpcProxy(EnrichmentServiceConfig.AMQP_CONFIG, timeout=None) as rpc:
rpc = None
if True:
for entity in entities:
# If new enriched data provided, merge it into received entity
if enrichment_data and len(enrichment_data) > 0:
enrichment_data.append(
EnrichmentData('last_run_time',
datetime.datetime.now(),
'override-no-change'))
# enrichment_data.append(EnrichmentData('data_source', enrichment_source.source_type, 'override'))
# enrichment_data.append(EnrichmentData('enrich_key', enrichment_source.source_key, 'override'))
changed |= entity.merge_data(
enrichment_source.source_type,
enrichment_source.source_key, enrichment_data)
#changed |= entity.merge_data('System', 'nokey', enrichment_data)
if changed or enrichment_behavior.digest:
changed = entity.digest()
# Initiate engagement manager to enrich via providers
if True:
EngagementManager().spawn_engagers_sequentially(
providers, entity_type, entity,
enrichment_behavior, changed)
else:
rpc.engagement_manager.spawn_engagers.call_async(
providers, entity_type, entity.to_json_string(),
enrichment_behavior.force,
enrichment_behavior.force_save)
except EnrichmentException as e:
self.logger.warning(e)
if enrichment_behavior.webhook:
r = AcureRateUtils.announce(
enrichment_behavior.webhook, {
'status_message': e.message,
'status_code': e.code,
'ts': time.time()
})
if r:
self.logger.info(
'Sent post request to webhook at %s. Content: %s. Code: %s',
enrichment_behavior.webhook, r.content, r.status_code)
except Exception as e:
msg = 'Failed to enrich %s entity. Key: %s. Reason: %s' % (
entity_type, enrichment_key, e)
self.logger.error(msg, exc_info=True)
if enrichment_behavior.webhook:
r = AcureRateUtils.announce(
enrichment_behavior.webhook, {
'status_message': msg,
'status_code': EnrichmentException.FATAL_ERROR,
'ts': time.time()
})
if r:
self.logger.info(
'Sent post request to webhook at %s. Content: %s. Code: %s',
enrichment_behavior.webhook, r.content, r.status_code)
return updated_entities