def check_company(self, name, domain):
if domain == 'angel.co' or domain == 'itunes.apple.com':
return
# Check if appears in database
#q = {'deduced.aliases': name}
q = {'deduced.domain': domain}
cursor = Store.get_entities('company', q, mongo_query=True)
for entity in cursor:
self.found += 1
self.logger.info('Found %s in our DB (so far found %d)', name,
self.found)
pass
def redigest(self, redigest_properties):
if True:
return
self.logger.info('Started redigest according to these properties: %s', redigest_properties)
# TODO: fix the below to work also with companies - be agnostic to the entity type!
# Run a query to get all those entities who have the value in the field "unrecognized_<prop>":
for k, v in redigest_properties.items():
if k == 'name': # TODO: temp code. Need to align Mongo properties and all is well...
k = 'companies'
q = {'deduced.unrecognized_%s' % k: v}
cursor = Store.get_entities('people', q, mongo_query=True)
for entity in cursor:
entity.digest()
Store.set_entity('people', entity)
self.logger.info('Done redigesting entities of property %s', k)
self.logger.info('Done redigesting all entities.')
pass
def _enrich_entity(self,
entity_type,
enrichment_key,
enrichment_behavior,
enrichment_data=None,
enrichment_source=None):
"""
Enrich a person - either with provided data or external enrichment (or both)
:param enrichment_key: the search key to be used to retrieve the object
:param enrichment_behavior: object determining external enrichment, dates, force, new, etc.
;param enrichment_data: an EnrichmentData object or Array of objects including data rows to add
;param enrichment_source: an EnrichmentSource object specifying the source of the added data
:return: the person entity after the enrichment process
"""
status_code = EnrichmentException.ALL_OK
status_message = "Enrichment completed succesfully (behavior: %s)" % str(
enrichment_behavior)
# Validate parameters
if enrichment_data and not enrichment_source:
raise EnrichmentException(
"Cannot enrich with additional data without enrichment source.",
EnrichmentException.BAD_REQUEST)
# Decide which external providers are to be used (all, selective list or empty list)
providers = self._decide_providers(enrichment_behavior)
try:
updated_entities = []
changed = False
# Get person from the Store
# TODO: in case too many results are returned - they are in-memory - need to limit
entities = Store.get_entities(
entity_type,
enrichment_key,
single_result=False,
mongo_query=enrichment_behavior.mongo_query)
if len(entities) == 0:
if enrichment_behavior.create_new:
self.logger.info(
'Enriching on %s. Could not locate entities in %s collection, creating a new entity.',
enrichment_key, entity_type)
if entity_type == 'people':
entities = [AcureRatePerson()]
elif entity_type == 'company':
entities = [AcureRateCompany()]
# If no provider, add a Dummy engager, so the system digests and stores the data
if not providers:
providers = ['System']
elif 'System' not in providers:
providers.insert(0, 'System')
else:
msg = 'Attempting enrichment on key %s. Could not locate entities matching key (Behavior::create_new = False)' % enrichment_key
raise EnrichmentException(
msg, EnrichmentException.CONTACT_NOT_FOUND)
elif len(entities) > 1 and not enrichment_behavior.enrich_multiple:
msg = 'Enrichment data %s returns %d entities but enrich_multiple=False. Not enriching' % (
enrichment_key, len(entities))
raise EnrichmentException(
msg, EnrichmentException.MULTIPLE_CONTACTS)
# Go over all entities retrieved from store (per given key)
#with ClusterRpcProxy(EnrichmentServiceConfig.AMQP_CONFIG, timeout=None) as rpc:
rpc = None
if True:
for entity in entities:
# If new enriched data provided, merge it into received entity
if enrichment_data and len(enrichment_data) > 0:
enrichment_data.append(
EnrichmentData('last_run_time',
datetime.datetime.now(),
'override-no-change'))
# enrichment_data.append(EnrichmentData('data_source', enrichment_source.source_type, 'override'))
# enrichment_data.append(EnrichmentData('enrich_key', enrichment_source.source_key, 'override'))
changed |= entity.merge_data(
enrichment_source.source_type,
enrichment_source.source_key, enrichment_data)
#changed |= entity.merge_data('System', 'nokey', enrichment_data)
if changed or enrichment_behavior.digest:
changed = entity.digest()
# Initiate engagement manager to enrich via providers
if True:
EngagementManager().spawn_engagers_sequentially(
providers, entity_type, entity,
enrichment_behavior, changed)
else:
rpc.engagement_manager.spawn_engagers.call_async(
providers, entity_type, entity.to_json_string(),
enrichment_behavior.force,
enrichment_behavior.force_save)
except EnrichmentException as e:
self.logger.warning(e)
if enrichment_behavior.webhook:
r = AcureRateUtils.announce(
enrichment_behavior.webhook, {
'status_message': e.message,
'status_code': e.code,
'ts': time.time()
})
if r:
self.logger.info(
'Sent post request to webhook at %s. Content: %s. Code: %s',
enrichment_behavior.webhook, r.content, r.status_code)
except Exception as e:
msg = 'Failed to enrich %s entity. Key: %s. Reason: %s' % (
entity_type, enrichment_key, e)
self.logger.error(msg, exc_info=True)
if enrichment_behavior.webhook:
r = AcureRateUtils.announce(
enrichment_behavior.webhook, {
'status_message': msg,
'status_code': EnrichmentException.FATAL_ERROR,
'ts': time.time()
})
if r:
self.logger.info(
'Sent post request to webhook at %s. Content: %s. Code: %s',
enrichment_behavior.webhook, r.content, r.status_code)
return updated_entities