def test_indexed_doc(setup_es):
"""Test the ES data of an indexed companies house company."""
ch_company = CompaniesHouseCompanyFactory()
doc = ESCompaniesHouseCompany.es_document(ch_company)
elasticsearch.bulk(actions=(doc, ), chunk_size=1)
setup_es.indices.refresh()
indexed_ch_company = setup_es.get(
index=ESCompaniesHouseCompany.get_write_index(),
doc_type=CompaniesHouseCompanySearchApp.name,
id=ch_company.pk,
)
assert indexed_ch_company == {
'_index': ESCompaniesHouseCompany.get_target_index_name(),
'_type': CompaniesHouseCompanySearchApp.name,
'_id': str(ch_company.pk),
'_version': indexed_ch_company['_version'],
'found': True,
'_source': {
'id':
str(ch_company.pk),
'name':
ch_company.name,
'registered_address_1':
ch_company.registered_address_1,
'registered_address_2':
ch_company.registered_address_2,
'registered_address_town':
ch_company.registered_address_town,
'registered_address_county':
ch_company.registered_address_county,
'registered_address_postcode':
ch_company.registered_address_postcode,
'registered_address_country': {
'id': str(ch_company.registered_address_country.pk),
'name': ch_company.registered_address_country.name,
},
'company_number':
ch_company.company_number,
'company_category':
ch_company.company_category,
'company_status':
ch_company.company_status,
'sic_code_1':
ch_company.sic_code_1,
'sic_code_2':
ch_company.sic_code_2,
'sic_code_3':
ch_company.sic_code_3,
'sic_code_4':
ch_company.sic_code_4,
'uri':
ch_company.uri,
'incorporation_date':
format_date_or_datetime(ch_company.incorporation_date),
},
}
def test_bulk(es_bulk, mock_es_client):
"""Tests detailed company search."""
actions = []
chunk_size = 10
elasticsearch.bulk(actions=actions, chunk_size=chunk_size)
es_bulk.assert_called_with(
mock_es_client.return_value,
actions=actions,
chunk_size=chunk_size,
max_chunk_bytes=settings.ES_BULK_MAX_CHUNK_BYTES,
)
def test_indexed_doc(es):
"""Test the ES data of an Large investor profile."""
investor_company = CompanyFactory()
large_investor_profile = LargeCapitalInvestorProfileFactory(
investor_company=investor_company,
)
doc = ESLargeInvestorProfile.es_document(large_investor_profile)
elasticsearch.bulk(actions=(doc, ), chunk_size=1)
es.indices.refresh()
indexed_large_investor_profile = es.get(
index=ESLargeInvestorProfile.get_write_index(),
doc_type=LargeInvestorProfileSearchApp.name,
id=large_investor_profile.pk,
)
assert indexed_large_investor_profile['_id'] == str(large_investor_profile.pk)
assert indexed_large_investor_profile['_source'] == {
'_document_type': LargeInvestorProfileSearchApp.name,
'id': str(large_investor_profile.pk),
'asset_classes_of_interest': [],
'country_of_origin': {
'id': str(large_investor_profile.country_of_origin.pk),
'name': str(large_investor_profile.country_of_origin.name),
},
'investor_company': {
'id': str(investor_company.pk),
'name': str(investor_company.name),
'trading_names': investor_company.trading_names,
},
'created_by': None,
'investor_type': None,
'required_checks_conducted': None,
'deal_ticket_sizes': [],
'investment_types': [],
'minimum_return_rate': None,
'time_horizons': [],
'restrictions': [],
'construction_risks': [],
'minimum_equity_percentage': None,
'desired_deal_roles': [],
'uk_region_locations': [],
'other_countries_being_considered': [],
'investable_capital': None,
'investor_description': '',
'created_on': '2019-01-01T00:00:00+00:00',
'notes_on_locations': '',
'global_assets_under_management': None,
'modified_on': '2019-01-01T00:00:00+00:00',
}
def sync_objects(es_model, model_objects, read_indices, write_index, post_batch_callback=None):
"""Syncs an iterable of model instances to Elasticsearch."""
actions = list(
es_model.db_objects_to_es_documents(model_objects, index=write_index),
)
num_actions = len(actions)
bulk(
actions=actions,
chunk_size=num_actions,
request_timeout=BULK_INDEX_TIMEOUT_SECS,
)
if post_batch_callback:
post_batch_callback(read_indices, write_index, actions)
return num_actions
def delete_documents(index, es_docs):
"""
Deletes `es_docs` from `index`.
:raises DataHubException: in case of non 404 errors
"""
delete_actions = (_create_delete_action(index, es_doc['_type'],
es_doc['_id'])
for es_doc in es_docs)
_, errors = bulk(
actions=delete_actions,
chunk_size=BULK_CHUNK_SIZE,
request_timeout=BULK_DELETION_TIMEOUT_SECS,
raise_on_error=False,
)
non_404_errors = [
error for error in errors if error['delete']['status'] != 404
]
if non_404_errors:
raise DataHubException(
f'One or more errors during an Elasticsearch bulk deletion operation: '
f'{non_404_errors!r}', )
def test_indexed_doc(Factory, setup_es):
"""Test the ES data of an indexed order."""
order = Factory()
invoice = order.invoice
doc = ESOrder.es_document(order)
elasticsearch.bulk(actions=(doc, ), chunk_size=1)
setup_es.indices.refresh()
indexed_order = setup_es.get(
index=OrderSearchApp.es_model.get_write_index(),
doc_type=OrderSearchApp.name,
id=order.pk,
)
assert indexed_order == {
'_index': OrderSearchApp.es_model.get_target_index_name(),
'_type': OrderSearchApp.name,
'_id': str(order.pk),
'_version': indexed_order['_version'],
'found': True,
'_source': {
'id':
str(order.pk),
'created_by': {
'id': str(order.created_by.pk),
'first_name': order.created_by.first_name,
'last_name': order.created_by.last_name,
'name': order.created_by.name,
'dit_team': {
'id': str(order.created_by.dit_team.id),
'name': order.created_by.dit_team.name,
},
},
'company': {
'id': str(order.company.pk),
'name': order.company.name,
'trading_name': order.company.alias,
},
'contact': {
'id': str(order.contact.pk),
'first_name': order.contact.first_name,
'last_name': order.contact.last_name,
'name': order.contact.name,
},
'primary_market': {
'id': str(order.primary_market.pk),
'name': order.primary_market.name,
},
'sector': {
'id':
str(order.sector.pk),
'name':
order.sector.name,
'ancestors': [{
'id': str(ancestor.pk),
} for ancestor in order.sector.get_ancestors()],
},
'uk_region': {
'id': str(order.uk_region.pk),
'name': order.uk_region.name,
},
'service_types': [{
'id': str(service_type.pk),
'name': service_type.name,
} for service_type in order.service_types.all()],
'created_on':
order.created_on.isoformat(),
'modified_on':
order.modified_on.isoformat(),
'reference':
order.reference,
'status':
order.status,
'description':
order.description,
'contacts_not_to_approach':
order.contacts_not_to_approach,
'further_info':
order.further_info,
'existing_agents':
order.existing_agents,
'delivery_date':
order.delivery_date.isoformat(),
'contact_email':
order.contact_email,
'contact_phone':
order.contact_phone,
'subscribers': [{
'id': str(subscriber.adviser.pk),
'name': subscriber.adviser.name,
'first_name': subscriber.adviser.first_name,
'last_name': subscriber.adviser.last_name,
} for subscriber in order.subscribers.all()],
'assignees': [{
'id': str(assignee.adviser.pk),
'name': assignee.adviser.name,
'first_name': assignee.adviser.first_name,
'last_name': assignee.adviser.last_name,
'dit_team': {
'id': str(assignee.adviser.dit_team.id),
'name': assignee.adviser.dit_team.name,
},
} for assignee in order.assignees.all()],
'po_number':
order.po_number,
'discount_value':
order.discount_value,
'vat_status':
order.vat_status,
'vat_number':
order.vat_number,
'vat_verified':
order.vat_verified,
'net_cost':
order.net_cost,
'payment_due_date':
None if not invoice else invoice.payment_due_date.isoformat(),
'subtotal_cost':
order.subtotal_cost,
'vat_cost':
order.vat_cost,
'total_cost':
order.total_cost,
'billing_company_name':
order.billing_company_name,
'billing_contact_name':
order.billing_contact_name,
'billing_email':
order.billing_email,
'billing_phone':
order.billing_phone,
'billing_address_1':
order.billing_address_1,
'billing_address_2':
order.billing_address_2,
'billing_address_town':
order.billing_address_town,
'billing_address_county':
order.billing_address_county,
'billing_address_postcode':
order.billing_address_postcode,
'billing_address_country': {
'id': str(order.billing_address_country.pk),
'name': order.billing_address_country.name,
},
'paid_on':
order.paid_on.isoformat() if order.paid_on else None,
'completed_by': {
'id': str(order.completed_by.pk),
'first_name': order.completed_by.first_name,
'last_name': order.completed_by.last_name,
'name': order.completed_by.name,
} if order.completed_by else None,
'completed_on':
order.completed_on.isoformat() if order.completed_on else None,
'cancelled_by': {
'id': str(order.cancelled_by.pk),
'first_name': order.cancelled_by.first_name,
'last_name': order.cancelled_by.last_name,
'name': order.cancelled_by.name,
} if order.cancelled_by else None,
'cancelled_on':
order.cancelled_on.isoformat() if order.cancelled_on else None,
'cancellation_reason': {
'id': str(order.cancellation_reason.pk),
'name': order.cancellation_reason.name,
} if order.cancellation_reason else None,
},
}
