def test_indexed_doc(setup_es): """Test the ES data of an indexed companies house company.""" ch_company = CompaniesHouseCompanyFactory() doc = ESCompaniesHouseCompany.es_document(ch_company) elasticsearch.bulk(actions=(doc, ), chunk_size=1) setup_es.indices.refresh() indexed_ch_company = setup_es.get( index=ESCompaniesHouseCompany.get_write_index(), doc_type=CompaniesHouseCompanySearchApp.name, id=ch_company.pk, ) assert indexed_ch_company == { '_index': ESCompaniesHouseCompany.get_target_index_name(), '_type': CompaniesHouseCompanySearchApp.name, '_id': str(ch_company.pk), '_version': indexed_ch_company['_version'], 'found': True, '_source': { 'id': str(ch_company.pk), 'name': ch_company.name, 'registered_address_1': ch_company.registered_address_1, 'registered_address_2': ch_company.registered_address_2, 'registered_address_town': ch_company.registered_address_town, 'registered_address_county': ch_company.registered_address_county, 'registered_address_postcode': ch_company.registered_address_postcode, 'registered_address_country': { 'id': str(ch_company.registered_address_country.pk), 'name': ch_company.registered_address_country.name, }, 'company_number': ch_company.company_number, 'company_category': ch_company.company_category, 'company_status': ch_company.company_status, 'sic_code_1': ch_company.sic_code_1, 'sic_code_2': ch_company.sic_code_2, 'sic_code_3': ch_company.sic_code_3, 'sic_code_4': ch_company.sic_code_4, 'uri': ch_company.uri, 'incorporation_date': format_date_or_datetime(ch_company.incorporation_date), }, }
def test_bulk(es_bulk, mock_es_client): """Tests detailed company search.""" actions = [] chunk_size = 10 elasticsearch.bulk(actions=actions, chunk_size=chunk_size) es_bulk.assert_called_with( mock_es_client.return_value, actions=actions, chunk_size=chunk_size, max_chunk_bytes=settings.ES_BULK_MAX_CHUNK_BYTES, )
def test_indexed_doc(es): """Test the ES data of an Large investor profile.""" investor_company = CompanyFactory() large_investor_profile = LargeCapitalInvestorProfileFactory( investor_company=investor_company, ) doc = ESLargeInvestorProfile.es_document(large_investor_profile) elasticsearch.bulk(actions=(doc, ), chunk_size=1) es.indices.refresh() indexed_large_investor_profile = es.get( index=ESLargeInvestorProfile.get_write_index(), doc_type=LargeInvestorProfileSearchApp.name, id=large_investor_profile.pk, ) assert indexed_large_investor_profile['_id'] == str(large_investor_profile.pk) assert indexed_large_investor_profile['_source'] == { '_document_type': LargeInvestorProfileSearchApp.name, 'id': str(large_investor_profile.pk), 'asset_classes_of_interest': [], 'country_of_origin': { 'id': str(large_investor_profile.country_of_origin.pk), 'name': str(large_investor_profile.country_of_origin.name), }, 'investor_company': { 'id': str(investor_company.pk), 'name': str(investor_company.name), 'trading_names': investor_company.trading_names, }, 'created_by': None, 'investor_type': None, 'required_checks_conducted': None, 'deal_ticket_sizes': [], 'investment_types': [], 'minimum_return_rate': None, 'time_horizons': [], 'restrictions': [], 'construction_risks': [], 'minimum_equity_percentage': None, 'desired_deal_roles': [], 'uk_region_locations': [], 'other_countries_being_considered': [], 'investable_capital': None, 'investor_description': '', 'created_on': '2019-01-01T00:00:00+00:00', 'notes_on_locations': '', 'global_assets_under_management': None, 'modified_on': '2019-01-01T00:00:00+00:00', }
def sync_objects(es_model, model_objects, read_indices, write_index, post_batch_callback=None): """Syncs an iterable of model instances to Elasticsearch.""" actions = list( es_model.db_objects_to_es_documents(model_objects, index=write_index), ) num_actions = len(actions) bulk( actions=actions, chunk_size=num_actions, request_timeout=BULK_INDEX_TIMEOUT_SECS, ) if post_batch_callback: post_batch_callback(read_indices, write_index, actions) return num_actions
def delete_documents(index, es_docs): """ Deletes `es_docs` from `index`. :raises DataHubException: in case of non 404 errors """ delete_actions = (_create_delete_action(index, es_doc['_type'], es_doc['_id']) for es_doc in es_docs) _, errors = bulk( actions=delete_actions, chunk_size=BULK_CHUNK_SIZE, request_timeout=BULK_DELETION_TIMEOUT_SECS, raise_on_error=False, ) non_404_errors = [ error for error in errors if error['delete']['status'] != 404 ] if non_404_errors: raise DataHubException( f'One or more errors during an Elasticsearch bulk deletion operation: ' f'{non_404_errors!r}', )
def test_indexed_doc(Factory, setup_es): """Test the ES data of an indexed order.""" order = Factory() invoice = order.invoice doc = ESOrder.es_document(order) elasticsearch.bulk(actions=(doc, ), chunk_size=1) setup_es.indices.refresh() indexed_order = setup_es.get( index=OrderSearchApp.es_model.get_write_index(), doc_type=OrderSearchApp.name, id=order.pk, ) assert indexed_order == { '_index': OrderSearchApp.es_model.get_target_index_name(), '_type': OrderSearchApp.name, '_id': str(order.pk), '_version': indexed_order['_version'], 'found': True, '_source': { 'id': str(order.pk), 'created_by': { 'id': str(order.created_by.pk), 'first_name': order.created_by.first_name, 'last_name': order.created_by.last_name, 'name': order.created_by.name, 'dit_team': { 'id': str(order.created_by.dit_team.id), 'name': order.created_by.dit_team.name, }, }, 'company': { 'id': str(order.company.pk), 'name': order.company.name, 'trading_name': order.company.alias, }, 'contact': { 'id': str(order.contact.pk), 'first_name': order.contact.first_name, 'last_name': order.contact.last_name, 'name': order.contact.name, }, 'primary_market': { 'id': str(order.primary_market.pk), 'name': order.primary_market.name, }, 'sector': { 'id': str(order.sector.pk), 'name': order.sector.name, 'ancestors': [{ 'id': str(ancestor.pk), } for ancestor in order.sector.get_ancestors()], }, 'uk_region': { 'id': str(order.uk_region.pk), 'name': order.uk_region.name, }, 'service_types': [{ 'id': str(service_type.pk), 'name': service_type.name, } for service_type in order.service_types.all()], 'created_on': order.created_on.isoformat(), 'modified_on': order.modified_on.isoformat(), 'reference': order.reference, 'status': order.status, 'description': order.description, 'contacts_not_to_approach': order.contacts_not_to_approach, 'further_info': order.further_info, 'existing_agents': order.existing_agents, 'delivery_date': order.delivery_date.isoformat(), 'contact_email': order.contact_email, 'contact_phone': order.contact_phone, 'subscribers': [{ 'id': str(subscriber.adviser.pk), 'name': subscriber.adviser.name, 'first_name': subscriber.adviser.first_name, 'last_name': subscriber.adviser.last_name, } for subscriber in order.subscribers.all()], 'assignees': [{ 'id': str(assignee.adviser.pk), 'name': assignee.adviser.name, 'first_name': assignee.adviser.first_name, 'last_name': assignee.adviser.last_name, 'dit_team': { 'id': str(assignee.adviser.dit_team.id), 'name': assignee.adviser.dit_team.name, }, } for assignee in order.assignees.all()], 'po_number': order.po_number, 'discount_value': order.discount_value, 'vat_status': order.vat_status, 'vat_number': order.vat_number, 'vat_verified': order.vat_verified, 'net_cost': order.net_cost, 'payment_due_date': None if not invoice else invoice.payment_due_date.isoformat(), 'subtotal_cost': order.subtotal_cost, 'vat_cost': order.vat_cost, 'total_cost': order.total_cost, 'billing_company_name': order.billing_company_name, 'billing_contact_name': order.billing_contact_name, 'billing_email': order.billing_email, 'billing_phone': order.billing_phone, 'billing_address_1': order.billing_address_1, 'billing_address_2': order.billing_address_2, 'billing_address_town': order.billing_address_town, 'billing_address_county': order.billing_address_county, 'billing_address_postcode': order.billing_address_postcode, 'billing_address_country': { 'id': str(order.billing_address_country.pk), 'name': order.billing_address_country.name, }, 'paid_on': order.paid_on.isoformat() if order.paid_on else None, 'completed_by': { 'id': str(order.completed_by.pk), 'first_name': order.completed_by.first_name, 'last_name': order.completed_by.last_name, 'name': order.completed_by.name, } if order.completed_by else None, 'completed_on': order.completed_on.isoformat() if order.completed_on else None, 'cancelled_by': { 'id': str(order.cancelled_by.pk), 'first_name': order.cancelled_by.first_name, 'last_name': order.cancelled_by.last_name, 'name': order.cancelled_by.name, } if order.cancelled_by else None, 'cancelled_on': order.cancelled_on.isoformat() if order.cancelled_on else None, 'cancellation_reason': { 'id': str(order.cancellation_reason.pk), 'name': order.cancellation_reason.name, } if order.cancellation_reason else None, }, }