def get_records_from_values(entity_name, values=None): """ Fetch entity data based for the specified values in that entity Args: entity_name (str): Name of the entity for which records are to be fetched values (list, optional): List of str values for which the data is to be fetched Returns: dict: dictionary mapping the entity_value to a dictionary Sample: { 'entity_value': { 'en': { '_id': 'Random ES ID', 'value': ['Variant 1', 'Variant 2'] }, 'hi': { '_id': 'Random ES ID', 'value': ['Variant 1', 'Variant 2'] } } """ datastore_obj = DataStore() results = datastore_obj.get_entity_data(entity_name=entity_name, values=values) merged_records = {} for result in results: merged_records.setdefault(result['_source']['value'], {}) merged_records[result['_source']['value']][ result['_source']['language_script']] = { '_id': result['_id'], 'value': result['_source']['variants'], } return merged_records
def update_dictionary(request): """ This function is used to update the dictionary entities. Args: request (HttpResponse): HTTP response from url Returns: HttpResponse : HttpResponse with appropriate status and error message. """ response = {"success": False, "error": "", "result": []} try: external_api_data = json.loads(request.POST.get(EXTERNAL_API_DATA)) entity_name = external_api_data.get(ENTITY_NAME) entity_data = external_api_data.get(ENTITY_DATA) language_script = external_api_data.get(LANGUAGE_SCRIPT) datastore_obj = DataStore() datastore_obj.update_entity_data(entity_name=entity_name, entity_data=entity_data, language_script=language_script) response['success'] = True except (DataStoreSettingsImproperlyConfiguredException, EngineNotImplementedException, EngineConnectionException, FetchIndexForAliasException) as error_message: response['error'] = str(error_message) ner_logger.exception('Error: %s' % error_message) return HttpResponse(json.dumps(response), content_type='application/json', status=500) except BaseException as e: response['error'] = str(e) ner_logger.exception('Error: %s' % e) return HttpResponse(json.dumps(response), content_type='application/json', status=500) return HttpResponse(json.dumps(response), content_type='application/json', status=200)
def update_crf_training_data(request): """ This function is used to update the training data Args: request (HttpRequest): HTTP response from url Returns: HttpResponse : HttpResponse with appropriate status and error message. Example for data present in Post request body key: "external_api_data" value: {"sentence_list":["hello pratik","hello hardik"], "entity_list":[["pratik"], ["hardik"]], "entity_name":"training_try3", "language_script": "en"} """ response = {"success": False, "error": "", "result": []} try: external_api_data = json.loads(request.POST.get(EXTERNAL_API_DATA)) sentences = external_api_data.get(SENTENCES) entity_name = external_api_data.get(ENTITY_NAME) DataStore().update_entity_crf_data(entity_name=entity_name, sentences=sentences) response['success'] = True except (DataStoreSettingsImproperlyConfiguredException, EngineNotImplementedException, EngineConnectionException, FetchIndexForAliasException) as error_message: response['error'] = str(error_message) ner_logger.exception('Error: %s' % error_message) return HttpResponse(json.dumps(response), content_type='application/json', status=500) except BaseException as e: response['error'] = str(e) ner_logger.exception('Error: %s' % e) return HttpResponse(json.dumps(response), content_type='application/json', status=500) return HttpResponse(json.dumps(response), content_type='application/json', status=200)
def transfer_entities(request): """ This method is used to transfer entities from the source to destination. Args: request (HttpResponse): HTTP response from url Returns: HttpResponse : HttpResponse with appropriate status and error message. """ response = {"success": False, "error": "", "result": []} try: external_api_data = json.loads(request.POST.get(EXTERNAL_API_DATA)) entity_list = external_api_data.get(ENTITY_LIST) datastore_object = DataStore() datastore_object.transfer_entities_elastic_search(entity_list=entity_list) response['success'] = True except (IndexNotFoundException, InvalidESURLException, SourceDestinationSimilarException, InternalBackupException, AliasNotFoundException, PointIndexToAliasException, FetchIndexForAliasException, DeleteIndexFromAliasException, AliasForTransferException, IndexForTransferException, NonESEngineTransferException) as error_message: response['error'] = str(error_message) ner_logger.exception('Error: %s' % error_message) return HttpResponse(json.dumps(response), content_type='application/json', status=500) except BaseException as e: response['error'] = str(e) ner_logger.exception('Error: %s' % e) return HttpResponse(json.dumps(response), content_type='application/json', status=500) return HttpResponse(json.dumps(response), content_type='application/json', status=200)
def train_model_from_es_data(self): """ This method is used to train the crf model by first extracting training data from ES for the entity and training the crf model for the same. """ datastore_object = DataStore() ner_logger.debug('Fetch of data from ES for ENTITY: %s started' % self.entity_name) result = datastore_object.get_crf_data_for_entity_name( entity_name=self.entity_name) sentence_list = result.get(SENTENCE_LIST, []) entity_list = result.get(ENTITY_LIST, []) if not sentence_list: raise ESCrfTrainingTextListNotFoundException() if not entity_list: raise ESCrfTrainingEntityListNotFoundException() ner_logger.debug('Fetch of data from ES for ENTITY: %s completed' % self.entity_name) ner_logger.debug('Length of text_list %s' % str(len(sentence_list))) model_path = self.train_crf_model_from_list( entity_list=entity_list, sentence_list=sentence_list) return model_path
def get_crf_training_data(request): """ This function is used obtain the training data given the entity_name. Args: request (HttpResponse): HTTP response from url Returns: HttpResponse : With data consisting of a dictionary consisting of sentence_list and entity_list Examples: get request params key: "entity_name" value: "city" """ response = {"success": False, "error": "", "result": []} try: entity_name = request.GET.get(ENTITY_NAME) datastore_obj = DataStore() result = datastore_obj.get_crf_data_for_entity_name(entity_name=entity_name) response['result'] = result response['success'] = True except (DataStoreSettingsImproperlyConfiguredException, EngineNotImplementedException, EngineConnectionException, FetchIndexForAliasException) as error_message: response['error'] = str(error_message) ner_logger.exception('Error: %s' % error_message) return HttpResponse(json.dumps(response), content_type='application/json', status=500) except BaseException as e: response['error'] = str(e) ner_logger.exception('Error: %s' % e) return HttpResponse(json.dumps(response), content_type='application/json', status=500) return HttpResponse(json.dumps(response), content_type='application/json', status=200)
def entity_supported_languages(entity_name): """ Fetch list of supported languages for the specific entity Args: entity_name (str): Name of the entity for which unique values are to be fetched Returns: list: List of language_codes """ datastore_obj = DataStore() return datastore_obj.get_entity_supported_languages( entity_name=entity_name)
def delete_records_by_values(entity_name, values): """ Delete entity data based for the specified values in that entity Args: entity_name (str): Name of the entity for which records are to be fetched values (list): List of str values for which the data is to be fetched Returns: None """ datastore_obj = DataStore() datastore_obj.delete_entity_data_by_values(entity_name=entity_name, values=values)
def entity_update_languages(entity_name, new_language_list): """ Updates the language support list of the entity by creating dummy records. Currently does not support removal of a language. It creates empty variant records for all the unique values present in this entity. Args: entity_name (str): Name of the entity for which unique values are to be fetched new_language_list (list): List of language codes for the new entity Returns: bool: Success flag if the update Raises: APIHandlerException (Exception): for any validation errors """ old_language_list = entity_supported_languages(entity_name) languages_added = set(new_language_list) - set(old_language_list) languages_removed = set(old_language_list) - set(new_language_list) if languages_removed: # raise exception as it is not currently supported raise APIHandlerException( 'Removing languages is not currently supported.') if not languages_added: # no change in language list. raise error raise APIHandlerException( 'No new languages provided. Nothing changed.') # fetch all words # TODO: If possible add records in single ES query instead of # two (get_entity_unique_values + db.add_entity_data) values = get_entity_unique_values(entity_name=entity_name) if not values: raise APIHandlerException( 'This entity does not have any records. Please verify the entity name' ) records_to_create = [] for language_script in languages_added: # create records for all words for value in values: if value and language_script: records_to_create.append({ 'value': value, 'language_script': language_script, 'variants': [] }) datastore_obj = DataStore() datastore_obj.add_entity_data(entity_name, records_to_create) return True
def get_entity_word_variants(request): """ This function is used obtain the entity dictionary given the dictionary name. Args: request (HttpResponse): HTTP response from url Returns: HttpResponse : With data consisting of a list of value variants. """ response = {"success": False, "error": "", "result": []} try: entity_name = request.GET.get(ENTITY_NAME) datastore_obj = DataStore() result = datastore_obj.get_entity_dictionary(entity_name=entity_name) structured_result = [] # The list around result.keys() is to make it compatible to python3 key_list = list(result.keys()) key_list.sort() for value in key_list: structured_result.append({ 'value': value, 'variants': result[value] }) result = structured_result response['result'] = result response['success'] = True except (DataStoreSettingsImproperlyConfiguredException, EngineNotImplementedException, EngineConnectionException, FetchIndexForAliasException) as error_message: response['error'] = str(error_message) ner_logger.exception('Error: %s' % error_message) return HttpResponse(json.dumps(response), content_type='application/json', status=500) except BaseException as e: response['error'] = str(e) ner_logger.exception('Error: %s' % e) return HttpResponse(json.dumps(response), content_type='application/json', status=500) return HttpResponse(json.dumps(response), content_type='application/json', status=200)
def update_entity_records(entity_name, data): """ Update dictionary data with the edited and deleted records Args: entity_name (str): Name of the entity for which records are to be fetched data (dict): Dictionary of edited, deleted data. If replace flag is true, then all existing data is deleted before adding the records Returns: None """ # Delete some records first records_to_delete = data.get('deleted', []) records_to_create = data.get('edited', []) replace_data = data.get('replace') if replace_data: # TODO: Delete everything for the `entity_name` without having to fetch values first! # https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docs-delete-by-query.html values_to_delete = get_entity_unique_values(entity_name) else: values_to_delete = [record['word'] for record in records_to_delete] values_to_delete.extend( [record['word'] for record in records_to_create]) value_variants_to_create = [] for record in records_to_create: for language_script, variants in record.get('variants', {}).items(): if record['word'] and language_script: value_variants_to_create.append({ 'value': record['word'], 'language_script': language_script, 'variants': variants.get('value', []) }) # delete words delete_records_by_values(entity_name=entity_name, values=values_to_delete) datastore_obj = DataStore() datastore_obj.add_entity_data(entity_name, value_variants_to_create)
def update_entity_records(entity_name, data): """ Update dictionary data with the edited and deleted records Args: entity_name (str): Name of the entity for which records are to be fetched data (dict): Dictionary of edited, deleted data. If replace flag is true, then all existing data is deleted before adding the records Returns: None """ # Delete some records first records_to_delete = data.get('deleted', []) records_to_create = data.get('edited', []) replace_data = data.get('replace') if replace_data: values_to_delete = get_entity_unique_values(entity_name) else: values_to_delete = [record['word'] for record in records_to_delete] values_to_delete.extend( [record['word'] for record in records_to_create]) value_variants_to_create = [] for record in records_to_create: for language_script, variants in record.get('variants', {}).items(): if record['word'] and language_script: value_variants_to_create.append({ 'value': record['word'], 'language_script': language_script, 'variants': variants.get('value', []) }) # delete words delete_records_by_values(entity_name=entity_name, values=values_to_delete) datastore_obj = DataStore() datastore_obj.add_entity_data(entity_name, value_variants_to_create)
def get_entity_unique_values(entity_name, empty_variants_only=False, value_search_term=None, variant_search_term=None): """ Get a list of unique values belonging to this entity Args: entity_name (str): Name of the entity for which unique values are to be fetched empty_variants_only (bool, optional): Flag to search for values with empty variants only value_search_term (str, optional): Search term to filter values from this entity data variant_search_term (str, optional): Search term to filter out variants from the entity data Returns: list: List of strings which are unique values in the entity """ datastore_obj = DataStore() return datastore_obj.get_entity_unique_values( entity_name=entity_name, value_search_term=value_search_term, variant_search_term=variant_search_term, empty_variants_only=empty_variants_only)