def init_analyzers(self, index_config):
print("init_analyzers")
print(index_config)
print(get_index_name(index_config))
if (self.es.indices.exists(get_index_name(index_config))):
self.es.indices.delete(index=get_index_name(index_config))
self.es.indices.create(index=get_index_name(index_config))
def search_columns_data(self, index_config, source_names):
if source_names:
result = list(
scan(self.es,
index=get_index_name(index_config),
doc_type='service',
query={
"query": {
"constant_score": {
"filter": {
"terms": {
"source": source_names
}
}
}
}
}))
else:
result = list(
scan(self.es,
index=get_index_name(index_config),
doc_type='service',
query={"query": {
"match_all": ""
}}))
return result
def index_column(self, column, source_name, index_config):
logging.info("Indexing column " + str(column))
body = column.to_json()
body['source'] = source_name
self.es.index(index=get_index_name(index_config),
doc_type=source_name,
body=body)
def search_columns_data(self, index_config, source_names):
result = list(
scan(self.es,
index=get_index_name(index_config),
doc_type=','.join(source_names),
query={"query": {
"match_all": {}
}}))
return result
def index_column(self, column, source_name, index_config):
body = column.to_json()
body['source'] = source_name
try:
self.es.index(index=get_index_name(index_config),
doc_type="service",
body=body)
return True
except RequestError:
print("Error")
return False
def search_similar_text_data(self, index_config, value_text, source_names):
try:
text = value_text
if source_names:
result = self.es.search(index=get_index_name(index_config),
doc_type='service',
body={
"query": {
"bool": {
"must": {
"match": {
"textual": text
}
},
"filter": {
"terms": {
"source":
source_names
}
}
}
}
},
size=10)
else:
result = self.es.search(
index=get_index_name(index_config),
doc_type='service',
body={"query": {
"match": {
"textual": text
}
}},
size=10)
except Exception:
result = {"hits": {"hits": []}}
logging.exception("Exception when querying elasticsearch")
return result
def search_similar_text_data(self, index_config, value_text, source_names):
try:
text = value_text
if source_names:
result = self.es.search(
index=get_index_name(index_config),
doc_type='service',
body={
"query": {
"bool": {
"must": {
"match": {
"textual": text
}
},
"filter": {
"terms": {
"source": source_names
}
}
}
}
},
size=10)
else:
result = self.es.search(
index=get_index_name(index_config),
doc_type='service',
body={"query": {
"match": {
"textual": text
}
}},
size=10)
except Exception:
result = {"hits": {"hits": []}}
logging.exception("Exception when querying elasticsearch")
return result
def delete_column(self, attr_name, source_name, index_config):
bulk_deletes = []
for result in scan(self.es, query={
"query": {
"match": {
"name": attr_name,
}
}
}, index=get_index_name(index_config), doc_type="service", _source=False,
track_scores=False, scroll='5m'):
result['_op_type'] = 'delete'
bulk_deletes.append(result)
bulk(self.es, bulk_deletes)
def search_columns_data(self, index_config, source_names):
if source_names:
result = list(
scan(
self.es,
index=get_index_name(index_config),
doc_type='service',
query={"query": {
"constant_score": {
"filter": {
"terms": {
"source": source_names
}
}
}
}}))
else:
result = list(
scan(
self.es, index=get_index_name(index_config), doc_type='service', query={"query": {
"match_all": ""
}}))
return result
def delete_column(self, attr_name, source_name, index_config):
bulk_deletes = []
for result in scan(self.es,
query={"query": {
"match": {
"name": attr_name,
}
}},
index=get_index_name(index_config),
doc_type="service",
_source=False,
track_scores=False,
scroll='5m'):
result['_op_type'] = 'delete'
bulk_deletes.append(result)
bulk(self.es, bulk_deletes)
def search_similar_text_data(self, index_config, value_text, source_names):
try:
text = value_text
result = self.es.search(
index=get_index_name(index_config),
doc_type=','.join(source_names),
body={"query": {
"match": {
"textual": text,
}
}},
size=10)
except Exception as e:
logging.warning("Search similar text data not possible")
result = {"hits": {"hits": []}}
return result
def index_source(self, source, index_config):
self.es.indices.put_mapping(index=get_index_name(index_config),
doc_type="service",
body={
"service": {
"properties": {
"source": {
"type": "string",
"index": "not_analyzed"
}
}
}
})
for column in source.column_map.values():
if column.semantic_type:
if len(column.value_list) > 0:
self.index_column(column, source.index_name, index_config)
else:
logging.warning(
"Indexer: IGNORE COLUMN `%s` in source `%s` because of empty values",
column.name, source.name)
def init_analyzers(self, index_config):
logging.info("Initializing analyzers")
self.es.indices.create(
index=get_index_name(index_config),
body={
"settings": {
"analysis": {
"analyzer": {
"textual": {
"filter": [
"standard",
"lowercase",
"stop",
],
"type": "custom",
"tokenizer": "standard"
},
"number_text": {
"filter": [
"lowercase",
"word_delimiter",
"stop",
],
"type": "custom",
"tokenizer": "standard"
},
"whitespace_text": {
"filter": ["lowercase", "stop", "kstem"],
"type": "custom",
"tokenizer": "whitespace"
}
}
}
}
})
logging.debug("Done: Initializing analyzers")
def index_column(self, column, source_name, index_config):
body = column.to_json()
body['source'] = source_name
self.es.index(index=get_index_name(index_config),
doc_type="service",
body=body)
def init_analyzers(self, index_config):
self.es.indices.create(index=get_index_name(index_config))
def delete_column(self, index_config):
logging.info("Deleting index for column")
if self.es.indices.exists(get_index_name(index_config)):
self.es.delete(index=get_index_name(index_config))
return True
return False
def init_analyzers(self, index_config):
self.es.indices.create(index=get_index_name(index_config))
def index_column(self, column, source_name, index_config):
body = column.to_json()
body['source'] = source_name
self.es.index(index=get_index_name(index_config), doc_type="service",
body=body)