def delete_all():
es = Elasticsearch()
search_body = {"from": 0, "size": 10000, "query": {"query_string": {"query": "*"}}}
for e in es.search(body=search_body).get("hits").get("hits"):
print(e["_type"])
print(e["_id"])
es.delete(index=config.db_name, doc_type=e["_type"], id=e["_id"])
def es_delete(self):
es = Elasticsearch()
try:
es.delete(index=ES_FAMILIAS_INDEX, doc_type=ES_FAMILIAS_DOC_TYPE, id=self.cod_material)
# @todo NotFoundError
except:
pass
def delete():
es = Elasticsearch([{'host':'219.224.134.214','port':9202}])
query_body = {"size":100000,"query":{ "filtered": {
"filter":{"range":{"publish_time":{"gte": 1520380800,"lte": 1528329600}}}
}}}
res = es.search(index="announcement", doc_type="basic_info", body=query_body,request_timeout=100)
hits = res['hits']['hits']
print len(hits)
num = 0
for hit in hits:
es.delete(index="announcement", doc_type="basic_info", id=hit['_id'])
if num %1000 == 0:
print num
num += 1
def _elastic(self, doc_id=None, doc={}, option='create'):
"""
option:
init: 初始化文档结构。（当config.ini中的init为True时才会执行。）
create: 若文档已存在，则不执行任何操作。 若文档不存在，则直接创建。
update: 若文档已存在，则直接更新。 若文档不存在，则不执行任何操作。
delete: 若文档已存在，则直接删除。若文档不存在，则不执行任何操作。
"""
esclient = Elasticsearch([self.elastic])
status = 'Success !'
if 'create' == option:
try:
esclient.create(
index=self.elastic['index'],
doc_type=self.elastic['type'],
id=doc_id,
body=doc,
)
except ConflictError:
status = 'Fail(existsd) !'
elif 'update' == option:
try:
esclient.update(
index=self.elastic['index'],
doc_type=self.elastic['type'],
id=doc_id,
body={'doc': doc},
)
except NotFoundError:
status = 'Fail(not existsd) !'
elif 'delete' == option:
try:
esclient.delete(
index=self.elastic['index'],
doc_type=self.elastic['type'],
id=doc_id,
)
except NotFoundError:
status = 'Fail(not existsd) !'
elif 'init' == option:
try:
IndicesClient(esclient).create(
index=self.elastic['index'],
body=doc,
)
except RequestError:
status = 'Fail(existsd) !'
self.logger.record('Sync@%s < %s-%s-%s > %s' % (
option,
self.elastic['index'],
self.elastic['type'],
doc_id,
status,
))
class StreamingIndexer(TwythonStreamer):
def __init__(self, consumer_key=None, consumer_secret=None,
access_token=None, access_token_secret=None,
es_host=None, es_port=None, es_index=None):
super(StreamingIndexer, self).__init__(consumer_key, consumer_secret,
access_token,
access_token_secret)
self._es = Elasticsearch([{'host': es_host, 'port': es_port}])
self._index = es_index
def on_success(self, tweet):
if 'delete' in tweet:
status_id = tweet['delete']['status']['id']
self._es.delete(self._index, 'tweet', status_id)
return
if 'retweeted_status' in tweet:
tweet = tweet['retweeted_status']
for url in tweet['entities']['urls']:
if 'theguardian.com' in url['expanded_url']:
url['domain'] = 'theguardian.com'
self._es.index(index=self._index, doc_type='tweet',
id=tweet['id_str'], body=tweet)
class ElasticSearch(TableStorage):
"""ElasticSearch provider for TableStorage."""
def __init__(self, config: ElasticSearchConfig):
self._es_instance = Elasticsearch(config.access_url,
http_auth=('elastic',
config.access_key))
self._doc_type = 'entries'
def write(self, resource):
"""Write resource to table.
:param resource: Expecting Resource object
(see Common.Contracts.Resource)
:return: None
"""
entry = resource.to_dict()
location = entry['location']
identifier = str(uuid.uuid3(uuid.NAMESPACE_DNS, entry['id']))
self._es_instance.index(index=location,
doc_type=self._doc_type,
id=identifier,
body=entry)
def query(self, partition_key, row_key):
"""Get entry with specified partition and row keys.
:param partition_key: Partition key for entry
:param row_key: Row key for entry
:return: Entity if found, None otherwise
"""
task = self._es_instance.get(index=partition_key,
doc_type=self._doc_type,
id=row_key)
return task
def query_list(self):
"""Get entities from table.
:return: List of entities from table
"""
return self._es_instance.search("*")
def delete(self, partition_key, row_key):
"""Delete entry with specified partition and row keys.
:param partition_key: Partition key for entry
:param row_key: Row key for entry
:return: None
"""
self._es_instance.delete(index=partition_key,
doc_type=self._doc_type,
id=row_key)
@staticmethod
def create():
"""Initialize ElasticSearch service.
:return: ElasticSearch service object
"""
return ElasticSearch(ElasticSearchConfig())
def delete_es(video):
es = Elasticsearch(
ES_URL,
timeout=ES_TIMEOUT,
max_retries=ES_MAX_RETRIES,
retry_on_timeout=True,
)
if es.ping():
try:
if ES_VERSION == 7:
delete = es.delete(index=ES_INDEX,
id=video.id,
refresh=True,
ignore=[400, 404])
else:
delete = es.delete(
index=ES_INDEX,
doc_type="pod",
id=video.id,
refresh=True,
ignore=[400, 404],
)
if DEBUG:
logger.info(delete)
return delete
except TransportError as e:
logger.error("An error occured during delete video : %s-%s : %s" %
(e.status_code, e.error, e.info))
class ElasticsearchModule(AbstractModule):
"""
Override the elastic API. (Not really usefull for the moment)
"""
def __init__(self, config_parser):
self.SETTINGS = {
"host": config_parser.get('elasticsearch', 'host'),
"port": config_parser.get('elasticsearch', 'port')
}
self.es_stream = Elasticsearch()
self.insert_number = 0
self.update_number = 0
self.delete_number = 0
def insert(self, index, doc_type, id, doc):
self.es_stream.index(index, doc_type, doc, id)
self.insert_number += 1
def delete(self, index, doc_type, id):
self.es_stream.delete(index, doc_type, id)
self.delete_number += 1
def update(self, index, doc_type, id, doc):
self.insert(index, doc_type, id, doc)
self.update_number += 1
@property
def server_information(self):
return {
'elastic_avalaible': self.es_stream.ping(),
'insert_number': self.insert_number,
'update_number': self.update_number,
'delete_number': self.delete_number
}
def unindexer(self, handle, ring=None, idx=None):
# Connect to Elastic Search Node
es = Elasticsearch([ES_NODE])
out = {}
out['unindexed'] = []
if handle and ring and idx:
es.delete(index=handle, doc_type=ring, id=idx, ignore=[400, 404])
i = '%s/%s/%s' % (handle, ring, idx)
elif handle and ring:
#es.delete(index=handle,doc_type=ring, ignore=[400, 404]) This doesnt work
# Could not make elasticsearch_py delete a doc_type only
requests.delete('%s/%s/%s' % (ES_NODE, handle, ring))
i = '%s/%s' % (handle, ring)
elif handle:
es.indices.delete(index=handle, ignore=[400, 404])
i = '%s' % (handle)
self.lggr.info('UnIndexing:%s' % i)
out['unindexed'].append(i)
d = {}
d['json_out'] = json.dumps(out)
d['template'] = 'base_json.html'
return d
def instaimport(searchindex, thisdoctype, newsbody):
# Index by Elastic
es = Elasticsearch(['media-audit.com'],
http_auth=('elastic', 'changeme'),
port=9200)
if newsbody:
searchindex = 'instagram'
try:
es.create(index=searchindex,
doc_type=thisdoctype,
body=newsbody,
id=newsbody['id'])
except:
es.delete(index=searchindex,
doc_type=thisdoctype,
id=newsbody['id'])
print('Delete Id =' + newsbody['id'])
try:
es.create(index=searchindex,
doc_type=thisdoctype,
body=newsbody,
id=newsbody['id'])
print('Id = ' + newsbody['id'] + ' was upload \n')
except:
es.delete(index=searchindex,
doc_type=thisdoctype,
id=newsbody['id'])
print("Skip")
return 'ok'
def delete():
es = Elasticsearch([{'host': '219.224.134.214', 'port': 9202}])
query_body = {
"size": 100000,
"query": {
"filtered": {
"filter": {
"range": {
"publish_time": {
"gte": 1520380800,
"lte": 1528329600
}
}
}
}
}
}
res = es.search(index="announcement",
doc_type="basic_info",
body=query_body,
request_timeout=100)
hits = res['hits']['hits']
print len(hits)
num = 0
for hit in hits:
es.delete(index="announcement", doc_type="basic_info", id=hit['_id'])
if num % 1000 == 0:
print num
num += 1
class ESModule:
'''
elasticsearch 操作类
'''
def __init__(self):
self.es = Elasticsearch(ES_ADDRESS)
def update(self, index, my_id, json_data, doc_type='data'):
'''
es 更新操作
:param index: es 的索引
:param my_id: 每条数据的 ID
:param json_data: 需要写入的 json 数据
:param doc_type: 类型，默认为 data
:return: None
'''
body = list()
body.append(
{'update': {
'_id': my_id,
'_type': doc_type,
'_index': index
}})
body.append({'doc': json_data, 'doc_as_upsert': True})
self.es.bulk(body)
def delete(self, index, my_id, doc_type='data'):
'''
es 删除操作
:param index: es 的索引
:param my_id: 每条数据的 ID
:param doc_type: 类型，默认为 data
:return: None
'''
self.es.delete(index=index, doc_type=doc_type, id=my_id)
class NewsDelInfo(object):
def __init__(self):
self.hbase_con = HbaseInfoTask()
self.redis_con = RedisTools()
self.es = Elasticsearch(ES_ADDR, ignore=404)
def es_ping(self):
if not self.es.ping():
self.es = Elasticsearch(ES_ADDR, ignore=404)
def run(self):
while True:
rowkey = self.redis_con.get_yy_rowkey("es:news:del:info")
_id = trans_md5(rowkey)
self.es_ping()
try:
boo = self.es.exists(index="xw_info",doc_type="sino",id=_id)
if boo:
self.es.delete(index="xw_info",doc_type="sino",id=_id)
except Exception as e:
log_info = "news info delete error %s" %str(e)
logging.error(log_info)
boo = self.es.exists(index="xw_info", doc_type="sino", id=_id)
if boo:
self.es.delete(index="xw_info", doc_type="sino", id=_id)
class ElasticsearchTest(unittest.TestCase):
def setUp(self) -> None:
self.elasticsearch = Elasticsearch(host='localhost', port=9200)
self.index = "elasticsearch_test"
self.data = {"test": "elasticsearch_service"}
def test_ES_connection(self):
res = requests.get('http://localhost:9200')
self.assertEqual(200, res.status_code)
def test_service_running(self):
ServiceV().validate('isrunning', ["elasticsearch"])
def test_insert_data(self):
# Insert data
res = self.elasticsearch.index(index=self.index, id=1, body=self.data)
self.assertEqual(res['result'], "created")
self.elasticsearch.delete(index=self.index, id=1)
def test_get_data(self):
# Get data
self.elasticsearch.index(index=self.index, id=1, body=self.data)
res = self.elasticsearch.get(index=self.index, id=1)
self.assertIs(res['found'], True)
self.elasticsearch.delete(index=self.index, id=1)
def delete_data(self):
# Delete index
self.elasticsearch.index(index=self.index, id=1, body=self.data)
res = self.elasticsearch.delete(index=self.index)
self.assertIs(res['acknowledged'], True)
def deleteESItem(elasticsearchDomain, documentId):
host = elasticsearchDomain
if (documentId):
service = 'es'
ss = boto3.Session()
credentials = ss.get_credentials()
region = ss.region_name
awsauth = AWS4Auth(credentials.access_key,
credentials.secret_key,
region,
service,
session_token=credentials.token)
es = Elasticsearch(hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
if es.exists(index="textract", doc_type="document", id=documentId):
es.delete(index="textract", doc_type="document", id=documentId)
print("Deleted document: {}".format(documentId))
class ElasticSearchUtils(object):
def __init__(self):
self.es = Elasticsearch(hosts=[{
'host': ELASTIC_SEARCH_CONFIG['host'],
'port': ELASTIC_SEARCH_CONFIG['port']
}])
def insert(self, index, doc_type, body, id=None):
"""
插入文档
:param index: 索引名称
:param doc_type: 文档名称
:param body: 文档内容
:param id: 序列号
:return:
"""
self.es.index(index, doc_type, body, id=id)
def count(self, index_name):
"""
:param index_name:
:return: 统计index总数
"""
return self.es.count(index=index_name)
def delete(self, index_name, doc_type, id):
"""
:param index_name:
:param doc_type:
:param id:
:return: 删除index中具体的一条
"""
self.es.delete(index=index_name, doc_type=doc_type, id=id)
def get(self, index_name, doc_type, id):
"""
根据文档编号索引数据
:param index_name: 索引名称
:param doc_type: 文档类型名称
:param id: 编号
:return:
"""
return self.es.get(index_name, doc_type, id=id)
def search(self, index_name, doc_type, body, size=10):
"""
根据条件进行搜索
:param index_name: 索引名称
:param doc_type: 文档类型名称
:param body: 查询条件
:param size: 查询数据条目
:return:
"""
try:
results = self.es.search(index=index_name,
doc_type=doc_type,
body=body)
return results['hits']['hits'][:size]
except Exception as err:
logger.info(err)
def set_previous_config():
hosts = os.environ['ES_HOSTS'].split(',')
es = Elasticsearch(hosts=hosts, maxsize=1)
prev_conf = {
"weights": {
"desktop": {
"1": 0.499,
"2": 0.501,
"updated": "2017-12-04 03:10:47"
},
"mobile": {}
}
}
result = es.index(
index='bandit_pio',
doc_type='bandit_config',
id=0,
body=json.dumps(prev_conf),
)
yield prev_conf
es.delete(
index='bandit_pio',
doc_type='bandit_config',
id=0,
)
def deleteData(urlCrawl, dataDate, u_id):
es = Elasticsearch([{
'host': app.config['ELASTICSEARCH_URI'],
'port': app.config['ELASTICSEARCH_PORT']
}])
# doc = {
# 'src_ip': urlCrawl.src_ip
# , 'dst_ip': urlCrawl.dst_ip
# , 'uri': urlCrawl.uri
# , 'domain': urlCrawl.domain
# , 'isCrawled': urlCrawl.isCrawled
# , 'depth': urlCrawl.depth
# , 'src_geoip': urlCrawl.src_geoip
# , 'dst_geoip': urlCrawl.dst_geoip
# , 'desc': urlCrawl.desc
# , 'register_path': urlCrawl.register_path
# , '@timestamp': datetime.utcnow().isoformat()
# , 'min_timestamp': datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
#
# }
dataindex = 'gsp-{0}'.format(dataDate)
#.format(datetime.now().strftime("%Y.%m.%d"))
try:
es.delete(index=dataindex, doc_type="url_jobs", id=u_id)
# delete(self, index, doc_type, id, params=None)
except Exception as e:
raise e
return json.dumps({'success': True}), 200, {
'ContentType': 'application/json'
}
class ElasticService():
def __init__(self, host, port, username, password):
super().__init__()
requests.packages.urllib3.disable_warnings()
try:
log.info(f"connecting to elastic on host: {host}")
self.es = Elasticsearch(f"https://{host}:{port}",
http_auth=(username, password),
verify_certs=False)
self.es.info()
except Exception as e:
log.error(e)
raise
def create_index(self, index_name):
try:
print("creating index if not exists")
self.es.indices.create(index=index_name, ignore=400)
except Exception as e:
log.error(e)
def create_doc(self, index_name, id, body):
try:
log.info("creating doc in elastic")
self.es.create(index=index_name, id=id, body=body)
except Exception as e:
log.error(e)
def delete_doc(self, index_name, id):
try:
log.info("deleting doc from elastic")
self.es.delete(index=index_name, id=id)
except Exception as e:
log.error(e)
class SearchEngine():
def __init__(self, config) -> None:
url = config['URL'] if config['URL'] else 'localhost:9200'
try:
self.search_engine = Elasticsearch([url])
except Exception as err:
print(f'Ooops! Search engine error: {err}')
def is_connected(self):
return self.search_engine.ping()
def index(self, index, id, body):
if self.is_connected():
self.search_engine.index(index=index, id=id, body=body)
def delete(self, index, id):
if self.is_connected():
self.search_engine.delete(index=index, id=id)
def delete_index(self, index):
if self.is_connected():
self.search_engine.indices.delete(index=index)
def search(self, index, body):
if self.is_connected():
return self.search_engine.search(index=index, body=body)
else:
return None
def update_index():
es_settings = getattr(settings, "ELASTICSEARCH", "")
es = Elasticsearch([es_settings])
published_ids = set()
for service in Service.objects.all():
if service.publish:
ser = ServiceSerializer(service)
es.update(index='employment',
doc_type='employment_service',
id=service.id,
body={
'doc': ser.data,
'doc_as_upsert': True
})
published_ids.add(service.id)
else:
es.delete(index='employment',
doc_type='employment_service',
id=service.id,
ignore=[400, 404])
for doc in es.search(index="employment")['hits']['hits']:
if not int(doc['_id']) in published_ids:
print(doc['_id'])
es.delete(index='employment',
doc_type='employment_service',
id=int(doc['_id']),
ignore=[400, 404])
class ElasticSearchTeamMemberRepository(TeamMemberRepository):
def __init__(self):
self.es = Elasticsearch(
hosts=[{'host': 'elasticsearch'}]
)
def delete_by_id(self, id):
self.es.delete(index="pits", doc_type="_doc", id=id)
def update(self, id, team_member):
json_team_member = json.dumps(team_member.__dict__)
self.es.index(index="pits", doc_type="_doc", id=id, body=json_team_member)
def add(self, id, team_member):
json_team_member = json.dumps(team_member.__dict__)
self.es.index(index="pits", doc_type="_doc", id=id, body=json_team_member)
def get_all(self):
team_members = []
result = self.es.search(index="pits", doc_type="_doc", body={"query": {"match_all": {}}})
for hit in result['hits']['hits']:
hit['_source']['id']=hit['_id']
team_member=hit['_source']
team_members.append(team_member)
return team_members
class ElasticSearch(object):
def check_node_status(self):
res = requests.get('http://localhost:9200')
if res.status_code == 200:
return (res.content)
return None
def connect_es(self):
self.es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
def add_document(self, id, data):
self.es.index(index='green_bond', doc_type='report', id=id, body=data)
def update_document(self, index, doc, id, data):
self.es.delete(index=index, doc_type=doc, id=id)
self.add_document(id, data)
def check_document_exists(self, index, doc, id):
return self.es.exists(index=index, doc_type=doc, id=id)
def find_document(self, index, doc, id):
return self.es.get(index=index, doc_type=doc, id=id)
def get_all_document(self, index, doc):
return self.es.search(index=index, doc_type=doc, size=1000, pretty=1)
def deleteKids(ids):
from elasticsearch import Elasticsearch
{
"nested": {
"path": "is_conflation_of",
"query": {
"nested": {
"path": "is_conflation_of.types",
"query": {
"terms": {
"is_conflation_of.place_id": ids
}
}
}
}
}
}
q = {"query": {"terms": {"": ds}}}
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
for i in ids:
try:
es.delete(index='whg', doc_type='place', id=i)
except:
print('failed delete for: ', id)
pass
def delete_post():
# "POST /delete/post" -> "deleted.html"のレンダリング
title = get_title('削除完了')
isbn10 = request.form['isbn10'] # 削除対象書籍ISBN-10コード
es = Elasticsearch('elasticsearch')
book_title = es.get_source(index='book', id=isbn10)['title'] # 削除対象書籍タイトル
es.delete(index='book', id=isbn10) # bookインデックスから対象書籍削除
logger.debug('書籍の削除に成功しました (ISBN-10: {})'.format(isbn10))
es.indices.refresh(
index='book') # bookインデックス更新 <- 後のD2Vモデル再訓練時に削除した書籍が混入しないようにするため
es.close()
# 削除した書籍を推薦対象外とするため，削除ごとにDoc2Vecモデルを再構築
global d2v
d2v = Doc2VecWrapper(model_path=Path('/projects/model/d2v.model'),
initialize=True)
d2v.train()
return render_template('deleted.html',
shishosan=config['shishosan'],
title=title,
isbn10=isbn10,
book_title=book_title)
def run_clean(json_dict):
global es_client
try:
es_client = Elasticsearch(hosts=es_hosts)
except (ElasticsearchException, Exception) as ex:
logger.warn('failed to connect to %s: %s' % (es_hosts, ex))
es_index = json_dict.pop('index')
es_type = json_dict.pop('type')
search_project = json_dict.pop('project')
search_body = {
"query": {
"bool": {
"must": [
{
"terms": {
"project": [
search_project
]
}
}
]
}
},
"from": 0,
"size": 50
}
for key, value in json_dict.items():
search_body["query"]["bool"]["must"].append({
"match_phrase": {
key: value
}
})
while True:
try:
logger.info('searching %s:%s from %d to %d ' %
(es_index, search_body["query"]["bool"]["must"],
search_body["from"], search_body["from"] + search_body["size"]))
es_response = es_client.search(
index=es_index,
body=search_body,
_source=False
)
except (ElasticsearchException, Exception) as ex:
logger.warn('failed to search %s: %s' % (search_body, ex))
break
result_len = len(es_response['hits']['hits'])
logger.info("search %d results" % result_len)
if result_len == 0:
break
search_body["from"] += result_len
for hit in es_response['hits']['hits']:
logger.info("deleting %s" % (hit["_id"]))
es_client.delete(index=es_index, doc_type=es_type, id=hit["_id"])
def remove_duplicate(self):
es = Elasticsearch(settings.ELASTIC_SEARCH_HOST)
try:
es.delete(index="full-search-base",
id="devon_bleibtrey",
doc_type="quest")
except NotFoundError:
pass
class Search():
def __init__(self, index_name):
super().__init__()
self.logger = logging.getLogger(__name__)
self.__es = Elasticsearch(['pulsing.jhk.org:9200'], sniff_on_start=True)
self.__index_name = index_name
if self.__es.indices.exists(self.__index_name):
self.logger.debug('index exists so deleting ' + self.__index_name)
self.__es.indices.delete(self.__index_name)
self.__es.indices.create(self.__index_name)
self.__es.cluster.health(wait_for_status='yellow')
def index(self, type_name, id_value, content):
self.logger.debug('index %s/%s : %s', type_name, id_value, content)
self.__es.index(index=self.__index_name, doc_type=type_name, id=id_value, body=content)
def map(self, type_name, mapping):
self.logger.debug('map %s', type_name)
self.__es.indices.put_mapping(index=self.__index_name, doc_type=type_name, body={type_name: mapping})
def search(self, type_name, query={'match_all': {}}):
self.logger.debug('search %s : %s', type_name, query)
return self.__es.search(index=self.__index_name, doc_type=type_name, body={'query': query})
def get(self, type_name, id_value):
self.logger.debug('get %s/%s', type_name, id_value)
document = self.__es.get(index=self.__index_name, doc_type=type_name, id=id_value)
self.logger.debug('got document ' + document)
return document
def delete(self, type_name, id_value):
self.logger.debug('delete %s/%s', type_name, id_value)
self.__es.delete(index=self.__index_name, doc_type=type_name, id=id_value)
def optimize(self):
"""
forcemerge allows removal of deleted documents and reducing the number of segments
(documents are marked as tombstone [like cassandra] but not purged from the segment's
index for performance reasons)
"""
self.logger.debug('optimize')
self.__es.forcemerge(self.__index_name)
@property
def es(self):
return self.__es
def __eq__(self, other):
return self.__es == other.__es
def __str__(self):
return self.__es.__str__()
def __hash__(self):
return self.__es.__hash__()
class SearchIndex(metaclass=Singleton):
def __init__(self):
self.es = Elasticsearch([{"host": "localhost", "port": "9200"}])
mapping = {
"mappings": {
"properties": {
"title_vector": {
"type": "dense_vector",
"dims": 128
},
"title_name": {
"type": "keyword"
},
}
}
}
try:
self.es.indices.create(index="final_face_recognition",
body=mapping)
except RequestError:
print("Index already exists!!")
def push(self, emb, index, image_name=None):
doc = {"title_vector": emb, "title_name": image_name}
self.es.create("final_face_recognition", id=index, body=doc)
def delete(self, index):
self.es.delete(index="final_face_recognition", id=index)
def search(self, emb, size):
"""
size : # nearest neighbours
"""
query = {
"size": size, # foe ex 5 nearest neighbours
"query": {
"script_score": {
"query": {
"match_all": {}
},
"script": {
"source":
"cosineSimilarity(params.queryVector, 'title_vector')+1",
# "source": "1 / (1 + l2norm(params.queryVector, 'title_vector'))", #euclidean distance
"params": {
"queryVector": list(emb)
},
},
}
},
}
res = self.es.search(index="final_face_recognition", body=query)
return res
class XiaomiElasticSearchPipeline(object):
items_buffer = []
def __init__(self):
self.settings = get_project_settings()
# settings = get_project_settings()
# self.settings = settings
uri = "{}:{}".format(self.settings['ELASTICSEARCH_SERVER'], self.settings['ELASTICSEARCH_PORT'])
self.es = Elasticsearch([uri])
# uri = "%s:%d" % (self.settings['ELASTICSEARCH_SERVER'], self.settings['ELASTICSEARCH_PORT'])
# self.es = Elasticsearch(, serializer=JSONSerializerPython2())
# print uri
# print type(settings)
def index_item(self, item):
index_name = self.settings['ELASTICSEARCH_INDEX']
index_suffix_format = self.settings.get('ELASTICSEARCH_INDEX_DATE_FORMAT', None)
if index_suffix_format:
index_name += "-" + datetime.strftime(datetime.now(), index_suffix_format)
index_action = {
'_index': index_name,
'_type': self.settings['ELASTICSEARCH_TYPE'],
'_source': dict(item)
}
self.items_buffer.append(index_action)
# index_name = self.settings['ELASTICSEARCH_INDEX']
# self.es.index(index_name, doc_type="test-type", body=dict(item), id=item['appid'], op_type='create')
if len(self.items_buffer) == self.settings.get('ELASTICSEARCH_BUFFER_LENGTH', 500):
self.send_items()
self.items_buffer = []
def send_items(self):
helpers.bulk(self.es, self.items_buffer)
def process_item(self, item, spider):
if isinstance(item, types.GeneratorType) or isinstance(item, types.ListType):
for each in item:
self.process_item(each, spider)
else:
self.index_item(item)
# index_name = self.settings['ELASTICSEARCH_INDEX']
# self.es.index(dict(item), index_name, self.settings['ELASTICSEARCH_TYPE'], op_type='create')
logging.info("Remove old values in Elasticsearch if exit")
self.es.delete(self.settings['ELASTICSEARCH_INDEX'], self.settings['ELASTICSEARCH_TYPE'], id=item['appid'], ignore=[400, 404])
self.es.index(self.settings['ELASTICSEARCH_INDEX'], self.settings['ELASTICSEARCH_TYPE'], dict(item), id=item['appid'], op_type='create', )
# self.es.index()
return item
def close_spider(self, spider):
if not self.items_buffer:
self.send_items()
class IndexCreator(object):
nameseg = NameSegmenter()
identifier = Identifier()
stopwords = stopword.get_standard_stopwords()
seg = Segmenter()
def __init__(self, es=None):
global logger_index
self.logger = logger_index
if not es:
host, port = tsbconfig.get_es_config()
self.logger.info(','.join([host, port]))
self.es = Elasticsearch([{'host': host, 'port': port}])
# host, port, user, pswd = tsbconfig.get_es_config()
# self.logger.info(','.join([host, port, user, pswd]))
# self.es = Elasticsearch([{'host': host, 'port': port}], http_auth=(user, pswd))
else:
self.es = es
self.logger.info(self.es)
self.logger.info('Index Creator inited')
def __check(self):
if not self.es.indices.exists(["xiniudata"]):
self.logger.info('Creating index xiniudata')
self.es.indices.create("xiniudata")
self.logger.info('Created')
self.es.indices.put_mapping("company", mappings.get_company_mapping(),
"xiniudata")
self.logger.info('Company mapping created')
self.es.indices.put_mapping("completion",
mappings.get_completion_mapping(),
"xiniudata")
self.logger.info('Completion mapping created')
self.es.indices.put_mapping("dealCompletion",
mappings.get_deal_completion_mapping(),
"xiniudata")
self.logger.info('Deal Completion mapping created')
self.es.indices.put_mapping("deal", mappings.get_deal_mapping(),
"xiniudata")
self.logger.info('Deal mapping created')
self.es.indices.put_mapping("interior",
mappings.get_company_interior_mapping(),
"xiniudata")
self.logger.info('Interior mapping created')
self.es.indices.put_mapping("digital_token",
mappings.get_digital_token_mapping(),
"xiniudata")
self.logger.info('Token mapping created')
def delete_index(self, doc_type, id):
try:
self.es.delete('xiniudata', doc_type, id)
except NotFoundError, efe:
pass
def delete(_id):
try:
from elasticsearch import Elasticsearch
es = Elasticsearch()
es.delete(index="pessoa", doc_type="_doc",id=str(_id))
return jsonify({"resultado":"A entrada com id: " + str(_id) + ", foi efetuada"})
except:
return jsonify({"resultado":"Erro de execução"})
def get_whois_ip(ip,refresh=None):
es = Elasticsearch()
print repr(ip)
id_num = str(ip).replace(".","0")
does_exist = es.exists(index='rwhois2', doc_type='ipaddr', id = id_num)
print does_exist
if does_exist is True and refresh is None:
status = 200
print "Found it!"
get_record = es.get(index='rwhois2',doc_type='ipaddr', id = id_num)
results = jsonify(get_record['_source'])
elif does_exist is True and refresh is not None:
status = 200
print "Forcing refresh!"
es.delete(index='rwhois2', doc_type='ipaddr', id = id_num)
try:
ipwhois.net.socks.setdefaultproxy(ipwhois.net.socks.SOCKS5,"localhost")
obj = IPWhois(ip)
try:
results_raw = obj.lookup_whois(get_referral=True,inc_nir=True)
except:
results_raw = obj.lookup_whois()
status = 200
results = jsonify(results_raw)
es.index(index='rwhois2', doc_type='ipaddr', id=id_num, body=results_raw)
except Exception as e:
print e
results = jsonify({'status': "not_found"})
status = 404
else:
try:
obj = IPWhois(ip)
try:
results_raw = obj.lookup_whois(get_referral=True)
except:
results_raw = obj.lookup_whois()
status = 200
results = jsonify(results_raw)
id_num = str(ip).replace(".","0")
print results
try:
es.index(index='rwhois2', doc_type='ipaddr', id=id_num, body=results_raw)
except Exception as e:
print "Elasticsearch encountered a problem ", e
pass
except Exception as e:
#print results_raw
print e
results_raw = jsonify({'status': "not_found"})
status = 404
results = jsonify({'status': "not_found"})
return results,status
def index_remove(self):
es = Elasticsearch(settings.ELASTICSEARCH, timeout=60)
name = self.get_basename()
try:
es.delete(index=name.lower(), doc_type="log", id=self.id)
except Exception as e:
logger.debug("delete index: %s" % e)
self.is_indexed = False
self.save()
def index_remove(self):
es = Elasticsearch(settings.ELASTICSEARCH, timeout=60)
name = self.get_basename()
try:
es.delete(index=name.lower(), doc_type="log", id=self.id)
except Exception as e:
logger.debug("delete index: %s" % e)
self.is_indexed = False
self.save()
def deleteDocs(ids):
from elasticsearch import Elasticsearch
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
for i in ids:
try:
es.delete(index='whg', doc_type='place', id=i)
except:
print('failed delete for: ', id)
pass
def remove_duplicate_plebs_search(self):
es = Elasticsearch(settings.ELASTIC_SEARCH_HOST)
for pleb in Pleb.nodes.all():
try:
es.delete(index='full-search-base',
doc_type='profile',
id=pleb.username)
except NotFoundError:
pass
def search(keys):
service = 'es'
region = 'us-west-2'
# headers = {"Content-Type" : "application/json"}
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key,
credentials.secret_key,
region,
service,
session_token=credentials.token)
es = Elasticsearch(hosts=[{
'host': HOST,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
results = []
for key in keys:
# print(key)
r = es.search(index="event",
body={
"from": 0,
"size": 10,
"query": {
"match": {
"labels": key
}
}
})
for r in r["hits"]["hits"]:
print('find', r)
esId = r['_id']
eventTime = r['_source'].get('eventTime')
# print(eventTime)
try:
if not eventTime or datetime.datetime.strptime(
eventTime,
'%Y-%m-%d %H:%M:%S.%f') < datetime.datetime.now():
es.delete(index="event", id=esId)
print('delete', r)
continue
except:
es.delete(index="event", id=esId)
print('delete', r)
continue
eventId = r['_source']['eventId']
results.append(eventId)
print('put', r)
return results
def delete_course_from_index(target):
es = Elasticsearch()
for language in settings.LANGUAGE_CODES:
try:
es.delete(index='courses',
doc_type='course_%s' % language,
id=target.id)
except TransportError:
pass
def deleteQuestion( request, id ):
#get the response using id and search for the question
es = ES('http://127.0.0.1:9200/')
es.delete( index="test-rule1", doc_type="book", id=id )
template = loader.get_template('polls/qdelete.html')
context = RequestContext( request )
return HttpResponse(template.render(context))
class ESClient:
def __init__(self, es_params):
self.es = Elasticsearch(es_params)
def get_metric_metadata(self, metric_name, tenant_id):
"""
Get document from index metric_metadata for a given metric name and tenant id
"""
document_id = self.get_document_id(tenant_id=tenant_id, metric_name=metric_name)
try:
return self.es.get(index='metric_metadata', doc_type='metrics', id=document_id, routing=tenant_id)
except NotFoundError as e:
return e.info
def get_enums_data(self, metric_name, tenant_id):
"""
Get document from index enums for a given metric name and tenant id
"""
document_id = self.get_document_id(tenant_id=tenant_id, metric_name=metric_name)
try:
return self.es.get(index='enums', doc_type='metrics', id=document_id, routing=tenant_id)
except NotFoundError as e:
return e.info
def delete_metric_metadata(self, metric_name, tenant_id):
"""
Delete document from index metric_metadata for metric_metadata dictionary(obtained from get_metric_metadata
call) and tenant id
"""
document_id = self.get_document_id(tenant_id=tenant_id, metric_name=metric_name)
self.es.delete(index='metric_metadata', doc_type='metrics', id=document_id, routing=tenant_id)
print 'Deleted from index metric_metadata for _id: [%s] routing: [%s]' % (document_id, tenant_id)
def delete_enums_data(self, metric_name, tenant_id):
"""
Delete document from index enums for enums dictionary(obtained from get_enums_data
call) and tenant id
"""
document_id = self.get_document_id(tenant_id=tenant_id, metric_name=metric_name)
self.es.delete(index='enums', doc_type='metrics', id=document_id, routing=tenant_id)
print 'Deleted from index enums for _id: [%s] routing: [%s]' % (document_id, tenant_id)
def get_document_id(self, tenant_id, metric_name):
"""
Construct _id of elastic search from tenant id and metric name
"""
return tenant_id + ':' + metric_name
class ElasticSearchDb(PersistenceBase):
def __init__(self):
base = PersistenceBase()
base.__init__()
self.session = Elasticsearch()
self.database = Config().elasticsearchindex
if (not self.session.indices.exists(index=self.database)):
self.session.indices.create(index=self.database)
def selectalltables(self):
tables = []
res = self.session.indices.get_mapping(index=self.database)
for map in res[self.database]['mappings']:
tables.append(map)
return tables
def selectallcolumns(self, tablename):
columns = []
res = self.session.indices.get_mapping(index=self.database)
for column in res[self.database]['mappings'][tablename]['properties']:
columns.append(column)
return columns
def selectall(self, tablename):
res = self.session.get(index=self.database, doc_type=tablename)
return res
def selectone(self, tablename, id):
queryfilter = {'uuid': id}
res = self.session.get(index=self.database, doc_type=tablename, body={'query': queryfilter})
return res
def insert(self, obj, tablename):
obj.updatedAt = datetime.isoformat(datetime.now())
obj.uuid = str(self.getuuid())
serialized_obj = self.getallvaluesfromobject(obj)
self.session.index(index=self.database, doc_type=tablename, body=serialized_obj, id=obj.uuid)
def update(self, obj, tablename):
obj.updatedAt = datetime.isoformat(datetime.now())
self.session.update(index=self.database, doc_type=tablename, id=obj.uuid)
def delete(self, obj, tablename):
#todo: add to a table that manage deleted items (just to know if the obj was deleted and we don't have to add again)
self.session.delete(index=self.database, doc_type=tablename, id=obj.uuid)
def getallvaluesfromobject(self, obj):
print(obj)
ret = json.dumps(obj, default=lambda o: o.__dict__)
return ret
def get(self, request, *args, **kwargs):
note_id = args[0]
es = Elasticsearch("s3.zserg.net:9200")
res = es.delete(index="notes", doc_type='note', id = note_id)
context = {'result':res['found'], 'note_id':note_id}
return render(request, self.template_name, context)
class ElasticSearchProvider(object):
"""
elasticsearch全文搜索SDK
"""
def __init__(self, hosts=None):
self._es = Elasticsearch(hosts)
def insert(self, index, doc_type, doc):
"""
:arg schema: es的_index
:arg table: es的_type
:arg row: 需要更新的doc
"""
res = self._es.index(index, doc_type, doc, doc['id'])
return res['created']
def update(self, index, doc_type, doc):
"""
:arg schema: es的_index
:arg table: es的_type
:arg row: 需要更新的doc
"""
self._es.index(index, doc_type, doc, doc['id'])
return True
def delete(self, index, doc_type, doc):
"""
:arg schema: es的_index
:arg table: es的_type
:arg row: 需要更新的doc
"""
res = self._es.delete(index, doc_type, doc['id'])
return res['found']
class ObjectManager(object):
def __init__(self, index, doc_type, model_class):
super(ObjectManager, self).__init__()
self.index = index
self.doc_type = doc_type
self.model_class = model_class
self.es = Elasticsearch()
self.mapper = ObjectMapper()
def find_one(self, pk):
source_dict = self.es.get(index=self.index, doc_type=self.doc_type, id=pk)
return self.mapper.from_dict_to_model(source_dict, self.model_class)
def save(self, model):
model_dict = self.mapper.from_model_to_dict(model)
res = self.es.index(index=self.index, doc_type=self.doc_type, id=model.get_identity(), body=model_dict)
return res['created']
def find_all(self):
res = self.es.search(index=self.index, doc_type=self.doc_type, body={"query": {"match_all": {}}})
return [self.mapper.from_dict_to_model(model, self.model_class) for model in res['hits']['hits']]
def update(self, model):
model_dict = self.mapper.from_model_to_dict(model)
res = self.es.update(index=self.index, doc_type=self.doc_type, id=model.pk, body={"doc": model_dict})
return res
def delete(self, pk):
return self.es.delete(index=self.index, doc_type=self.doc_type, id=pk)
class ESIndex:
def __init__(self, hosts, index = "", doc_type = ""):
self.es = Elasticsearch(hosts)
self.index = index
self.doc_type = doc_type
def index(self, doc_id, body, index = "", doc_type = ""):
index_ = self.index if index == "" else index
doc_type_ = self.doc_type if doc_type == "" else doc_type
return self.es.index(index=index_, doc_type=doc_type_, body=body, id=doc_id)
def delete(self,doc_id, index = "", doc_type = ""):
index_ = self.index if index == "" else index
doc_type_ = self.doc_type if doc_type == "" else doc_type
return self.es.delete(index=index_, doc_type = doc_type_, id = doc_id)
def bulk(self, docs, index = "", doc_type = "", op_type = 'index'):
'''
bulk sample:
{"_op_type":"index", _index" : "test", "_type" : "type1", "_id" : "1" , "_source":{"field1":"value1", "field2":"value2"}}
{ "_op_type":"delete" , "_index" : "test", "_type" : "type1", "_id" : "2" }
'''
index_ = self.index if index == "" else index
doc_type_ = self.doc_type if doc_type == "" else doc_type
allow_op = ['index', 'delete']
if op_type not in allow_op:
raise exceptions.RequestError(400, '{"msg":"op_type is not allowed, you can use index or delete"}')
actions = []
for doc in docs:
action = {}
action["_index"] = index_
action["_type"] = doc_type_
action["_id"] = doc["_id"]
if op_type == 'index':
del doc["_id"]
action["_source"] = doc
action["_op_type"] = op_type
actions.append(action)
return helpers.parallel_bulk(self.es, actions)
def getDoc(self,doc_id, index = "", doc_type = ""):
index_ = self.index if index == "" else index
doc_type_ = self.doc_type if doc_type == "" else doc_type
return self.es.get(index=index_, doc_type=doc_type_, id=doc_id)
def putMapping(self, body, index = "", doc_type =""):
index_ = self.index if index == "" else index
doc_type_ = self.doc_type if doc_type == "" else doc_type
return self.es.indices.put_mapping(index=index_, doc_type=doc_type_, body=body)
def create(self, body = {}, index = "", timeout = 30):
index_ = self.index if index == "" else index
return self.es.indices.create(index_, body=body)
class ProjectDB(BaseProjectDB):
__type__ = 'project'
def __init__(self, hosts, index='pyspider'):
self.index = index
self.es = Elasticsearch(hosts=hosts)
self.es.indices.create(index=self.index, ignore=400)
if not self.es.indices.get_mapping(index=self.index, doc_type=self.__type__):
self.es.indices.put_mapping(index=self.index, doc_type=self.__type__, body={
"_all": {"enabled": False},
"properties": {
"updatetime": {"type": "double"}
}
})
def insert(self, name, obj={}):
obj = dict(obj)
obj['name'] = name
obj['updatetime'] = time.time()
obj.setdefault('group', '')
obj.setdefault('status', 'TODO')
obj.setdefault('script', '')
obj.setdefault('comments', '')
obj.setdefault('rate', 0)
obj.setdefault('burst', 0)
return self.es.index(index=self.index, doc_type=self.__type__, body=obj, id=name,
refresh=True)
def update(self, name, obj={}, **kwargs):
obj = dict(obj)
obj.update(kwargs)
obj['updatetime'] = time.time()
return self.es.update(index=self.index, doc_type=self.__type__,
body={'doc': obj}, id=name, refresh=True, ignore=404)
def get_all(self, fields=None):
for record in elasticsearch.helpers.scan(self.es, index=self.index, doc_type=self.__type__,
query={'query': {"match_all": {}}},
_source_include=fields or []):
yield record['_source']
def get(self, name, fields=None):
ret = self.es.get(index=self.index, doc_type=self.__type__, id=name,
_source_include=fields or [], ignore=404)
return ret.get('_source', None)
def check_update(self, timestamp, fields=None):
for record in elasticsearch.helpers.scan(self.es, index=self.index, doc_type=self.__type__,
query={'query': {"range": {
"updatetime": {"gte": timestamp}
}}}, _source_include=fields or []):
yield record['_source']
def drop(self, name):
return self.es.delete(index=self.index, doc_type=self.__type__, id=name, refresh=True)
class DataCatalogModel(object):
"""
Base for the application's model classes.
"""
def __init__(self):
self._config = DCConfig()
self._log = logging.getLogger(type(self).__name__)
self._elastic_search = Elasticsearch(
'{}:{}'.format(self._config.elastic.elastic_hostname,
self._config.elastic.elastic_port))
def _get_entry(self, entry_id):
"""
shortcut to ElasticSearch.get function
Standard elastic (index/doc_type) params are added
:param entry_id: elastic search id
:raises NotFoundError: entry not found in Elastic Search
:raises ConnectionError: problem with connecting to Elastic Search
:return: elastic search structure
"""
return self._elastic_search.get(
index=self._config.elastic.elastic_index,
doc_type=self._config.elastic.elastic_metadata_type,
id=entry_id)
def _delete_entry(self, entry_id):
"""
shortcut to ElasticSearch.delete function
data flush is performed after delete
Standard elastic (index/doc_type) params are added
:param entry_id: elastic search id
:raises NotFoundError: entry not found in Elastic Search
:raises ConnectionError: problem with connecting to Elastic Search
:rtype: None
"""
self._elastic_search.delete(
index=self._config.elastic.elastic_index,
doc_type=self._config.elastic.elastic_metadata_type,
id=entry_id)
# flushing data - so immediate searches are aware of change
self._elastic_search.indices.flush()
def delete_river_config(river, name, host="localhost", port=9200):
"""
Removes the given Elasticsearch River configuration from Elasticsearch.
The given river type and name are formatted like "$river-$name" to form the type name.
Arguments
---------
river: string
Name of the River plugin to use.
name: string
Name of this instance of the River plugin.
host: string
Elasticsearch host to connect to, defaults to localhost.
port: int
Elasticsearch port to connect to, defaults to 9200.
"""
elastic = Elasticsearch(hosts=[{ "host": host, "port": port }])
elastic.delete(index="_river", doc_type="{0}-{1}".format(river, name), id=None)
def fixsmallblogs ():
client = Elasticsearch([Util.config['eshost']])
response = client.search(
index="blogs",
body={
"size": 400,
"query" : {
"range": {
"length": {
"lte": 50
}
}
}
}
)
for hit in response['hits']['hits']:
print hit['_id']
client.delete('blogs','blog', hit['_id'])
class Es(object):
'''just es '''
def __init__(self, es_index=None, es_type=None):
'''es'''
hosts = HOSTS
if es_index:
self.es_index = es_index
else:
self.es_index = ES_INDEX
if es_type:
self.es_type = es_type
else:
self.es_type = ES_TYPE;
self.es_client = Elasticsearch( hosts, cluster=ES_CLUSTER )
#print self.es_client.info()
def index( self , d_id, doc):
'''索引信息
id : userid
doc : {}
'''
res = self.es_client.index(index=self.es_index, doc_type=self.es_type, id=d_id, body=doc)
def search( self, query):
'''q'''
response = self.es_client.search( body=query)
print response['hits']
def delete( self, doc_id ):
'''delete'''
self.es_client.delete(index=self.es_index, doc_type=self.es_type, id=doc_id )
def upsert( self, doc_id, doc_update, doc_index, script=None ):
'''添加更新'''
try:
self.es_client.update( index=self.es_index, doc_type=self.es_type, id=doc_id, body=doc_update, script=script, lang='mvel' )
except Exception, e:
print e
self.index( doc_id, doc_index )
def app_embedded(app_id):
try:
embedded_info = ApplicationEmbedded.objects(application_id=request.app.id).get()
except ApplicationEmbedded.DoesNotExist:
embedded_info = None
if request.method == 'POST':
use_embedded = request.form.get('use') is not None
embedded_iframe_uri = request.form.get('uri')
campus_ids = request.form.getlist('campus')
from elasticsearch import Elasticsearch
es = Elasticsearch(hosts=app.config.get('ELASTICSEARCH_HOSTS'))
if not use_embedded and embedded_info:
es.delete(index='embedded_app_list', doc_type='embedded_app', id=str(embedded_info.id))
embedded_info.delete()
embedded_info = None
elif use_embedded:
if not embedded_info:
embedded_info = ApplicationEmbedded()
embedded_info.application_id = request.app.id
embedded_info.iframe_uri = embedded_iframe_uri
embedded_info.campus_ids = campus_ids
embedded_info.save()
embedded_info = ApplicationEmbedded.objects(application_id=request.app.id).get()
es.index(index='embedded_app_list', doc_type='embedded_app', id=str(embedded_info.id), body={
'app_id': str(request.app.id),
'name': request.app.name,
'description': request.app.description,
'campus_ids': campus_ids
})
return render_template('developer/console/app/embedded.html',
campuses=Campus.objects(), use_embedded=embedded_info is not None,
embedded_iframe_uri=embedded_info.iframe_uri if embedded_info else '',
use_campus_ids=embedded_info.campus_ids if embedded_info else [])
class ElasticSearchHandler:
def __init__(self, host=None):
if not host:
host = os.getenv("ES_HOST")
self.es_handle = Elasticsearch(hosts=host)
def check_index(self, index):
return self.es_handle.indices.exists(index)
def create_index(self, index):
if not self.check_index(index):
self.es_handle.indices.create(index)
def add_to_es(self, index, document_type, body):
return self.es_handle.create(index=index, doc_type=document_type, body=body)
def get_from_es(self, index, id):
return self.es_handle.get(index, id=id)
def delete_by_id(self, index, document_type, id):
self.es_handle.delete(index, document_type, id)
def main():
es = Elasticsearch()
# query =
dashboards = {}
for hit in es.search(index="kibana-int", doc_type="dashboard", size=1000)["hits"]["hits"]:
data = hit["_source"]
dashboards[hit["_id"]] = hit["_source"]
for id_, data in dashboards.items():
dashboard = json.loads(data["dashboard"])
# Here the modification takes place
if dashboard["index"]["pattern"] == ORIG_INDEX_PATTERN:
dashboard["index"]["pattern"] = NEW_INDEX_PATTERN
dashboards[id_]["dashboard"] = json.dumps(dashboard)
for id_, data in dashboards.items():
es.delete(index="kibana-int", doc_type="dashboard", id=id_)
es.create(index="kibana-int", doc_type="dashboard", id=id_, body=data)
def update_video_index(sender, instance=None, created=False, **kwargs):
list_of_models = ('ChapterPods', 'EnrichPods', 'ContributorPods', 'Pod')
if sender.__name__ in list_of_models: # this is the dynamic part you want
pod = None
if sender.__name__ == "Pod":
pod = instance
else:
pod = instance.video
es = Elasticsearch(ES_URL)
if pod.is_draft == False and pod.encodingpods_set.all().count() > 0:
res = es.index(index="pod", doc_type='pod', id=pod.id, body=pod.get_json_to_index(), refresh=True)
else:
delete = es.delete(index="pod", doc_type='pod', id=pod.id, refresh=True, ignore=[400, 404])
def get_whois_domain(domain,refresh=None):
es = Elasticsearch()
id_num = domain
does_exist = es.exists(index='domain', doc_type='domain', id = domain)
print does_exist
if does_exist is True and refresh is None:
status = 200
print "Found it!"
get_record = es.get(index='domain',doc_type='domain', id = domain)
results = jsonify(get_record['_source'])
elif does_exist is True and refresh is not None:
status = 200
print "Forcing refresh!"
es.delete(index='domain', doc_type='domain', id = domain)
try:
obj = whois.whois(domain)
status = 200
results = jsonify(obj)
es.index(index='domain', doc_type='domain', id=domain, body=obj)
except Exception as e:
print e
results_raw = jsonify({'status': "not_found"})
status = 404
else:
try:
obj = whois.whois(domain)
status = 200
results = jsonify(obj)
es.index(index='domain', doc_type='domain', id=domain, body=obj)
except Exception as e:
print e
results_raw = jsonify({'status': "not_found"})
status = 404
results = jsonify({'status': "not_found"})
return results,status
def reindex_language():
rd = redis.StrictRedis(host='localhost', port=6379, db=0)
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
body = {'query': {'match_all': {}}}
result = es.search(index='throwtable', doc_type='implementation',
body=body, size=34589)
for impl in result['hits']['hits']:
language = impl['_source']['language']
if rd.hexists('rosetta-language-mapping', language):
print impl['_id'],
language = rd.hget('rosetta-language-mapping', language)
print '->', language
print 'new id = ', replace_lang_from_id(impl['_id'],
language.decode('utf8'))
source = impl['_source']
source['language'] = language.decode('utf8')
es.index(index='throwtable', doc_type='implementation',
id=replace_lang_from_id(impl['_id'], language.decode('utf8')),
body=source)
es.delete(index='throwtable', doc_type='implementation',
id=impl['_id'])
def removealltravelblogblogs ():
client = Elasticsearch([Util.config['eshost']])
response = client.search(
index="blogs",
body={
"size": 400,
"query": {
"prefix": {
"url.rawurl": {
"value": "http://www.travelblog.org"
}
}
}
}
)
for hit in response['hits']['hits']:
print hit['_id']
Util.deletefromazure(hit['_id'])
client.delete('blogs','blog', hit['_id'])
class ElasticsearchBackend(BaseBackend):
def __init__(self, conn):
if Elasticsearch is None:
raise ImportError("Plz. install elasticsearch library for ElasticsearchBackend.")
self._es = Elasticsearch(**conn.options.get('elasticsearch', {}))
super().__init__(conn)
def _gen_es_id_for_data(self, schema_name, _data):
key_names = self._conn.schema.get_primary_key(schema_name)
return '_n_'.join([str(_data[key]) for key in key_names])
def _gen_es_id_for_id(self, _id):
if isinstance(_id, str):
return _id
elif isinstance(_id, (tuple, list)):
return '_n_'.join(_id)
else:
return _id
def put_item(self, schema_name, _data, overwrite=False):
op_type = 'create' if not overwrite else 'index'
result = self._es.index(index=schema_name, doc_type=schema_name, id=self._gen_es_id_for_data(schema_name, _data), body=_data, op_type=op_type)
return result.get('_version', 0) > 0
def get_item(self, schema_name, _id):
try:
result = self._es.get(index=schema_name, doc_type=schema_name, id=self._gen_es_id_for_id(_id))
except NotFoundError:
raise ItemNotFound("Item not found for id {} in {}.".format(_id, schema_name))
return result['_source']
def delete_item(self, schema_name, _id):
result = self._es.delete(index=schema_name, doc_type=schema_name, id=self._gen_es_id_for_id(_id))
return result['found'] is True
def query(self, schema_name, _w, limit=10):
return self.scan(schema_name, _w, limit)
def scan(self, schema_name, _w, limit=10):
query = elastic_parse_wt(_w, {})
query["size"] = limit
result = self._es.search(index=schema_name, doc_type=schema_name, body=query)
return [hit['_source'] for hit in result["hits"]["hits"]]
def query_count(self, schema_name, _w):
query = elastic_parse_wt(_w, {})
result = self._es.count(index=schema_name, doc_type=schema_name, body=query)
return result.get('count', 0)
def correct_false_warning(judge_image_dir):
es = Elasticsearch(esport)
if 'file_name' in request.args:
md5_file_name = hashlib.md5(request.args['file_name']).hexdigest()
print md5_file_name + ' for ' + request.args['file_name']
else:
del(es)
return 'Error: no file name in request\n'
judge_image_dir = 'judgeresult:' + judge_image_dir
try:
res = es.delete(index = judge_image_dir, doc_type = 'judgeResult', id = md5_file_name)
except:
del(es)
return 'Error: file do not exist\n'
del(es)
return json.dumps(res['_shards'])
