def analyse(request, name):
del keys[:]
es = ElasticSearch(settings.ELASTIC_SEARCH)
segment = Segments.objects.get(es_index=name)
mapping = es.get_mapping(index=segment.es_index, doc_type=segment.name)
new_mapping = {}
for i in mapping:
new_mapping['map'] = mapping[i]['mappings'][segment.name]['properties']
for key in new_mapping['map']:
keys.append(key)
return render(request, 'analyse_segment.html', {
'segment': segment,
'columns': keys
})
def cli(index_name, doc_type, file_name, size):
"""
Export data from ElasticSearch to CSV file.
\b
Help:
python es2csv.py --help
\b
Example:
python es2csv.py --index-name=index_name --doc-type=typename
--file-name=/tmp/save_file.csv
"""
es = ElasticSearch(ES_CONF['host'])
mapping = es.get_mapping(index=index_name, doc_type=doc_type)
fieldnames = mapping[index_name]['mappings'][doc_type]['properties'].keys()
print "Fields Total: %d" % len(fieldnames)
writer = csv.writer(file(file_name, 'wb'), quoting=csv.QUOTE_NONNUMERIC)
writer.writerow(fieldnames)
print fieldnames
data = es.search("*", index=index_name, doc_type=doc_type, size=1)
total = data['hits']['total']
print "Total: %d" % total
size = 1000
for es_from in range(0, total + 1, size):
data = es.search("*",
index=index_name,
doc_type=doc_type,
es_from=es_from,
size=size)
data = data['hits']['hits']
format_data = []
for row in data:
for k in fieldnames:
if k not in row['_source']:
row['_source'][k] = ''
format_data.append([row['_source'][k] for k in fieldnames])
writer.writerows(format_data)
print "Saved count %d" % (es_from + size)
print 'ok'
def cli(index_name, doc_type, file_name, size):
"""
Export data from ElasticSearch to CSV file.
\b
Help:
python es2csv.py --help
\b
Example:
python es2csv.py --index-name=index_name --doc-type=typename
--file-name=/tmp/save_file.csv
"""
es = ElasticSearch(ES_CONF['host'])
mapping = es.get_mapping(index=index_name, doc_type=doc_type)
fieldnames = mapping[index_name]['mappings'][doc_type]['properties'].keys()
print "Fields Total: %d" % len(fieldnames)
writer = csv.writer(file(file_name, 'wb'), quoting=csv.QUOTE_NONNUMERIC)
writer.writerow(fieldnames)
print fieldnames
data = es.search("*", index=index_name, doc_type=doc_type, size=1)
total = data['hits']['total']
print "Total: %d" % total
size = 1000
for es_from in range(0, total+1, size):
data = es.search("*", index=index_name, doc_type=doc_type,
es_from=es_from, size=size)
data = data['hits']['hits']
format_data = []
for row in data:
for k in fieldnames:
if k not in row['_source']:
row['_source'][k] = ''
format_data.append([row['_source'][k] for k in fieldnames])
writer.writerows(format_data)
print "Saved count %d" % (es_from + size)
print 'ok'
# "index": "analyzed",
# "type": "integer"
# },
# "suggest" : {
# "type" : "completion",
# "analyzer" : "simple", 这里指定了分词方式
# "search_analyzer" : "simple",
# "payloads" : true
# }
# }
# }
# }
# }'
settings = es.get_settings('test')
mappings = es.get_mapping(index='test', doc_type='test')
print('settings:', settings)
print('mappings:', mappings)
docs = [{
'id': 1,
'name': '李干',
'age': 32,
'title': '抽象tv Coder'
}, {
'id': 2,
'name': 'Jessica Coder',
'age': 31,
'title': 'Programmer'
}, {
input = len(sys.argv)
if input < 2:
usage()
sys.exit(1)
else:
qname = sys.argv[1]
from pyelasticsearch import ElasticSearch
es = ElasticSearch(elasticsearch)
try:
s = es.status('oplog')
except:
print "Creating index: oplog"
try:
s = es.create_index('oplog')
print "sleeping for 5 to ensure index exists"
time.sleep(5)
except:
print "ERROR: index creation failed!"
sys.exit()
print "Creating queue: %s" % qname
try:
es.put_mapping('oplog',qname,{"properties" : { "from" : {"type" : "string", "null_value" : "na"}, "sent" : {"type" : "string", "null_value" : "na"}, "submitted" : {"type" : "date"}, "subject" : {"type" : "string", "null_value" : "na"}, "message" : {"type" : "string", "null_value" : "na"} }})
print "Created queue with mapping:"
print es.get_mapping('oplog',qname)
except:
print "ERROR: queue creation failed!"
