def analyse(request, name): del keys[:] es = ElasticSearch(settings.ELASTIC_SEARCH) segment = Segments.objects.get(es_index=name) mapping = es.get_mapping(index=segment.es_index, doc_type=segment.name) new_mapping = {} for i in mapping: new_mapping['map'] = mapping[i]['mappings'][segment.name]['properties'] for key in new_mapping['map']: keys.append(key) return render(request, 'analyse_segment.html', { 'segment': segment, 'columns': keys })
def cli(index_name, doc_type, file_name, size): """ Export data from ElasticSearch to CSV file. \b Help: python es2csv.py --help \b Example: python es2csv.py --index-name=index_name --doc-type=typename --file-name=/tmp/save_file.csv """ es = ElasticSearch(ES_CONF['host']) mapping = es.get_mapping(index=index_name, doc_type=doc_type) fieldnames = mapping[index_name]['mappings'][doc_type]['properties'].keys() print "Fields Total: %d" % len(fieldnames) writer = csv.writer(file(file_name, 'wb'), quoting=csv.QUOTE_NONNUMERIC) writer.writerow(fieldnames) print fieldnames data = es.search("*", index=index_name, doc_type=doc_type, size=1) total = data['hits']['total'] print "Total: %d" % total size = 1000 for es_from in range(0, total + 1, size): data = es.search("*", index=index_name, doc_type=doc_type, es_from=es_from, size=size) data = data['hits']['hits'] format_data = [] for row in data: for k in fieldnames: if k not in row['_source']: row['_source'][k] = '' format_data.append([row['_source'][k] for k in fieldnames]) writer.writerows(format_data) print "Saved count %d" % (es_from + size) print 'ok'
def cli(index_name, doc_type, file_name, size): """ Export data from ElasticSearch to CSV file. \b Help: python es2csv.py --help \b Example: python es2csv.py --index-name=index_name --doc-type=typename --file-name=/tmp/save_file.csv """ es = ElasticSearch(ES_CONF['host']) mapping = es.get_mapping(index=index_name, doc_type=doc_type) fieldnames = mapping[index_name]['mappings'][doc_type]['properties'].keys() print "Fields Total: %d" % len(fieldnames) writer = csv.writer(file(file_name, 'wb'), quoting=csv.QUOTE_NONNUMERIC) writer.writerow(fieldnames) print fieldnames data = es.search("*", index=index_name, doc_type=doc_type, size=1) total = data['hits']['total'] print "Total: %d" % total size = 1000 for es_from in range(0, total+1, size): data = es.search("*", index=index_name, doc_type=doc_type, es_from=es_from, size=size) data = data['hits']['hits'] format_data = [] for row in data: for k in fieldnames: if k not in row['_source']: row['_source'][k] = '' format_data.append([row['_source'][k] for k in fieldnames]) writer.writerows(format_data) print "Saved count %d" % (es_from + size) print 'ok'
# "index": "analyzed", # "type": "integer" # }, # "suggest" : { # "type" : "completion", # "analyzer" : "simple", 这里指定了分词方式 # "search_analyzer" : "simple", # "payloads" : true # } # } # } # } # }' settings = es.get_settings('test') mappings = es.get_mapping(index='test', doc_type='test') print('settings:', settings) print('mappings:', mappings) docs = [{ 'id': 1, 'name': '李干', 'age': 32, 'title': '抽象tv Coder' }, { 'id': 2, 'name': 'Jessica Coder', 'age': 31, 'title': 'Programmer' }, {
input = len(sys.argv) if input < 2: usage() sys.exit(1) else: qname = sys.argv[1] from pyelasticsearch import ElasticSearch es = ElasticSearch(elasticsearch) try: s = es.status('oplog') except: print "Creating index: oplog" try: s = es.create_index('oplog') print "sleeping for 5 to ensure index exists" time.sleep(5) except: print "ERROR: index creation failed!" sys.exit() print "Creating queue: %s" % qname try: es.put_mapping('oplog',qname,{"properties" : { "from" : {"type" : "string", "null_value" : "na"}, "sent" : {"type" : "string", "null_value" : "na"}, "submitted" : {"type" : "date"}, "subject" : {"type" : "string", "null_value" : "na"}, "message" : {"type" : "string", "null_value" : "na"} }}) print "Created queue with mapping:" print es.get_mapping('oplog',qname) except: print "ERROR: queue creation failed!"