forked from codeforamerica/streetscope
-
Notifications
You must be signed in to change notification settings - Fork 0
/
index_addresses.py
39 lines (34 loc) · 1.27 KB
/
index_addresses.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
import sys
import csv
import re
import os
from urlparse import urlparse
from elasticsearch import Elasticsearch
if os.environ.get('BONSAI_URL'):
url = urlparse(os.environ['BONSAI_URL'])
ELASTICSEARCH_HOST = url.hostname
ELASTICSEARCH_AUTH = url.username + ':' + url.password
es = Elasticsearch([{'host': ELASTICSEARCH_HOST}], http_auth=ELASTICSEARCH_AUTH)
else:
es = Elasticsearch()
files_given = sys.argv
for file_name in files_given:
if file_name == 'index_addresses.py':
continue
else:
file_path = file_name
print 'adding ' + file_path
with open(file_path, 'r') as csvfile:
print "open file"
csv_reader = csv.DictReader(csvfile, fieldnames=[], restkey='undefined-fieldnames', delimiter=',')
current_row = 0
for row in csv_reader:
current_row += 1
if current_row == 1:
csv_reader.fieldnames = row['undefined-fieldnames']
continue
address = row
if current_row % 1000 == 0:
print "%s addresses indexed" % current_row
es.index(index='addresses', doc_type='address', id=current_row-1, body={'NUMBER': address[' NUMBER'], 'STREET': address[' STREET'], 'ADDRESS': address[' NUMBER'] + ' ' + address[' STREET'], 'X': address['LON'], 'Y': address[' LAT']})
csvfile.close()