def run():
for row in whitmanArray:
print whitmanArray.index(row)/float(len(whitmanArray))
words =[]
entry = False
try:
entry = NE_Record(str(row[4]),str(row[1]))
entry.url = str(row[0])
entry.properDate = str(row[2])
#date processing
dateCompnents = word_tokenize(entry.properDate)
entry.year = dateCompnents[len(dateCompnents) - 1]
entry.month = dateCompnents[0]
entry.normalizedTime = row[3]
words.extend(entry.gpe)
words.extend(entry.locations)
words.extend(entry.organizations)
except Exception as e:
pass
for word in words:
address = binarySearch(word)
if address:
newrow = []
newrow.append(word)
newrow.append(row[2])
newrow.append(row[3])
newrow.append(row[0])
newrow.append(address)
try:
geocode_url = "http://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=true®ion=us" % address
data = json.loads(requests.get(geocode_url).content)
lat = data['results'][0]['geometry']['location']['lat']
lng = data['results'][0]['geometry']['location']['lng']
newrow.append([lat,lng])
if entry:
entry.latlngs.append((lat,lng))
except:
newrow.append('n/a')
newrow.append(address)
fileToWrite.writerow(newrow)
foundCities.append(word)
newRow = row.append(foundCities)
dataWithCities.append(newRow)
if entry:
dataToPickle.append(entry)
pickle.dump(dataToPickle,open('../data/taggedData.txt','wb'))
saveToFileAsBSON(dataToPickle,'../data/whitmanbson.bson')
saveToFileAsJson(dataToPickle,'../data/whitmanjson.json',readable=False)
if 'walt' and 'whitman' in letter.recipient.lower():
incomingLetters += 1
incomingWords += len(letter.words)
except Exception:
pass
if count == len(data) - 1:
outgoingLetterTraffic.append(outgoingLetters)
outgoingWordTraffic.append(outgoingWords)
incomingLetterTraffic.append(incomingLetters)
incomingWordTraffic.append(incomingWords)
count += 1
class Datum:
def __init__(self):
self.outgoingLetterTraffic = outgoingLetterTraffic
self.outgoingWordTraffic = outgoingWordTraffic
self.incomingLetterTraffic = incomingLetterTraffic
self.incomingWordTraffic = incomingWordTraffic
self.locations = locations
self.normalizedDates = nrmDates
class Letter:
def __init__(self):
self.locations = locations
d = Datum()
jsonmaker.saveToFileAsBSON(d,'../data/postageTrafficByYear.bson')
jsonmaker.saveToFileAsJson(d,'../data/postageTrafficByYear.json',readable=True)
