def watch_transactions():
sleep(5)
connection = r.connect(
host=app.config['DB_HOST'],
port=app.config['DB_PORT'],
db=app.config['DB_NAME'])
# FIXME(TF): Use changes and join from rethinkdb ....
feed = r.table("transactions").changes().run(connection)
# feed = r.table("transactions").changes().eq_join(
# "product_id", r.table("products")).zip().eq_join(
# "supplier_id", r.table("suppliers")).zip().run(connection)
print("Started live feed for new transactions ...")
for transaction in feed:
try:
full_transaction_query = r.table("transactions").filter(
{"id": transaction["new_val"]["id"]}).eq_join(
"supplier_id", r.table("suppliers")).zip()
if "product_id" in transaction["new_val"]:
full_transaction_query = full_transaction_query.eq_join(
"product_id", r.table("products")).zip()
full_transaction = full_transaction_query.run(connection).next()
full_transaction["timestamp"] = maya.MayaDT.from_datetime(
full_transaction["timestamp"]).rfc2822()
# geocode location
if full_transaction["location"]:
full_transaction["geocode"] = geocode(full_transaction["location"])
socketio.emit("new_transaction", full_transaction)
except Exception as exc:
print(f"Error during transaction send {exc}")
def get_places():
"""Extracts entities from text and returns lists and links to wikipedia"""
text = file_reader.read_file(session['filepath'])
target_lang = session['target_lang']
source_lang = (session['source_lang']).encode('ascii', 'replace')
#Uses the NER tagger to get entities
if source_lang == 'de':
nouns = german.pos(text)
organizations, locations, people = german.ner(text)
elif source_lang == 'es':
nouns = spanish.pos(text)
organizations, locations, people = spanish.ner(text)
else:
nouns = text_processing.nouns_only(text_processing.preprocess(text))
organizations, locations, people = text_processing.ner_tagger(text)
#Checks the type of entity that is being requested
ent = request.form['ent']
if ent == "places":
if locations:
loclist = wikipedia.get_entity_info(locations, target_lang, source_lang)
downfile = file_reader.write_csv_file(loclist)
geocodes = geocoding.geocode(locations)
return render_template("places.html", locations = loclist, geocodes = json.dumps(geocodes), downfile=downfile)
else:
return render_template("places.html")
elif ent == "organizations":
if organizations:
orglist = wikipedia.get_entity_info(organizations, target_lang, source_lang)
downfile = file_reader.write_csv_file(orglist)
return render_template("orgs.html", organizations = orglist, downfile=downfile)
else:
return render_template("orgs.html")
elif ent == "people":
if people:
peoplelist = wikipedia.get_entity_info(people, target_lang, source_lang)
downfile = file_reader.write_csv_file(peoplelist)
return render_template("people.html", people = peoplelist, downfile=downfile)
else:
return render_template("people.html")
elif ent == "nouns":
if nouns:
nounlist = wikipedia.get_entity_info(nouns, target_lang, source_lang)
downfile = file_reader.write_csv_file(nounlist)
return render_template("other.html", nouns = nounlist, downfile=downfile)
else:
return render_template("other.html")
def encode(data):
"""
channelGrouping,date,device,fullVisitorId,
geoNetwork,sessionId,socialEngagementType,totals,
trafficSource,visitId,visitNumber,visitStartTime
"""
visitor_id = data['fullVisitorId']
date = parse_date(data['date'])
totals = json.loads(data['totals'])
transaction = 0
if 'transactionRevenue' in totals:
transaction = float(totals['transactionRevenue']) / 1e6
if 'pageviews' not in totals:
totals['pageviews'] = 0
device = json.loads(data['device'])
# TODO: center and rescale each part independently
features = [
# Time features
*one_hot_encoding(date.tm_mon, MON),
*one_hot_encoding(date.tm_mday, MDAY),
*one_hot_encoding(date.tm_wday, WDAY),
*one_hot_encoding(
int(convert_posix(data['visitStartTime'], date) /
(24 * 60 * 60)), HOUR),
# totals features
int(totals['visits']),
int(totals['hits']),
int(totals['pageviews']),
# device features
*one_hot_encoding(device['browser'], DEVICE_BROWSERS),
*one_hot_encoding(device['operatingSystem'], DEVICE_OS),
*one_hot_encoding(device['deviceCategory'], DEVICE_CATEGORIES),
# channel grouping
*one_hot_encoding(data['channelGrouping'], CHANNEL_GROUPING),
# visit number
data['visitNumber'],
# Geo features
*GEOCODER[geocode(data['geoNetwork'])]
# Traffic source
]
features = np.array(features, dtype='float32')
return visitor_id, transaction, features
def get_geo(address, db):
'''Get the geocode of an address, caching to the database'''
if not address:
return None
if not _geo_cache.has_key(address):
cached = db.get('geocodes', address) if db else False
if not cached:
cached = geocoding.geocode(address)
if not cached:
cached = {}
if db:
db.save('geocodes', address, cached)
_geo_cache[address] = cached
return _geo_cache[address]
def get_geo(address, db):
'''Get the geocode of an address, caching to the database'''
if not address:
return None
if not _geo_cache.has_key(address):
cached = db.get('geocodes', address) if db else False
if not cached:
cached = geocoding.geocode(address)
if not cached:
cached = {}
if db:
db.save('geocodes', address, cached)
_geo_cache[address] = cached
return _geo_cache[address]
def import_kitchens(csv_file):
reader = csv.DictReader(csv_file.splitlines())
items = list(reader)
models.Kitchen.objects.all().delete()
for item in items:
if not item['Kitchen ID']:
continue
k = models.Kitchen()
k.kitchen_id = item['Kitchen ID']
k.address = item['Address']
k.postcode = item['Postcode']
k.capacity = int(item['Capacity'])
k.lat, k.lng = geocode(address=k.address) or (None, None)
k.save()
time.sleep(0.3)
def import_recipients(csv_file):
reader = csv.DictReader(csv_file.splitlines())
items = list(reader)
models.Recipient.objects.all().delete()
for k, g in itertools.groupby(items, lambda item: item['Booking ID']):
group = list(g)
if not k:
raise Exception('Booking ID contains empty value')
recipient = models.Recipient()
recipient.nickname = group[0]['Nickname'].decode('latin1')
recipient.postcode = group[0]['Primary Postal Code']
recipient.booking_id = k
if recipient.postcode:
recipient.lat, recipient.lng = geocode(recipient.postcode) or (None, None)
recipient.save()
for meal in group:
recipient.meal_set.create(meal_type=meal['Meal Type'], comment=meal['Comments'])
time.sleep(0.3)
# Description: This script calls scripts that standardizes and geocodes a list
# of addresses from an Excel spreadsheet
#==============================================================================
import openpyxl
import imperfect_addresses
import geocoding
print("\t\tLoading Spreadsheet...")
standardize_addresses = True
print("\t\t\nSPREADSHEET LOADED")
geocode = raw_input("\n\tWould you like to geocode the "
"spreadsheet?\n(Y/N)\n")
if geocode == "Y" or geocode == "y":
geocode_spreadsheet = True
else:
geocode_spreadsheet = False
print("\nExiting...")
if standardize_addresses:
imperfect_addresses.transform()
if geocode_spreadsheet:
print("\nGeocoding addresses...")
geocoding.geocode()
print("\nGEOCODING SUCCESSFUL")
def predict(destination, origin):
if request.method == 'POST':
try:
import geocoding
import bing_maps_routes
import bing_maps_distance_matrix
import bing_maps_elevations
import bing_maps_traffic_speed_flow
import get_weather
import get_heading
import get_wind_type
import get_drag
import get_rolling_friction
import get_speedLimits
import get_acceleration
import Azure_final
#--------------------------------------------------------------------------------------------------
#Module 1 - Geocoding Module
# destGeocoded = geocoding.geocode('Fishaways - Sasol Summerstrand')
destGeocoded = geocoding.geocode(destination)
#--------------------------------------------------------------------------------------------------
#Module 2 - Get Coordinates Module - coordinates for route to destination
# this module can return two lists
# coordinatesList provides more coordinates on route
# maneuverPointList retruns list of each segment - use this list to reduce computational overhead
# coordinatesResult, maneuverPointListResult = bing_maps_routes.get_coordinates(destGeocoded) to get both lists
# maneuverPointListResult = bing_maps_routes.get_coordinates( '-34.002053,25.670521', destGeocoded)
maneuverPointListResult, durationList = bing_maps_routes.get_coordinates(origin,destGeocoded)
print('printing maneuverPoints List coordinates------------')
print(maneuverPointListResult)
print('printing durations------------')
print(durationList)
# #--------------------------------------------------------------------------------------------------
#Module 3 - Get Distances each segment on route to destination
# Split list Coordinates in equal chunks smaller than 50
distancesList = bing_maps_distance_matrix.split_coordinates(maneuverPointListResult)
print('printing List of Distances------------')
print(distancesList)
print('total distance:',sum(distancesList))
# #--------------------------------------------------------------------------------------------------
# #Module 4 - Get elevations for each coordinate
elevationList = bing_maps_elevations.get_elevations(maneuverPointListResult)
print('printing elevations------------')
print(elevationList)
# #--------------------------------------------------------------------------------------------------
# #Module 5 - Get Speeds
# #Get speeds for list
speedList = bing_maps_traffic_speed_flow.get_speeds_from_maps(maneuverPointListResult)
print('Printing speed list from front----------')
print(speedList)
#
# #--------------------------------------------------------------------------------------------------
# #Module 6 - Get weather
listwindSpeed, listwindBearing, listtemperature, listpressure = get_weather.get_weather(maneuverPointListResult)
print('Weahter-----------')
print("WindSpeed")
print(listwindSpeed)
print("Wind Bearing")
print(listwindBearing)
print("temperature")
print(listtemperature)
print('pressure')
print(listpressure)
# #--------------------------------------------------------------------------------------------------
# #Module 7 - Get bearing
bearingList = get_heading.prepare_heading_list(maneuverPointListResult)
print('bearing')
print(bearingList)
##Module 8 - Get windType
crossWindList, tailWindList, headWindList = get_wind_type.prepare_wind_type(bearingList, listwindSpeed, listwindBearing)
print("crosswind")
print(crossWindList)
print("tailwind")
print(tailWindList)
print("headwind")
print(headWindList)
# #--------------------------------------------------------------------------------------------------
# #Module 9 - Get Drag
dragList = get_drag.prepare_drag(listtemperature, listpressure, speedList)
print("dragList")
print(dragList)
# #--------------------------------------------------------------------------------------------------
# #Module 10 - Get Rolling Friction and calculate angle
# return negative and positive elevation
rollingfList, elvDiffList_Pos, elvDiffList_Neg, angleList = get_rolling_friction.get_elv_diff(elevationList, distancesList)
print("rolling friction list")
print(rollingfList)
print("ELevation pos")
print(elvDiffList_Pos)
print("ELevation neg")
print(elvDiffList_Neg)
print("AngleList")
print(angleList)
#-----------------------------------------------------------------------------------------------
# #Module 11 - Acceleration
accList = get_acceleration.get_acceleration(speedList, durationList)
print("accList")
print(accList)
#-----------------------------------------------------------------------------------------------
# #Module 12 - SpeedLimits
speedLimitList = get_speedLimits.prepare_speedLimits(maneuverPointListResult)
print(speedLimitList)
#-----------------------------------------------------------------------------------------------
# #Module 13 -Print list lengths
distancesListF = []
speedList2 = []
crossWindListM = []
tailWindListM = []
headWindListM = []
speedList.pop(0)
for x in range(len(speedList)): #need to only get speeds not coordinates
speed = speedList[x][1]
speedF = speed * 0.277778
speedList2.append(speedF)
for x in range(len(distancesList)):
distance = distancesList[x]
distanceM = distance * 1000
distancesListF.append(distanceM)
for x in range(len(crossWindList)):
crosswind = crossWindList[x]
crossWindM = crosswind / 0.514444 #convert to meter per second
crossWindListM.append(crossWindM)
for x in range(len(tailWindList)):
tailwind = tailWindList[x]
tailWindM = tailwind / 0.514444 #convert to meter per second
tailWindListM.append(tailWindM)
for x in range(len(headWindList)):
headwind = headWindList[x]
headWindM = headwind / 0.514444 #convert to meter per second
headWindListM.append(headWindM)
speedLimitList.pop(0)
dragList.pop(0)
crossWindListM.pop(0)
tailWindList.pop(0)
headWindList.pop(0)
listpressure.pop(0)
listtemperature.pop(0)
speedListF = speedList2
speedLimitListF = speedLimitList
dragListF = dragList
crossWindListF = crossWindListM
headWindListF = headWindListM
tailWindListF = tailWindListM
listpressureF = listpressure
listtemperatureF = listtemperature
#send to azure
# speedLimitListF = speedLimitList.pop(0)
# dragListF = dragList.pop(0)
# crossWindListF = crossWindList.pop(0)
# headWindListF = tailWindList.pop(0)
# tailWindListF = headWindList.pop(0)
# listpressureF = listpressure.pop(0)
# listtemperatureF = listtemperature.pop(0)
print('distance', len(distancesListF))
print('acceleration',len(accList))
print('speedF', len(speedListF))
print('speedLimit',len(speedLimitListF))
print('drag',len(dragListF))
print('rolling',len(rollingfList))
print('elv diff positive',len(elvDiffList_Pos))
print('elv diff neg', len(elvDiffList_Neg))
print('cross', len(crossWindListF))
print('tail',len(tailWindListF))
print('headwind',len(headWindListF))
print('pressure',len(listpressureF))
print('temp',len(listtemperatureF))
print('angle', len(angleList))
except ValueError:
return jsonify("error")
newlist = [distancesListF,
speedListF,
accList,
listtemperatureF,
listpressureF,
speedLimitListF,
crossWindListF,
tailWindListF,
headWindListF,
dragListF,
rollingfList,
angleList,
elvDiffList_Pos,
elvDiffList_Neg]
send_to_azure = [list(t) for t in zip(*newlist)]
print(send_to_azure)
from time import gmtime, strftime
time = strftime("%Y-%m-%d %H:%M:%S", gmtime())
print("CURRENT TIME:",time)
Azure_final.calculate_EC(send_to_azure)
return json.dumps({
'distance': distancesList,
'acceleration': accList,
'speedF':speedListF,
'speedLimit':speedLimitListF,
'drag':dragListF,
'rolling':rollingfList,
'elv diff positive':elvDiffList_Pos,
'elv diff neg':elvDiffList_Neg,
'cross':crossWindListF,
'tail':tailWindListF,
'headwind':headWindListF,
'pressure':listpressureF,
'temp':listtemperatureF,
'angle':angleList})
# "coordinates" : maneuverPointListResult,
# "distances" : distancesList})
return('success')
def geocode(self, address): """synchronous geocoding""" return geocoding.geocode(address)
#!/usr/bin/env python
import fsapi
from geocoding import geocode
fsapi.get_pedigree()
data = []
for key, value in fsapi.people.iteritems():
place = value.oneplace()
if place:
geoplace = geocode(place['place'])
data.append([value.name, geoplace])
import json
fn = open('oneplace.json', 'w')
fn.write(json.dumps(data))
fn.close()
# vim: et sw=4 sts=4