Exemple #1
0
def watch_transactions():
    sleep(5)
    connection = r.connect(
            host=app.config['DB_HOST'],
            port=app.config['DB_PORT'],
            db=app.config['DB_NAME'])

    # FIXME(TF): Use changes and join from rethinkdb ....
    feed = r.table("transactions").changes().run(connection)
    # feed = r.table("transactions").changes().eq_join(
            # "product_id", r.table("products")).zip().eq_join(
                    # "supplier_id", r.table("suppliers")).zip().run(connection)
    print("Started live feed for new transactions ...")
    for transaction in feed:
        try:
            full_transaction_query = r.table("transactions").filter(
                    {"id": transaction["new_val"]["id"]}).eq_join(
                            "supplier_id", r.table("suppliers")).zip()
            if "product_id" in transaction["new_val"]:
                full_transaction_query = full_transaction_query.eq_join(
                        "product_id", r.table("products")).zip()

            full_transaction = full_transaction_query.run(connection).next()

            full_transaction["timestamp"] = maya.MayaDT.from_datetime(
                    full_transaction["timestamp"]).rfc2822()

            # geocode location
            if full_transaction["location"]:
                full_transaction["geocode"] = geocode(full_transaction["location"])

            socketio.emit("new_transaction", full_transaction)
        except Exception as exc:
            print(f"Error during transaction send {exc}")
Exemple #2
0
def get_places():
	"""Extracts entities from text and returns lists and links to wikipedia"""

	text = file_reader.read_file(session['filepath'])
	target_lang = session['target_lang']
	source_lang = (session['source_lang']).encode('ascii', 'replace')


	#Uses the NER tagger to get entities
	if source_lang == 'de':
		nouns = german.pos(text)
		organizations, locations, people = german.ner(text)
	elif source_lang == 'es':
		nouns = spanish.pos(text)
		organizations, locations, people = spanish.ner(text)
	else:
		nouns = text_processing.nouns_only(text_processing.preprocess(text))
		organizations, locations, people = text_processing.ner_tagger(text)


	#Checks the type of entity that is being requested
	ent = request.form['ent']
	
	if ent == "places":

		if locations: 
			loclist = wikipedia.get_entity_info(locations, target_lang, source_lang)
			downfile = file_reader.write_csv_file(loclist)
			geocodes = geocoding.geocode(locations)
			return render_template("places.html", locations = loclist, geocodes = json.dumps(geocodes), downfile=downfile)
		else: 
			return render_template("places.html")

	elif ent == "organizations":

		if organizations:
			orglist = wikipedia.get_entity_info(organizations, target_lang, source_lang)
			downfile = file_reader.write_csv_file(orglist)
			return render_template("orgs.html", organizations = orglist, downfile=downfile)
		else:
			return render_template("orgs.html")

	elif ent == "people":

		if people:
			peoplelist = wikipedia.get_entity_info(people, target_lang, source_lang)
			downfile = file_reader.write_csv_file(peoplelist)
			return render_template("people.html", people = peoplelist, downfile=downfile)
		else:
			return render_template("people.html")

	elif ent == "nouns":
			if nouns: 	
				nounlist = wikipedia.get_entity_info(nouns, target_lang, source_lang)
				downfile = file_reader.write_csv_file(nounlist)
				return render_template("other.html", nouns = nounlist, downfile=downfile)
			else:
				return render_template("other.html")
Exemple #3
0
def encode(data):
    """
channelGrouping,date,device,fullVisitorId,
geoNetwork,sessionId,socialEngagementType,totals,
trafficSource,visitId,visitNumber,visitStartTime

    """
    visitor_id = data['fullVisitorId']
    date = parse_date(data['date'])

    totals = json.loads(data['totals'])
    transaction = 0
    if 'transactionRevenue' in totals:
        transaction = float(totals['transactionRevenue']) / 1e6
    if 'pageviews' not in totals:
        totals['pageviews'] = 0

    device = json.loads(data['device'])

    # TODO: center and rescale each part independently

    features = [
        # Time features
        *one_hot_encoding(date.tm_mon, MON),
        *one_hot_encoding(date.tm_mday, MDAY),
        *one_hot_encoding(date.tm_wday, WDAY),
        *one_hot_encoding(
            int(convert_posix(data['visitStartTime'], date) /
                (24 * 60 * 60)), HOUR),

        # totals features
        int(totals['visits']),
        int(totals['hits']),
        int(totals['pageviews']),

        # device features
        *one_hot_encoding(device['browser'], DEVICE_BROWSERS),
        *one_hot_encoding(device['operatingSystem'], DEVICE_OS),
        *one_hot_encoding(device['deviceCategory'], DEVICE_CATEGORIES),

        # channel grouping
        *one_hot_encoding(data['channelGrouping'], CHANNEL_GROUPING),

        # visit number
        data['visitNumber'],

        # Geo features
        *GEOCODER[geocode(data['geoNetwork'])]

        # Traffic source
    ]

    features = np.array(features, dtype='float32')

    return visitor_id, transaction, features
Exemple #4
0
def get_geo(address, db):
    '''Get the geocode of an address, caching to the database'''
    if not address:
        return None
    if not _geo_cache.has_key(address):
        cached = db.get('geocodes', address) if db else False
        if not cached:
            cached = geocoding.geocode(address)
            if not cached:
                cached = {}
            if db:
                db.save('geocodes', address, cached)
        _geo_cache[address] = cached
    return _geo_cache[address]
Exemple #5
0
def get_geo(address, db):
    '''Get the geocode of an address, caching to the database'''
    if not address:
        return None
    if not _geo_cache.has_key(address):
        cached = db.get('geocodes', address) if db else False
        if not cached:
            cached = geocoding.geocode(address)
            if not cached:
                cached = {}
            if db:
                db.save('geocodes', address, cached)
        _geo_cache[address] = cached
    return _geo_cache[address]
Exemple #6
0
def import_kitchens(csv_file):
    reader = csv.DictReader(csv_file.splitlines())
    items = list(reader)

    models.Kitchen.objects.all().delete()
    for item in items:
        if not item['Kitchen ID']:
            continue
        k = models.Kitchen()
        k.kitchen_id = item['Kitchen ID']
        k.address = item['Address']
        k.postcode = item['Postcode']
        k.capacity = int(item['Capacity'])
        k.lat, k.lng = geocode(address=k.address) or (None, None)
        k.save()
        
        time.sleep(0.3)
Exemple #7
0
def import_recipients(csv_file):
    reader = csv.DictReader(csv_file.splitlines())
    items = list(reader)
    models.Recipient.objects.all().delete()
    
    for k, g in itertools.groupby(items, lambda item: item['Booking ID']):
        group = list(g)
        if not k:
            raise Exception('Booking ID contains empty value')
        recipient = models.Recipient()
        recipient.nickname = group[0]['Nickname'].decode('latin1')
        recipient.postcode = group[0]['Primary Postal Code']       	
        recipient.booking_id = k
        if recipient.postcode:
            recipient.lat, recipient.lng = geocode(recipient.postcode) or (None, None)
        recipient.save()
        for meal in group:
            recipient.meal_set.create(meal_type=meal['Meal Type'], comment=meal['Comments'])
        
        time.sleep(0.3)
Exemple #8
0
# Description: This script calls scripts that standardizes and geocodes a list
#  of addresses from an Excel spreadsheet

#==============================================================================

import openpyxl
import imperfect_addresses
import geocoding

print("\t\tLoading Spreadsheet...")

standardize_addresses = True
print("\t\t\nSPREADSHEET LOADED")
geocode = raw_input("\n\tWould you like to geocode the "
                    "spreadsheet?\n(Y/N)\n")

if geocode == "Y" or geocode == "y":
    geocode_spreadsheet = True

else:
    geocode_spreadsheet = False
    print("\nExiting...")

if standardize_addresses:
    imperfect_addresses.transform()

if geocode_spreadsheet:
    print("\nGeocoding addresses...")
    geocoding.geocode()
    print("\nGEOCODING SUCCESSFUL")
Exemple #9
0
def predict(destination, origin):
    if request.method == 'POST':
        try:
            import geocoding
            import bing_maps_routes
            import bing_maps_distance_matrix
            import bing_maps_elevations
            import bing_maps_traffic_speed_flow
            import get_weather
            import get_heading
            import get_wind_type
            import get_drag
            import get_rolling_friction
            import get_speedLimits
            import get_acceleration
            import Azure_final

          
            #--------------------------------------------------------------------------------------------------
            #Module 1 - Geocoding Module

            # destGeocoded = geocoding.geocode('Fishaways - Sasol Summerstrand')
            destGeocoded = geocoding.geocode(destination)
            #--------------------------------------------------------------------------------------------------
            #Module 2 - Get Coordinates Module - coordinates for route to destination
            # this module can return two lists
            # coordinatesList provides more coordinates on route
            # maneuverPointList retruns list of each segment - use this list to reduce computational overhead
            #  coordinatesResult, maneuverPointListResult = bing_maps_routes.get_coordinates(destGeocoded) to get both lists

            # maneuverPointListResult = bing_maps_routes.get_coordinates( '-34.002053,25.670521', destGeocoded)
            maneuverPointListResult, durationList = bing_maps_routes.get_coordinates(origin,destGeocoded)

            print('printing maneuverPoints List coordinates------------')
            print(maneuverPointListResult)

            print('printing durations------------')
            print(durationList)

            # #--------------------------------------------------------------------------------------------------
            #Module 3 - Get Distances each segment on route to destination
            # Split list Coordinates in equal chunks smaller than 50

            distancesList = bing_maps_distance_matrix.split_coordinates(maneuverPointListResult)
            print('printing List of Distances------------')
            print(distancesList)
            print('total distance:',sum(distancesList))

            # #--------------------------------------------------------------------------------------------------
            # #Module 4 - Get elevations for each coordinate
            elevationList = bing_maps_elevations.get_elevations(maneuverPointListResult)
            print('printing elevations------------')
            print(elevationList)

            # #--------------------------------------------------------------------------------------------------
            # #Module 5 - Get Speeds
            # #Get speeds for list
            speedList = bing_maps_traffic_speed_flow.get_speeds_from_maps(maneuverPointListResult)
            print('Printing speed list from front----------')
            print(speedList)
            #
            # #--------------------------------------------------------------------------------------------------
            # #Module 6 - Get weather
            listwindSpeed, listwindBearing, listtemperature, listpressure = get_weather.get_weather(maneuverPointListResult)
            print('Weahter-----------')
            print("WindSpeed")
            print(listwindSpeed)
            print("Wind Bearing")
            print(listwindBearing)
            print("temperature")
            print(listtemperature)
            print('pressure')
            print(listpressure)
            # #--------------------------------------------------------------------------------------------------
            # #Module 7 - Get bearing
            bearingList = get_heading.prepare_heading_list(maneuverPointListResult)
            print('bearing')
            print(bearingList)

            ##Module 8 - Get windType
            crossWindList, tailWindList, headWindList = get_wind_type.prepare_wind_type(bearingList, listwindSpeed, listwindBearing)
            print("crosswind")
            print(crossWindList)
            print("tailwind")
            print(tailWindList)
            print("headwind")
            print(headWindList)

            # #--------------------------------------------------------------------------------------------------
            # #Module 9 - Get Drag
            dragList = get_drag.prepare_drag(listtemperature, listpressure, speedList)
            print("dragList")
            print(dragList)
            # #--------------------------------------------------------------------------------------------------
            # #Module 10 - Get Rolling Friction and calculate angle
            # return negative and positive elevation

            rollingfList, elvDiffList_Pos, elvDiffList_Neg, angleList = get_rolling_friction.get_elv_diff(elevationList, distancesList)
            print("rolling friction list")
            print(rollingfList)
            print("ELevation pos")
            print(elvDiffList_Pos)
            print("ELevation neg")
            print(elvDiffList_Neg)
            print("AngleList")
            print(angleList)

            #-----------------------------------------------------------------------------------------------
            # #Module 11 - Acceleration
            accList = get_acceleration.get_acceleration(speedList, durationList)
            print("accList")
            print(accList)

            #-----------------------------------------------------------------------------------------------
            # #Module 12 - SpeedLimits
            speedLimitList = get_speedLimits.prepare_speedLimits(maneuverPointListResult)
            print(speedLimitList)

            #-----------------------------------------------------------------------------------------------
            # #Module 13 -Print list lengths


            distancesListF = []
            speedList2 = []
            crossWindListM = []
            tailWindListM = []
            headWindListM = []

            speedList.pop(0)



            for x in range(len(speedList)):   #need to only get speeds not coordinates
                speed = speedList[x][1]
                speedF = speed * 0.277778
                speedList2.append(speedF)

            for x in range(len(distancesList)):
                distance = distancesList[x]
                distanceM = distance * 1000
                distancesListF.append(distanceM)

            for x in range(len(crossWindList)):
                crosswind = crossWindList[x]
                crossWindM = crosswind / 0.514444 #convert to meter per second
                crossWindListM.append(crossWindM)

            for x in range(len(tailWindList)):
                tailwind = tailWindList[x]
                tailWindM = tailwind / 0.514444 #convert to meter per second
                tailWindListM.append(tailWindM)

            for x in range(len(headWindList)):
                headwind = headWindList[x]
                headWindM = headwind / 0.514444 #convert to meter per second
                headWindListM.append(headWindM)

            speedLimitList.pop(0)
            dragList.pop(0)
            crossWindListM.pop(0)
            tailWindList.pop(0)
            headWindList.pop(0)
            listpressure.pop(0)
            listtemperature.pop(0)

            speedListF = speedList2
            speedLimitListF = speedLimitList
            dragListF = dragList
            crossWindListF  = crossWindListM
            headWindListF = headWindListM
            tailWindListF  = tailWindListM
            listpressureF = listpressure
            listtemperatureF = listtemperature

            #send to azure
            # speedLimitListF = speedLimitList.pop(0)
            # dragListF = dragList.pop(0)
            # crossWindListF = crossWindList.pop(0)
            # headWindListF = tailWindList.pop(0)
            # tailWindListF = headWindList.pop(0)
            # listpressureF = listpressure.pop(0)
            # listtemperatureF = listtemperature.pop(0)

            print('distance', len(distancesListF))
            print('acceleration',len(accList))
            print('speedF', len(speedListF))
            print('speedLimit',len(speedLimitListF))
            print('drag',len(dragListF))
            print('rolling',len(rollingfList))
            print('elv diff positive',len(elvDiffList_Pos))
            print('elv diff neg', len(elvDiffList_Neg))
            print('cross', len(crossWindListF))
            print('tail',len(tailWindListF))
            print('headwind',len(headWindListF))
            print('pressure',len(listpressureF))
            print('temp',len(listtemperatureF))
            print('angle', len(angleList))

        except ValueError:
            return jsonify("error")

        newlist = [distancesListF,
           speedListF,
           accList,
           listtemperatureF,
           listpressureF,
           speedLimitListF,
           crossWindListF,
           tailWindListF,
           headWindListF,
           dragListF,
           rollingfList,
           angleList,
           elvDiffList_Pos,
           elvDiffList_Neg]

        send_to_azure = [list(t) for t in zip(*newlist)]

        print(send_to_azure)

        from time import gmtime, strftime
        time = strftime("%Y-%m-%d %H:%M:%S", gmtime())

        print("CURRENT TIME:",time)
        Azure_final.calculate_EC(send_to_azure)

        return json.dumps({
            'distance': distancesList,
            'acceleration': accList,
            'speedF':speedListF,
            'speedLimit':speedLimitListF,
            'drag':dragListF,
            'rolling':rollingfList,
            'elv diff positive':elvDiffList_Pos,
            'elv diff neg':elvDiffList_Neg,
            'cross':crossWindListF,
            'tail':tailWindListF,
            'headwind':headWindListF,
            'pressure':listpressureF,
            'temp':listtemperatureF,
            'angle':angleList})

            # "coordinates" : maneuverPointListResult,
            #                "distances" : distancesList})
        return('success')
 def geocode(self, address):
   """synchronous geocoding"""
   return geocoding.geocode(address)
Exemple #11
0
#!/usr/bin/env python

import fsapi
from geocoding import geocode

fsapi.get_pedigree()

data = []

for key, value in fsapi.people.iteritems():
    place = value.oneplace()
    if place:
        geoplace = geocode(place['place'])
        data.append([value.name, geoplace])

import json
fn = open('oneplace.json', 'w')
fn.write(json.dumps(data))
fn.close()

# vim: et sw=4 sts=4