def post(self):
user = self.get_current_user()
trip = dict()
trip_fields = {
'start_date': 'start_date',
'end_date': 'end_date',
'description': 'description',
'place_id': 'place_id',
'address': 'formatted_address',
'locality': 'locality',
'region': 'administrative_area_level_1',
'county': 'administrative_area_level_2',
'longitude': 'lng',
'latitude': 'lat'
}
for key in trip_fields:
trip[key] = self.get_argument(trip_fields[key], None)
trip_uuid = simpleflake()
trip['trip_id'] = base62().hash(trip_uuid, 12)
trip['created_at'] = r.now()
trip['updated_at'] = r.now()
trip['creator_user_id'] = user['id']
trip['geo'] = r.point(float(trip['longitude']), float(trip['latitude']))
r.table("trip").insert(trip).run()
self.redirect("/")
def post(self):
user = self.get_current_user()
trip = dict()
trip_fields = {
'start_date': 'start_date',
'end_date': 'end_date',
'description': 'description',
'place_id': 'place_id',
'address': 'formatted_address',
'locality': 'locality',
'region': 'administrative_area_level_1',
'county': 'administrative_area_level_2',
'longitude': 'lng',
'latitude': 'lat'
}
for key in trip_fields:
trip[key] = self.get_argument(trip_fields[key], None)
trip_uuid = simpleflake()
trip['trip_id'] = base62().hash(trip_uuid, 12)
trip['created_at'] = r.now()
trip['updated_at'] = r.now()
trip['creator_user_id'] = user['id']
trip['geo'] = r.point(float(trip['longitude']),
float(trip['latitude']))
r.table("trip").insert(trip).run()
self.redirect("/")
def get(self, *args):
self.set_header("Content-Type", "text/event-stream")
north, south, east, west = map(float, args)
start_t = datetime.now()
query_range = r.polygon(r.point(west, north), r.point(west, south), r.point(east, south), r.point(east, north))
selection = r.table("streets").get_intersecting(query_range, index="geometry")
initial_t = (datetime.now() - start_t).total_seconds()
cursor = selection.map(r.row["geometry"].to_geojson()).run(self.conn)
size = 0
for chunk in chunked(cursor, 2000):
size += len(chunk)
self.write_event(chunk)
self.write_event("done")
total_t = (datetime.now() - start_t).total_seconds()
print "street query took", initial_t, "s for the first batch",
print "(", total_t, "s total) and provided", size, "results."
def get(self, *args):
self.set_header('Content-Type', 'text/event-stream')
north, south, east, west = map(float, args)
start_t = datetime.now()
query_range = r.polygon(r.point(west, north), r.point(west, south),
r.point(east, south), r.point(east, north))
selection = (r.table('streets').get_intersecting(query_range,
index='geometry'))
initial_t = (datetime.now() - start_t).total_seconds()
cursor = selection.map(r.row['geometry'].to_geojson()).run(self.conn)
size = 0
for chunk in chunked(cursor, 2000):
size += len(chunk)
self.write_event(chunk)
self.write_event('done')
total_t = (datetime.now() - start_t).total_seconds()
print 'street query took', initial_t, 's for the first batch',
print '(', total_t, 's total) and provided', size, 'results.'
def refresh_quakes():
conn = r.connect(host=RDB_HOST, port=RDB_PORT, db=RDB_DB)
r.table("quakes").insert(
r.http(url)["features"].merge({
"time": r.epoch_time(r.row["properties"]["time"] / 1000),
"geometry": r.point(
r.row["geometry"]["coordinates"][0],
r.row["geometry"]["coordinates"][1])}),
conflict="replace").run(conn)
conn.close()
def convert_single_doc(record, v36, v84, vgrid):
vlon36, vlat36 = vgrid(record[4], record[5], inverse=True)
longitude, latitude = transform(v36, v84, vlon36, vlat36)
return {
'tiploc': record[0],
'name': record[1],
'created_at': make_timestamp(record[2]),
'expires_at': make_timestamp(record[3]),
'location': r.point(longitude, latitude),
'timing_point_type': TIMING_POINT_TYPES[record[6]],
'zone': record[7],
'stanox': record[8],
'off_network': bool(record[9] == 'Y')
}
def get_stop_docs(stops, refs):
the_docs = []
for stop in stops:
ref = find(lambda doc: doc['AtcoCode'] == stop['AtcoCode'], refs)
the_doc = {
'AtcoCode': stop['AtcoCode'],
'TiplocCode': ref['TiplocCode'],
'CommonName': stop['CommonName'],
'StationName': ref['StationName'],
'LocalityName': stop['LocalityName'],
'CrsCode': ref['CrsCode'],
'Location': r.point(float(stop['Longitude']), float(stop['Latitude']))
}
the_docs.append(the_doc)
return the_docs
def get(self, *args):
self.set_header("Content-Type", "application/json")
center = r.point(*map(float, args))
selection = r.table("points_of_interest").get_nearest(center, index="geometry", max_results=20, unit="mi")
start_t = datetime.now()
array = selection.map(
lambda row: {
"type": "Feature",
"geometry": row["doc"]["geometry"].to_geojson(),
"properties": row["doc"]["properties"].merge({"distance": row["dist"]}),
}
).run(self.conn)
total_t = (datetime.now() - start_t).total_seconds()
print "POI query took", total_t, "s and provided", len(array), "results"
self.write(json.dumps(array))
def on_data(self, data):
data = json.loads(data)
time = datetime.strptime(data["created_at"],
"%a %b %d %H:%M:%S +0000 %Y")
time = pytz.utc.localize(time)
data["created_at"] = time
r.table('tweets_without_location').insert(data).run()
if data["coordinates"] != None:
if data["coordinates"]["coordinates"][0] != 2.3508 or data[
"coordinates"]["coordinates"][1] != 45.8567:
data["coordinates"] = r.point(
data["coordinates"]["coordinates"][0],
data["coordinates"]["coordinates"][1])
r.table('tweets').insert(data).run()
return (True)
def get_unit_location(unit_id):
url = UNIT_LOCATION.format(unit_id)
resp = requests.get(url)
if resp.status_code == 200:
try:
data = resp.json()
if 'BusLocationData' in data and data['BusLocationData'] is not None:
buslocation = data['BusLocationData']
return {
'date': r.now(),
'location': r.point(buslocation['Latitude'], buslocation['Longitude']),
'unit_id': buslocation['UnitID'],
'variant_id': buslocation['VariantId'],
}, unit_id
except:
print('Error fetching unit location')
return None, unit_id
def setup_shapes(conn, routes_data):
table_name = 'shapes'
_create_table_if_not_exists(conn, table_name)
r.table(table_name).index_create('location', r.row['location'], geo=True).run(conn)
all_shapes = []
for route in routes_data:
for direction in route['directions']:
stop_url = '{}/data/shapes_{}_{}.json'.format(BASE_URL, route['route_id'], direction['direction_id'])
shapes_data = requests.get(stop_url).json()
for shape in shapes_data:
# add location as rethink geo point
shape['location'] = r.point(shape['shape_pt_lon'], shape['shape_pt_lat'])
all_shapes.extend(shapes_data)
r.table(table_name).insert(all_shapes).run(conn)
def create_tables():
for table in ('pokemon', 'pokestops', 'gyms'):
if not r.table_list().contains(table).run():
r.table_create(table).run()
log.info('Created table "{}"'.format(table))
for index, table in [('disappear_time', 'pokemon')]:
if index not in r.table(table).index_list().run():
# Create index for some fields, this _should_ improve performance
# when filtering and sorting by this field.
r.table(table).index_create(index).run()
log.info('Created secondary index "{}" on table "{}"'.format(index, table))
if 'location' not in r.table('pokemon').index_list().run():
r.table('pokemon').index_create('location', geo=True).run()
log.info('Created geo index on table {}.'.format('pokemon'))
r.table('pokemon').update(lambda p: {'location': r.point(p['longitude'], p['latitude'])}).run()
r.table('pokemon').replace(r.row.without('longitude').without('latitude')).run()
log.info('Updated existing pokemon locations.')
def post(self):
user = self.get_current_user()
event = dict()
event_fields = [
'title', 'start_date', 'end_date', 'start_time', 'end_time',
'description', 'website'
]
venue = dict()
venue_fields = [
'foursquare_id', 'name', 'address', 'locality', 'region',
'postal_code', 'longitude', 'latitude'
]
for key in event_fields:
event[key] = self.get_argument(key, None)
for key in venue_fields:
venue[key] = self.get_argument(key, None)
event_uuid = simpleflake()
event['event_id'] = base62().hash(event_uuid, 12)
venue_uuid = simpleflake()
venue['venue_id'] = base62().hash(venue_uuid, 12)
event['venue_id'] = venue['venue_id']
event['created_at'] = r.now()
event['updated_at'] = r.now()
event['creator_user_id'] = user['id']
venue['geo'] = r.point(float(venue['longitude']),
float(venue['latitude']))
venue['created_at'] = r.now()
venue['updated_at'] = r.now()
venue['creator_user_id'] = user['id']
r.table("event").insert(event).run()
r.table("venue").insert(venue).run()
self.redirect("/")
def setup_stops(conn, routes_data):
table_name = 'stops'
_create_table_if_not_exists(conn, table_name)
r.table(table_name).index_create('location', r.row['location'], geo=True).run(conn)
all_stops = []
for route in routes_data:
for direction in route['directions']:
# route_direction = str(route['route_id']) + '_' + str(direction['direction_id'])
stop_url = '{}/data/stops_{}_{}.json'.format(BASE_URL, route['route_id'], direction['direction_id'])
stops_data = requests.get(stop_url).json()
for stop in stops_data:
# add location as rethink geo point
stop['location'] = r.point(stop['stop_lon'], stop['stop_lat'])
# too similar to location.type attr in rethinkdb
del stop['location_type']
all_stops.extend(stops_data)
r.table(table_name).insert(all_stops).run(conn)
def get(self, *args):
self.set_header('Content-Type', 'application/json')
center = r.point(*map(float, args))
selection = r.table('points_of_interest').get_nearest(center,
index='geometry',
max_results=20,
unit='mi')
start_t = datetime.now()
array = selection.map(
lambda row: {
'type':
'Feature',
'geometry':
row['doc']['geometry'].to_geojson(),
'properties':
row['doc']['properties'].merge({'distance': row['dist']}),
}).run(self.conn)
total_t = (datetime.now() - start_t).total_seconds()
print 'POI query took', total_t, 's and provided', len(
array), 'results'
self.write(json.dumps(array))
def post(self):
user = self.get_current_user()
event = dict()
event_fields = [
'title', 'start_date', 'end_date', 'start_time', 'end_time', 'description',
'website'
]
venue = dict()
venue_fields = [
'foursquare_id', 'name', 'address', 'locality', 'region', 'postal_code',
'longitude', 'latitude'
]
for key in event_fields:
event[key] = self.get_argument(key, None)
for key in venue_fields:
venue[key] = self.get_argument(key, None)
event_uuid = simpleflake()
event['event_id'] = base62().hash(event_uuid, 12)
venue_uuid = simpleflake()
venue['venue_id'] = base62().hash(venue_uuid, 12)
event['venue_id'] = venue['venue_id']
event['created_at'] = r.now()
event['updated_at'] = r.now()
event['creator_user_id'] = user['id']
venue['geo'] = r.point(float(venue['longitude']), float(venue['latitude']))
venue['created_at'] = r.now()
venue['updated_at'] = r.now()
venue['creator_user_id'] = user['id']
r.table("event").insert(event).run()
r.table("venue").insert(venue).run()
self.redirect("/")
def get_unit_location(unit_id):
url = UNIT_LOCATION.format(unit_id)
resp = requests.get(url)
if resp.status_code == 200:
try:
data = resp.json()
if 'BusLocationData' in data and data[
'BusLocationData'] is not None:
buslocation = data['BusLocationData']
return {
'date':
r.now(),
'location':
r.point(buslocation['Latitude'], buslocation['Longitude']),
'unit_id':
buslocation['UnitID'],
'variant_id':
buslocation['VariantId'],
}, unit_id
except:
print('Error fetching unit location')
return None, unit_id
def parse_map(map_dict):
pokemon_list = []
pokestops = []
gyms = []
cells = map_dict['responses']['GET_MAP_OBJECTS']['map_cells']
for cell in cells:
for p in cell.get('wild_pokemons', []):
if p['encounter_id'] in pokemon_list:
continue # prevent unnecessary parsing
disappear_time = utc_localize(datetime.utcfromtimestamp(
(p['last_modified_timestamp_ms'] +
p['time_till_hidden_ms']) / 1000.0))
pokemon_list.append({
'id': p['encounter_id'],
'spawnpoint_id': p['spawn_point_id'],
'pokemon_id': p['pokemon_data']['pokemon_id'],
'location': r.point(p['longitude'], p['latitude']),
'disappear_time': disappear_time
})
for p in cell.get('catchable_pokemons', []):
if p['encounter_id'] in map(lambda x: x['id'], pokemon_list):
continue # prevent unnecessary parsing
log.critical("found catchable pokemon not in wild: {}".format(p))
disappear_time = utc_localize(datetime.utcfromtimestamp(
(p['last_modified_timestamp_ms'] +
p['time_till_hidden_ms']) / 1000.0))
pokemon_list.append({
'id': p['encounter_id'],
'spawnpoint_id': p['spawnpoint_id'],
'pokemon_id': p['pokemon_data']['pokemon_id'],
'location': r.point(p['longitude'], p['latitude']),
'disappear_time': disappear_time
})
for f in cell.get('forts', []):
if f['id'] in gyms or f['id'] in pokestops:
continue # prevent unnecessary parsing
if f.get('type') == 1: # Pokestops
if 'lure_info' in f:
lure_expiration = utc_localize(datetime.utcfromtimestamp(
f['lure_info']['lure_expires_timestamp_ms'] / 1000.0))
active_pokemon_id = f['lure_info']['active_pokemon_id']
else:
lure_expiration, active_pokemon_id = None, None
last_modified = utc_localize(datetime.utcfromtimestamp(f['last_modified_timestamp_ms'] / 1000.0))
pokestops.append({
'id': f['id'],
'enabled': f['enabled'],
'location': r.point(f['longitude'], f['latitude']),
'last_modified': last_modified,
'lure_expiration': lure_expiration,
'active_pokemon_id': active_pokemon_id
})
else:
last_modified = utc_localize(datetime.utcfromtimestamp(f['last_modified_timestamp_ms'] / 1000.0))
gyms.append({
'id': f['id'],
'team_id': f.get('owned_by_team', 0),
'guard_pokemon_id': f.get('guard_pokemon_id', 0),
'gym_points': f.get('gym_points', 0),
'enabled': f['enabled'],
'location': r.point(f['longitude'], f['latitude']),
'last_modified': last_modified,
})
if pokemon_list and config['parse_pokemon']:
log.debug("Inserting {} pokemon".format(len(pokemon_list)))
r.table('pokemon').insert(pokemon_list, conflict='update').run()
if pokestops and config['parse_pokestops']:
log.debug("Inserting {} pokestops".format(len(pokestops)))
r.table('pokestops').insert(pokestops, conflict='update').run()
if gyms and config['parse_gyms']:
log.debug("Inserting {} gyms".format(len(gyms)))
r.table('gyms').insert(gyms, conflict='update').run()
def update_event(event, tmptweet):
# location
locationa = (
event["coordinates"]["coordinates"][0] * event["numberOfTweets"] +
tmptweet["coordinates"]["coordinates"][0]) / (event["numberOfTweets"] +
1)
locationb = (
event["coordinates"]["coordinates"][1] * event["numberOfTweets"] +
tmptweet["coordinates"]["coordinates"][1]) / (event["numberOfTweets"] +
1)
# number of tweets
numberoftweets = event["numberOfTweets"] + 1
# start date
if tmptweet["created_at"] > event["startDate"]:
startdate = event["startDate"]
else:
startdate = tmptweet["created_at"]
# hashtag list
hashtagproperties = event["hashtags"]
hashtagproperties = get_hashtag_properties(hashtagproperties, tmptweet)
# hastag stats
if hashtagproperties["numberOfHashtag"] > 0:
numberofappearenceh = 0
mostusedhashtag = ""
for i in hashtagproperties["hashtagList"]:
if hashtagproperties["hashtagList"][i] > numberofappearenceh:
mostusedhashtag = i
numberofappearenceh = hashtagproperties["hashtagList"][i]
hashtagproperties["mostUsedHashtag"]["text"] = mostusedhashtag
hashtagproperties["mostUsedHashtag"][
"numberOfAppearence"] = numberofappearenceh
hashtagproperties["mostUsedHashtag"][
"frequencyOnTweets"] = numberofappearenceh / numberoftweets
hashtagproperties["mostUsedHashtag"][
"frequencyOnHashtags"] = numberofappearenceh / hashtagproperties[
"numberOfHashtag"]
hashtagproperties[
"avgNumberOfHashtag"] = numberofappearenceh / numberoftweets
# mentions properties
mentionproperties = event["mentions"]
mentionproperties = get_mention_properties(mentionproperties, tmptweet)
# mentions stats
if mentionproperties["numberOfMention"] > 0:
numberofappearencem = 0
mostusedmention = ""
for i in mentionproperties["mentionList"]:
if mentionproperties["mentionList"][i] > numberofappearencem:
mostusedmention = i
numberofappearencem = mentionproperties["mentionList"][i]
mentionproperties["mostUsedMention"]["text"] = mostusedmention
mentionproperties["mostUsedMention"][
"numberOfAppearence"] = numberofappearencem
mentionproperties["mostUsedMention"][
"frequencyOnTweets"] = numberofappearencem / numberoftweets
mentionproperties["mostUsedMention"][
"frequencyOnMentions"] = numberofappearencem / mentionproperties[
"numberOfMention"]
mentionproperties[
"avgNumberOfMention"] = numberofappearencem / numberoftweets
# users properties
userproperties = event["users"]
userproperties = get_user_properties(userproperties, tmptweet)
maxtweetperuser = max(userproperties["userList"].values())
userproperties[
"tweetPerUserAvg"] = numberoftweets / userproperties["numberOfUser"]
userproperties["ratioMaxTweetPerUser"] = maxtweetperuser / numberoftweets
userproperties["maxTweetPerUser"] = maxtweetperuser
# tweet list
tweetslist = event["tweetList"]
tweetslist.append(tmptweet)
# scoring
scoring = get_scoring(tweetslist)
commontheme = scoring["commonTheme"]
tweetslist = scoring["tweetList"]
# insertion in the DB
r.table('events').insert([{
"numberOfTweets": numberoftweets,
"commontheme": commontheme,
"coordinates": r.point(locationa, locationb),
"startDate": startdate,
"hashtags": hashtagproperties,
"mentions": mentionproperties,
"users": userproperties,
"tweetList": tweetslist,
"wordsCount": scoring["wordsCount"],
"wordsList": scoring["wordsList"]
}]).run()
def tweets_list_to_event(tweetslist):
# common theme
scoring = get_scoring(tweetslist)
commontheme = scoring["commonTheme"]
tweetslist = scoring["tweetList"]
if commontheme > 0.18:
locationa = 0
locationb = 0
startdate = 0
hashtagproperties = {"hashtagList": {}, "mostUsedHashtag": {}}
mentionproperties = {"mentionList": {}, "mostUsedMention": {}}
userproperties = {
"userList": {},
}
numberoftweets = len(tweetslist)
for tmptweet in tweetslist:
# get event center location
locationa = locationa + tmptweet["coordinates"]["coordinates"][0]
locationb = locationb + tmptweet["coordinates"]["coordinates"][1]
# get event start date
if startdate == 0 or startdate > tmptweet["created_at"]:
startdate = tmptweet["created_at"]
# hashtag, mention and user basic properties
hashtagproperties = get_hashtag_properties(hashtagproperties,
tmptweet)
mentionproperties = get_mention_properties(mentionproperties,
tmptweet)
userproperties = get_user_properties(userproperties, tmptweet)
if userproperties["numberOfUser"] > 1:
# hastag stats
if hashtagproperties["numberOfHashtag"] > 0:
numberofappearenceh = 0
mostusedhashtag = ""
for i in hashtagproperties["hashtagList"]:
if hashtagproperties["hashtagList"][
i] > numberofappearenceh:
mostusedhashtag = i
numberofappearenceh = hashtagproperties["hashtagList"][
i]
hashtagproperties["mostUsedHashtag"]["text"] = mostusedhashtag
hashtagproperties["mostUsedHashtag"][
"numberOfAppearence"] = numberofappearenceh
hashtagproperties["mostUsedHashtag"][
"frequencyOnTweets"] = numberofappearenceh / numberoftweets
hashtagproperties["mostUsedHashtag"][
"frequencyOnHashtags"] = numberofappearenceh / hashtagproperties[
"numberOfHashtag"]
hashtagproperties[
"avgNumberOfHashtag"] = numberofappearenceh / numberoftweets
# mentions stats
if mentionproperties["numberOfMention"] > 0:
numberofappearencem = 0
mostusedmention = ""
for i in mentionproperties["mentionList"]:
if mentionproperties["mentionList"][
i] > numberofappearencem:
mostusedmention = i
numberofappearencem = mentionproperties["mentionList"][
i]
mentionproperties["mostUsedMention"]["text"] = mostusedmention
mentionproperties["mostUsedMention"][
"numberOfAppearence"] = numberofappearencem
mentionproperties["mostUsedMention"][
"frequencyOnTweets"] = numberofappearencem / numberoftweets
mentionproperties["mostUsedMention"][
"frequencyOnMentions"] = numberofappearencem / mentionproperties[
"numberOfMention"]
mentionproperties[
"avgNumberOfMention"] = numberofappearencem / numberoftweets
# users stats
maxtweetperuser = max(userproperties["userList"].values())
userproperties[
"tweetPerUserAvg"] = numberoftweets / userproperties[
"numberOfUser"]
userproperties[
"ratioMaxTweetPerUser"] = maxtweetperuser / numberoftweets
userproperties["maxTweetPerUser"] = maxtweetperuser
if userproperties["ratioMaxTweetPerUser"] < 0.7:
if verify_event(numberoftweets, commontheme, hashtagproperties,
mentionproperties, userproperties) == 1:
# insertion in the DB
r.table('events').insert([{
"numberOfTweets":
numberoftweets,
"commontheme":
commontheme,
"coordinates":
r.point(locationa / len(listTweets),
locationb / len(listTweets)),
"startDate":
startdate,
"hashtags":
hashtagproperties,
"mentions":
mentionproperties,
"users":
userproperties,
"tweetList":
tweetslist,
"wordsCount":
scoring["wordsCount"],
"wordsList":
scoring["wordsList"]
}]).run()
return True
else:
return False
def to_rethink(self, value):
if isinstance(value, (list, tuple)):
return r.point(float(value[0]), float(value[1]))
else:
return None
'id':
loc_id,
"name":
location['name'],
"type":
location['type'],
"brgyCount":
int(location['brgyCount']),
"area":
location['area'],
"psgc":
location['psgc'],
"formattedAddress":
location['formattedAddress'],
"position":
r.point(location['coordinates']['longitude'],
location['coordinates']['latitude']),
"province":
location['province'],
"region":
location['region'],
}).run(conn)
locations = get_all('locations')
for location in locations:
res = check_has_same_loc(location['name'])
update_article_field(location['id'],
'hasSameName',
res['has_same_name'],
tbl='locations')
def _set_location(self, row):
row['location'] = rethinkdb.point(row['lon'], row['lat'])
with open('active.json', 'r') as actives:
active_data = json.load(actives)
except Exception, e:
raise e
try:
conn = r.connect(host='localhost', port=28015)
conn.repl()
except:
print 'RethinkDB : Connection Problem'
sys.exit(1)
stats = []
index = 0
for place in active_data:
circle = r.circle([float(place['lng'].encode('ascii','ignore')), float(place['lat'].encode('ascii','ignore'))], 3169, unit='m')
response = r.db('test').table('warehouse').filter(lambda instance: instance['available'] == True).pluck('cab_lat', 'cab_lng').distinct().map(lambda instance: circle.includes(r.point(instance['cab_lng'].coerce_to('number'),instance['cab_lat'].coerce_to('number')))).run()
t = 0
for res in response:
if res:
t += 1
index += 1
stats.append({'lat': place['lat'].encode('ascii','ignore'), 'lng': place['lng'].encode('ascii','ignore'), 'count': t})
print stats
