def post(self): user = self.get_current_user() trip = dict() trip_fields = { 'start_date': 'start_date', 'end_date': 'end_date', 'description': 'description', 'place_id': 'place_id', 'address': 'formatted_address', 'locality': 'locality', 'region': 'administrative_area_level_1', 'county': 'administrative_area_level_2', 'longitude': 'lng', 'latitude': 'lat' } for key in trip_fields: trip[key] = self.get_argument(trip_fields[key], None) trip_uuid = simpleflake() trip['trip_id'] = base62().hash(trip_uuid, 12) trip['created_at'] = r.now() trip['updated_at'] = r.now() trip['creator_user_id'] = user['id'] trip['geo'] = r.point(float(trip['longitude']), float(trip['latitude'])) r.table("trip").insert(trip).run() self.redirect("/")
def get(self, *args): self.set_header("Content-Type", "text/event-stream") north, south, east, west = map(float, args) start_t = datetime.now() query_range = r.polygon(r.point(west, north), r.point(west, south), r.point(east, south), r.point(east, north)) selection = r.table("streets").get_intersecting(query_range, index="geometry") initial_t = (datetime.now() - start_t).total_seconds() cursor = selection.map(r.row["geometry"].to_geojson()).run(self.conn) size = 0 for chunk in chunked(cursor, 2000): size += len(chunk) self.write_event(chunk) self.write_event("done") total_t = (datetime.now() - start_t).total_seconds() print "street query took", initial_t, "s for the first batch", print "(", total_t, "s total) and provided", size, "results."
def get(self, *args): self.set_header('Content-Type', 'text/event-stream') north, south, east, west = map(float, args) start_t = datetime.now() query_range = r.polygon(r.point(west, north), r.point(west, south), r.point(east, south), r.point(east, north)) selection = (r.table('streets').get_intersecting(query_range, index='geometry')) initial_t = (datetime.now() - start_t).total_seconds() cursor = selection.map(r.row['geometry'].to_geojson()).run(self.conn) size = 0 for chunk in chunked(cursor, 2000): size += len(chunk) self.write_event(chunk) self.write_event('done') total_t = (datetime.now() - start_t).total_seconds() print 'street query took', initial_t, 's for the first batch', print '(', total_t, 's total) and provided', size, 'results.'
def refresh_quakes(): conn = r.connect(host=RDB_HOST, port=RDB_PORT, db=RDB_DB) r.table("quakes").insert( r.http(url)["features"].merge({ "time": r.epoch_time(r.row["properties"]["time"] / 1000), "geometry": r.point( r.row["geometry"]["coordinates"][0], r.row["geometry"]["coordinates"][1])}), conflict="replace").run(conn) conn.close()
def convert_single_doc(record, v36, v84, vgrid): vlon36, vlat36 = vgrid(record[4], record[5], inverse=True) longitude, latitude = transform(v36, v84, vlon36, vlat36) return { 'tiploc': record[0], 'name': record[1], 'created_at': make_timestamp(record[2]), 'expires_at': make_timestamp(record[3]), 'location': r.point(longitude, latitude), 'timing_point_type': TIMING_POINT_TYPES[record[6]], 'zone': record[7], 'stanox': record[8], 'off_network': bool(record[9] == 'Y') }
def get_stop_docs(stops, refs): the_docs = [] for stop in stops: ref = find(lambda doc: doc['AtcoCode'] == stop['AtcoCode'], refs) the_doc = { 'AtcoCode': stop['AtcoCode'], 'TiplocCode': ref['TiplocCode'], 'CommonName': stop['CommonName'], 'StationName': ref['StationName'], 'LocalityName': stop['LocalityName'], 'CrsCode': ref['CrsCode'], 'Location': r.point(float(stop['Longitude']), float(stop['Latitude'])) } the_docs.append(the_doc) return the_docs
def get(self, *args): self.set_header("Content-Type", "application/json") center = r.point(*map(float, args)) selection = r.table("points_of_interest").get_nearest(center, index="geometry", max_results=20, unit="mi") start_t = datetime.now() array = selection.map( lambda row: { "type": "Feature", "geometry": row["doc"]["geometry"].to_geojson(), "properties": row["doc"]["properties"].merge({"distance": row["dist"]}), } ).run(self.conn) total_t = (datetime.now() - start_t).total_seconds() print "POI query took", total_t, "s and provided", len(array), "results" self.write(json.dumps(array))
def on_data(self, data): data = json.loads(data) time = datetime.strptime(data["created_at"], "%a %b %d %H:%M:%S +0000 %Y") time = pytz.utc.localize(time) data["created_at"] = time r.table('tweets_without_location').insert(data).run() if data["coordinates"] != None: if data["coordinates"]["coordinates"][0] != 2.3508 or data[ "coordinates"]["coordinates"][1] != 45.8567: data["coordinates"] = r.point( data["coordinates"]["coordinates"][0], data["coordinates"]["coordinates"][1]) r.table('tweets').insert(data).run() return (True)
def get_unit_location(unit_id): url = UNIT_LOCATION.format(unit_id) resp = requests.get(url) if resp.status_code == 200: try: data = resp.json() if 'BusLocationData' in data and data['BusLocationData'] is not None: buslocation = data['BusLocationData'] return { 'date': r.now(), 'location': r.point(buslocation['Latitude'], buslocation['Longitude']), 'unit_id': buslocation['UnitID'], 'variant_id': buslocation['VariantId'], }, unit_id except: print('Error fetching unit location') return None, unit_id
def setup_shapes(conn, routes_data): table_name = 'shapes' _create_table_if_not_exists(conn, table_name) r.table(table_name).index_create('location', r.row['location'], geo=True).run(conn) all_shapes = [] for route in routes_data: for direction in route['directions']: stop_url = '{}/data/shapes_{}_{}.json'.format(BASE_URL, route['route_id'], direction['direction_id']) shapes_data = requests.get(stop_url).json() for shape in shapes_data: # add location as rethink geo point shape['location'] = r.point(shape['shape_pt_lon'], shape['shape_pt_lat']) all_shapes.extend(shapes_data) r.table(table_name).insert(all_shapes).run(conn)
def create_tables(): for table in ('pokemon', 'pokestops', 'gyms'): if not r.table_list().contains(table).run(): r.table_create(table).run() log.info('Created table "{}"'.format(table)) for index, table in [('disappear_time', 'pokemon')]: if index not in r.table(table).index_list().run(): # Create index for some fields, this _should_ improve performance # when filtering and sorting by this field. r.table(table).index_create(index).run() log.info('Created secondary index "{}" on table "{}"'.format(index, table)) if 'location' not in r.table('pokemon').index_list().run(): r.table('pokemon').index_create('location', geo=True).run() log.info('Created geo index on table {}.'.format('pokemon')) r.table('pokemon').update(lambda p: {'location': r.point(p['longitude'], p['latitude'])}).run() r.table('pokemon').replace(r.row.without('longitude').without('latitude')).run() log.info('Updated existing pokemon locations.')
def post(self): user = self.get_current_user() event = dict() event_fields = [ 'title', 'start_date', 'end_date', 'start_time', 'end_time', 'description', 'website' ] venue = dict() venue_fields = [ 'foursquare_id', 'name', 'address', 'locality', 'region', 'postal_code', 'longitude', 'latitude' ] for key in event_fields: event[key] = self.get_argument(key, None) for key in venue_fields: venue[key] = self.get_argument(key, None) event_uuid = simpleflake() event['event_id'] = base62().hash(event_uuid, 12) venue_uuid = simpleflake() venue['venue_id'] = base62().hash(venue_uuid, 12) event['venue_id'] = venue['venue_id'] event['created_at'] = r.now() event['updated_at'] = r.now() event['creator_user_id'] = user['id'] venue['geo'] = r.point(float(venue['longitude']), float(venue['latitude'])) venue['created_at'] = r.now() venue['updated_at'] = r.now() venue['creator_user_id'] = user['id'] r.table("event").insert(event).run() r.table("venue").insert(venue).run() self.redirect("/")
def setup_stops(conn, routes_data): table_name = 'stops' _create_table_if_not_exists(conn, table_name) r.table(table_name).index_create('location', r.row['location'], geo=True).run(conn) all_stops = [] for route in routes_data: for direction in route['directions']: # route_direction = str(route['route_id']) + '_' + str(direction['direction_id']) stop_url = '{}/data/stops_{}_{}.json'.format(BASE_URL, route['route_id'], direction['direction_id']) stops_data = requests.get(stop_url).json() for stop in stops_data: # add location as rethink geo point stop['location'] = r.point(stop['stop_lon'], stop['stop_lat']) # too similar to location.type attr in rethinkdb del stop['location_type'] all_stops.extend(stops_data) r.table(table_name).insert(all_stops).run(conn)
def get(self, *args): self.set_header('Content-Type', 'application/json') center = r.point(*map(float, args)) selection = r.table('points_of_interest').get_nearest(center, index='geometry', max_results=20, unit='mi') start_t = datetime.now() array = selection.map( lambda row: { 'type': 'Feature', 'geometry': row['doc']['geometry'].to_geojson(), 'properties': row['doc']['properties'].merge({'distance': row['dist']}), }).run(self.conn) total_t = (datetime.now() - start_t).total_seconds() print 'POI query took', total_t, 's and provided', len( array), 'results' self.write(json.dumps(array))
def get_unit_location(unit_id): url = UNIT_LOCATION.format(unit_id) resp = requests.get(url) if resp.status_code == 200: try: data = resp.json() if 'BusLocationData' in data and data[ 'BusLocationData'] is not None: buslocation = data['BusLocationData'] return { 'date': r.now(), 'location': r.point(buslocation['Latitude'], buslocation['Longitude']), 'unit_id': buslocation['UnitID'], 'variant_id': buslocation['VariantId'], }, unit_id except: print('Error fetching unit location') return None, unit_id
def parse_map(map_dict): pokemon_list = [] pokestops = [] gyms = [] cells = map_dict['responses']['GET_MAP_OBJECTS']['map_cells'] for cell in cells: for p in cell.get('wild_pokemons', []): if p['encounter_id'] in pokemon_list: continue # prevent unnecessary parsing disappear_time = utc_localize(datetime.utcfromtimestamp( (p['last_modified_timestamp_ms'] + p['time_till_hidden_ms']) / 1000.0)) pokemon_list.append({ 'id': p['encounter_id'], 'spawnpoint_id': p['spawn_point_id'], 'pokemon_id': p['pokemon_data']['pokemon_id'], 'location': r.point(p['longitude'], p['latitude']), 'disappear_time': disappear_time }) for p in cell.get('catchable_pokemons', []): if p['encounter_id'] in map(lambda x: x['id'], pokemon_list): continue # prevent unnecessary parsing log.critical("found catchable pokemon not in wild: {}".format(p)) disappear_time = utc_localize(datetime.utcfromtimestamp( (p['last_modified_timestamp_ms'] + p['time_till_hidden_ms']) / 1000.0)) pokemon_list.append({ 'id': p['encounter_id'], 'spawnpoint_id': p['spawnpoint_id'], 'pokemon_id': p['pokemon_data']['pokemon_id'], 'location': r.point(p['longitude'], p['latitude']), 'disappear_time': disappear_time }) for f in cell.get('forts', []): if f['id'] in gyms or f['id'] in pokestops: continue # prevent unnecessary parsing if f.get('type') == 1: # Pokestops if 'lure_info' in f: lure_expiration = utc_localize(datetime.utcfromtimestamp( f['lure_info']['lure_expires_timestamp_ms'] / 1000.0)) active_pokemon_id = f['lure_info']['active_pokemon_id'] else: lure_expiration, active_pokemon_id = None, None last_modified = utc_localize(datetime.utcfromtimestamp(f['last_modified_timestamp_ms'] / 1000.0)) pokestops.append({ 'id': f['id'], 'enabled': f['enabled'], 'location': r.point(f['longitude'], f['latitude']), 'last_modified': last_modified, 'lure_expiration': lure_expiration, 'active_pokemon_id': active_pokemon_id }) else: last_modified = utc_localize(datetime.utcfromtimestamp(f['last_modified_timestamp_ms'] / 1000.0)) gyms.append({ 'id': f['id'], 'team_id': f.get('owned_by_team', 0), 'guard_pokemon_id': f.get('guard_pokemon_id', 0), 'gym_points': f.get('gym_points', 0), 'enabled': f['enabled'], 'location': r.point(f['longitude'], f['latitude']), 'last_modified': last_modified, }) if pokemon_list and config['parse_pokemon']: log.debug("Inserting {} pokemon".format(len(pokemon_list))) r.table('pokemon').insert(pokemon_list, conflict='update').run() if pokestops and config['parse_pokestops']: log.debug("Inserting {} pokestops".format(len(pokestops))) r.table('pokestops').insert(pokestops, conflict='update').run() if gyms and config['parse_gyms']: log.debug("Inserting {} gyms".format(len(gyms))) r.table('gyms').insert(gyms, conflict='update').run()
def update_event(event, tmptweet): # location locationa = ( event["coordinates"]["coordinates"][0] * event["numberOfTweets"] + tmptweet["coordinates"]["coordinates"][0]) / (event["numberOfTweets"] + 1) locationb = ( event["coordinates"]["coordinates"][1] * event["numberOfTweets"] + tmptweet["coordinates"]["coordinates"][1]) / (event["numberOfTweets"] + 1) # number of tweets numberoftweets = event["numberOfTweets"] + 1 # start date if tmptweet["created_at"] > event["startDate"]: startdate = event["startDate"] else: startdate = tmptweet["created_at"] # hashtag list hashtagproperties = event["hashtags"] hashtagproperties = get_hashtag_properties(hashtagproperties, tmptweet) # hastag stats if hashtagproperties["numberOfHashtag"] > 0: numberofappearenceh = 0 mostusedhashtag = "" for i in hashtagproperties["hashtagList"]: if hashtagproperties["hashtagList"][i] > numberofappearenceh: mostusedhashtag = i numberofappearenceh = hashtagproperties["hashtagList"][i] hashtagproperties["mostUsedHashtag"]["text"] = mostusedhashtag hashtagproperties["mostUsedHashtag"][ "numberOfAppearence"] = numberofappearenceh hashtagproperties["mostUsedHashtag"][ "frequencyOnTweets"] = numberofappearenceh / numberoftweets hashtagproperties["mostUsedHashtag"][ "frequencyOnHashtags"] = numberofappearenceh / hashtagproperties[ "numberOfHashtag"] hashtagproperties[ "avgNumberOfHashtag"] = numberofappearenceh / numberoftweets # mentions properties mentionproperties = event["mentions"] mentionproperties = get_mention_properties(mentionproperties, tmptweet) # mentions stats if mentionproperties["numberOfMention"] > 0: numberofappearencem = 0 mostusedmention = "" for i in mentionproperties["mentionList"]: if mentionproperties["mentionList"][i] > numberofappearencem: mostusedmention = i numberofappearencem = mentionproperties["mentionList"][i] mentionproperties["mostUsedMention"]["text"] = mostusedmention mentionproperties["mostUsedMention"][ "numberOfAppearence"] = numberofappearencem mentionproperties["mostUsedMention"][ "frequencyOnTweets"] = numberofappearencem / numberoftweets mentionproperties["mostUsedMention"][ "frequencyOnMentions"] = numberofappearencem / mentionproperties[ "numberOfMention"] mentionproperties[ "avgNumberOfMention"] = numberofappearencem / numberoftweets # users properties userproperties = event["users"] userproperties = get_user_properties(userproperties, tmptweet) maxtweetperuser = max(userproperties["userList"].values()) userproperties[ "tweetPerUserAvg"] = numberoftweets / userproperties["numberOfUser"] userproperties["ratioMaxTweetPerUser"] = maxtweetperuser / numberoftweets userproperties["maxTweetPerUser"] = maxtweetperuser # tweet list tweetslist = event["tweetList"] tweetslist.append(tmptweet) # scoring scoring = get_scoring(tweetslist) commontheme = scoring["commonTheme"] tweetslist = scoring["tweetList"] # insertion in the DB r.table('events').insert([{ "numberOfTweets": numberoftweets, "commontheme": commontheme, "coordinates": r.point(locationa, locationb), "startDate": startdate, "hashtags": hashtagproperties, "mentions": mentionproperties, "users": userproperties, "tweetList": tweetslist, "wordsCount": scoring["wordsCount"], "wordsList": scoring["wordsList"] }]).run()
def tweets_list_to_event(tweetslist): # common theme scoring = get_scoring(tweetslist) commontheme = scoring["commonTheme"] tweetslist = scoring["tweetList"] if commontheme > 0.18: locationa = 0 locationb = 0 startdate = 0 hashtagproperties = {"hashtagList": {}, "mostUsedHashtag": {}} mentionproperties = {"mentionList": {}, "mostUsedMention": {}} userproperties = { "userList": {}, } numberoftweets = len(tweetslist) for tmptweet in tweetslist: # get event center location locationa = locationa + tmptweet["coordinates"]["coordinates"][0] locationb = locationb + tmptweet["coordinates"]["coordinates"][1] # get event start date if startdate == 0 or startdate > tmptweet["created_at"]: startdate = tmptweet["created_at"] # hashtag, mention and user basic properties hashtagproperties = get_hashtag_properties(hashtagproperties, tmptweet) mentionproperties = get_mention_properties(mentionproperties, tmptweet) userproperties = get_user_properties(userproperties, tmptweet) if userproperties["numberOfUser"] > 1: # hastag stats if hashtagproperties["numberOfHashtag"] > 0: numberofappearenceh = 0 mostusedhashtag = "" for i in hashtagproperties["hashtagList"]: if hashtagproperties["hashtagList"][ i] > numberofappearenceh: mostusedhashtag = i numberofappearenceh = hashtagproperties["hashtagList"][ i] hashtagproperties["mostUsedHashtag"]["text"] = mostusedhashtag hashtagproperties["mostUsedHashtag"][ "numberOfAppearence"] = numberofappearenceh hashtagproperties["mostUsedHashtag"][ "frequencyOnTweets"] = numberofappearenceh / numberoftweets hashtagproperties["mostUsedHashtag"][ "frequencyOnHashtags"] = numberofappearenceh / hashtagproperties[ "numberOfHashtag"] hashtagproperties[ "avgNumberOfHashtag"] = numberofappearenceh / numberoftweets # mentions stats if mentionproperties["numberOfMention"] > 0: numberofappearencem = 0 mostusedmention = "" for i in mentionproperties["mentionList"]: if mentionproperties["mentionList"][ i] > numberofappearencem: mostusedmention = i numberofappearencem = mentionproperties["mentionList"][ i] mentionproperties["mostUsedMention"]["text"] = mostusedmention mentionproperties["mostUsedMention"][ "numberOfAppearence"] = numberofappearencem mentionproperties["mostUsedMention"][ "frequencyOnTweets"] = numberofappearencem / numberoftweets mentionproperties["mostUsedMention"][ "frequencyOnMentions"] = numberofappearencem / mentionproperties[ "numberOfMention"] mentionproperties[ "avgNumberOfMention"] = numberofappearencem / numberoftweets # users stats maxtweetperuser = max(userproperties["userList"].values()) userproperties[ "tweetPerUserAvg"] = numberoftweets / userproperties[ "numberOfUser"] userproperties[ "ratioMaxTweetPerUser"] = maxtweetperuser / numberoftweets userproperties["maxTweetPerUser"] = maxtweetperuser if userproperties["ratioMaxTweetPerUser"] < 0.7: if verify_event(numberoftweets, commontheme, hashtagproperties, mentionproperties, userproperties) == 1: # insertion in the DB r.table('events').insert([{ "numberOfTweets": numberoftweets, "commontheme": commontheme, "coordinates": r.point(locationa / len(listTweets), locationb / len(listTweets)), "startDate": startdate, "hashtags": hashtagproperties, "mentions": mentionproperties, "users": userproperties, "tweetList": tweetslist, "wordsCount": scoring["wordsCount"], "wordsList": scoring["wordsList"] }]).run() return True else: return False
def to_rethink(self, value): if isinstance(value, (list, tuple)): return r.point(float(value[0]), float(value[1])) else: return None
'id': loc_id, "name": location['name'], "type": location['type'], "brgyCount": int(location['brgyCount']), "area": location['area'], "psgc": location['psgc'], "formattedAddress": location['formattedAddress'], "position": r.point(location['coordinates']['longitude'], location['coordinates']['latitude']), "province": location['province'], "region": location['region'], }).run(conn) locations = get_all('locations') for location in locations: res = check_has_same_loc(location['name']) update_article_field(location['id'], 'hasSameName', res['has_same_name'], tbl='locations')
def _set_location(self, row): row['location'] = rethinkdb.point(row['lon'], row['lat'])
with open('active.json', 'r') as actives: active_data = json.load(actives) except Exception, e: raise e try: conn = r.connect(host='localhost', port=28015) conn.repl() except: print 'RethinkDB : Connection Problem' sys.exit(1) stats = [] index = 0 for place in active_data: circle = r.circle([float(place['lng'].encode('ascii','ignore')), float(place['lat'].encode('ascii','ignore'))], 3169, unit='m') response = r.db('test').table('warehouse').filter(lambda instance: instance['available'] == True).pluck('cab_lat', 'cab_lng').distinct().map(lambda instance: circle.includes(r.point(instance['cab_lng'].coerce_to('number'),instance['cab_lat'].coerce_to('number')))).run() t = 0 for res in response: if res: t += 1 index += 1 stats.append({'lat': place['lat'].encode('ascii','ignore'), 'lng': place['lng'].encode('ascii','ignore'), 'count': t}) print stats