def matrix_od(locations, idxSources="all", idxDestinations="all", useKey=None, modeTransportation='foot-walking'): """ Execute Matrix Service """ from gasp.web import json_fm_httpget API_KEY_TO_USE = API_KEY if not useKey else useKey URL = ("{_main}matrix?api_key={apik}&profile={transport}&" "locations={locStr}&sources={idx_src}&destinations={idx_des}" "&metrics=duration&units=m&optimized=true").format( _main=MAIN_URL, apik=API_KEY_TO_USE, transport=modeTransportation, locStr=locations, idx_src=idxSources, idx_des=idxDestinations) data = json_fm_httpget(URL) return data
def get_address(lat, lng): """ Get a address for a given point """ def sanitize_coord(coord): if ',' in str(coord): return str(coord).replace(',', '.') else: return str(coord) # Get Key to be used KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key() URL = '{url}latlng={__lat},{__long}&key={k}'.format( url = GOOGLE_GEOCODING_URL, k = GOOGLE_API_KEY, __lat = sanitize_coord(lat), __long = sanitize_coord(lng) ) # Record api utilization record_api_utilization(KEY_FID, NR_REQUESTS + 1) try: address = json_fm_httpget(URL) except: raise ValueError( 'Something went wrong. The URL was {}'.format(URL) ) return check_result(address, "GEOCODING")
def matrix(locations, idxSources="all", idxDestinations="all", useKey=None, modeTransportation='driving'): """ modeTransportation Options: * driving; * walking; * cycling; * driving-traffic. """ from gasp.web import json_fm_httpget API_KEY_TO_USE = API_KEY if not useKey else useKey URL = ("{murl}directions-matrix/v1/mapbox/{mtrans}/{coord}?" "sources={srcIdx}&destinations={desIdx}&access_token=" "{api_key}").format(murl=MAIN_URL, mtrans=modeTransportation, coord=locations, srcIdx=idxSources, desIdx=idxDestinations, api_key=API_KEY_TO_USE) data = json_fm_httpget(URL) return data
def matrix(origins, destinations, mode='driving'): """ Retrieve a distance matrix from a group of origins and destinations. origins and destinations = [(x, y), ..., (x, y)] or origins and destinations = '-8.052,40.052;-8.053,40.055' """ from gasp.web import json_fm_httpget def sanitize_coords(pair): x, y = str(pair[0]), str(pair[1]) return '{},{}'.format(x if ',' not in x else x.replace(',', '.'), y if ',' not in y else y.replace(',', '.')) origins = str(origins) if type(origins) == str or \ type(origins) == unicode else \ ';'.join([sanitize_coords(x) for x in origins]) \ if type(origins) == list else None destinations = str(destinations) if type(destinations) == str \ or type(destinations) == unicode else \ ';'.join([sanitize_coords(x) for x in destinations]) \ if type(destinations) == list else None if not origins or not destinations: raise ValueError('origins or destinations value is not valid') cnt = 0 __origins = origins.split(';') __dest = destinations.split(';') for i in range(len(__origins)): if not i: src = str(cnt) else: src += ';' + str(cnt) cnt += 1 for i in range(len(__dest)): if not i: dest = str(cnt) else: dest += ';' + str(cnt) cnt += 1 URL = "{}/table/v1/{}/{};{}?source={}&destinations={}".format( BASE_URL, mode, origins, destinations, src, dest) data = json_fm_httpget(URL) return data
def get_location(facebook_id): """ Return the absolute location (x, y) of some facebook id """ from gasp.web import json_fm_httpget url = '{grph}{__id}?fields=location&access_token={t_id}|{scret}'.format( grph=FACEBOOK_GRAPH_URL, __id=str(facebook_id), t_id=FACEBOOK_TOKEN['APP_ID'], scret=FACEBOOK_TOKEN['APP_SECRET']) data = json_fm_httpget(url)['location'] return data['longitude'], data['latitude']
def conditions_by_position(lat, lng): """ Get Climatic conditions for sensor in some position """ from gasp.web import json_fm_httpget URL = ( "https://api.openweathermap.org/data/2.5/weather?lat={}&lon={}" "&appid={}" ).format(str(lat), str(lng), API_KEY) print URL data = json_fm_httpget(URL) return data
def nearest(lat, lng, number=1, mode='driving'): """ Snaps a coordinate to the street network and returns the nearest n matches. """ from gasp.web import json_fm_httpget URL = "{base}/nearest/v1/{_mod_}/{_lat},{_lng}?number={nr}".format( URL=BASE_URL, _mod_=mode, _lat=str(lat) if ',' not in str(lat) else lat.replace(',', '.'), _lng=str(lng) if ',' not in str(lng) else lng.replace(',', '.'), nr=str(number)) data = json_fm_httpget(URL) return data
def get_all_fields_by_id(facebook_id, data_type): """ Return all data avaiable for a post, photo, video, etc. """ from gasp.web import json_fm_httpget url = '{base}{_id_}/?fields={fld}&access_token={t_id}|{scret}'.format( base=FACEBOOK_GRAPH_URL, _id_=str(facebook_id), fld=','.join(FACEBOOK_NODE_FIELDS[data_type]), t_id=FACEBOOK_TOKEN['APP_ID'], scret=FACEBOOK_TOKEN['APP_SECRET']) data = json_fm_httpget(url) return data
def directions(latOrigin, lngOrigin, latDestination, lngDestination, modeTransportation='foot-walking'): """ Get Shortest path between two points using Directions service profile options: * driving-car; * driving-hgv; * cycling-regular; * cycling-road; * cycling-safe; * cycling-mountain; * cycling-tour; * cycling-electric; * foot-walking; * foot-hiking; * wheelchair. preference options: * fastest, shortest, recommended format options: geojson, gpx DOC: https://openrouteservice.org/documentation/#/authentication/UserSecurity """ from gasp.web import json_fm_httpget URL = ("{_url_}directions?api_key={apik}&" "coordinates={ox},{oy}|{dx},{dy},&" "profile={prof}&preferences=fastest&" "format=geojson").format(_url_=MAIN_URL, apik=API_KEY, oy=latOrigin, ox=lngOrigin, dy=latDestination, dx=lngDestination, prof=modeTransportation) data = json_fm_httpget(URL) return data
def get_places_by_radius(lat, lng, radius, keyword=None, _type=None): """ Get Places """ def sanitize_coord(coord): if ',' in str(coord): return str(coord).replace(',', '.') else: return str(coord) from gasp import unicode_to_str # Get Google Maps Key KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key() # Prepare URL keyword = unicode_to_str(keyword) if type(keyword) == unicode else \ keyword str_keyword = '' if not keyword else '&keyword={}'.format(keyword) _type = unicode_to_str(_type) if type(_type) == unicode else \ _type str_type = '' if not _type else '&type={}'.format(_type) URL = '{url}location={lt},{lg}&radius={r}&key={apik}{kw}{typ}'.format( url=GOOGLE_PLACES_URL, apik=GOOGLE_API_KEY, lt=sanitize_coord(lat), lg=sanitize_coord(lng), r=sanitize_coord(radius), kw=str_keyword, typ=str_type) data = json_fm_httpget(URL) # Record Key utilization record_api_utilization(KEY_FID, NR_REQUESTS) return data
def route(latOrig, lngOrig, latDest, lngDest, profile='driving'): """ Get route between two points """ """ URL Parameters: service - One of the following values: route, nearest, table, match, trip, tile version - Version of the protocol implemented by the service. v1 for all OSRM 5.x installations profile - Mode of transportation, is determined statically by the Lua profile that is used to prepare the data using osrm-extract. Typically car, bike or foot if using one of the supplied profiles. coordinates - String of format {longitude},{latitude};{longitude},{latitude}[;{longitude},{latitude} ...] or polyline({polyline}) or polyline6({polyline6}). format - Only json is supported at the moment. This parameter is optional and defaults to json """ from gasp.web import json_fm_httpget URL = "{base}/route/v1/{pro}/{lng_o},{lat_o};{lng_d},{lat_d}".format( base=BASE_URL, lng_o=str(lngOrig) if ',' not in str(lngOrig) else lngOrig.replace( ',', '.'), lat_o=str(latOrig) if ',' not in str(latOrig) else latOrig.replace( ',', '.'), lng_d=str(lngDest) if ',' not in str(lngDest) else lngDest.replace( ',', '.'), lat_d=str(latDest) if ',' not in str(latDest) else latDest.replace( ',', ','), pro=profile) data = json_fm_httpget(URL) return data
def pnts_elev(pnt_coords): """ Get Points Elevation """ # Get key KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key() URL = '{url}locations={loc}&key={key}'.format(url=GOOGLE_ELEVATION_URL, key=GOOGLE_API_KEY, loc=pnt_coords) record_api_utilization(KEY_FID, NR_REQUESTS + 1) try: elv = json_fm_httpget(URL) except: raise ValueError('Something went wrong. The URL was {}'.format(URL)) return elv
def pnt_elev(x, y): """ Get point elevation """ # Get key KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key() # Record KEY utilization record_api_utilization(KEY_FID, GOOGLE_API_KEY, NR_REQUESTS + 1) try: elev_array = json_fm_httpget( '{url}locations={lat},{lng}&key={key}'.format( url=GOOGLE_ELEVATION_URL, lat=str(y), lng=str(x), key=GOOGLE_API_KEY)) except: raise ValueError('Something went wrong. The URL was {}'.format(URL)) return check_result(elev_array, "ELEVATION")
def point_to_point(latA, lngA, latB, lngB, mode="driving"): """ Go from A to B with Google Maps Directions API DRIVING OPTIONS: driving; walking; bicycling """ import polyline # Get Key to be used KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key() path = json_fm_httpget(( '{url}origin={lat},{lng}&' 'destination={__lat},{__lng}&' 'key={k}' ).format( url=GOOGLE_GEOCODING_URL, lat=str(latA), lng=str(lngA), __lat=str(latB), __lng=str(lngB), k=GOOGLE_API_KEY )) # Record api utilization record_api_utilization(KEY_FID, NR_REQUESTS + 1) results = path['routes'][0] results['polyline'] = polyline.decode( results['overview_polyline']['points'] ) results['general_distance'] = results['legs'][0]['distance'] results['general_duration'] = results['legs'][0]['duration'] del results['overview_polyline']['points'] del results['legs'][0]['distance'] del results['legs'][0]['duration'] return results
def isochrones(locations, range, range_type='time', modeTransportation='foot-walking', intervals=None, useKey=None): """ Obtain areas of reachability from given locations The Isochrone Service supports time and distance analyses for one single or multiple locations. You may also specify the isochrone interval or provide multiple exact isochrone range values. This service allows the same range of profile options listed in the ORS Routing section which help you to further customize your request to obtain a more detailed reachability area response. """ from gasp.web import json_fm_httpget url_intervals = "&interval={}".format(str(intervals)) if intervals \ else "" API_KEY_TO_USE = API_KEY if not useKey else useKey URL = ("{_url_}isochrones?api_key={apik}&" "locations={loc}&profile={transport}&range_type={rng_type}&" "range={rng}{_int}").format(_url_=MAIN_URL, apik=API_KEY_TO_USE, loc=locations, transport=modeTransportation, rng_type=range_type, rng=range, _int=url_intervals) data = json_fm_httpget(URL) return data
def pnt_to_pnt_duration(latA, lngA, latB, lngB, mode="driving"): """ Return duration from going from A to B DRIVING OPTIONS: driving; walking; bicycling """ # Get Key to be used KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key() path = json_fm_httpget(( '{url}origin={lat},{lng}&' 'destination={__lat},{__lng}&' 'key={k}' ).format( url=GOOGLE_GEOCODING_URL, lat=str(latA), lng=str(lngA), __lat=str(latB), __lng=str(lngB), k=GOOGLE_API_KEY )) # Record api utilization record_api_utilization(KEY_FID, NR_REQUESTS + 1) # Return result return path["routes"][0]["legs"][0]["duration"]["value"]
def get_position(address, country=None, language=None, locality=None, postal_code=None): """ Get position of an address * Address examples address = ['1600 Amphitheatre+Parkway', 'Mountain+View', 'CA'] address = 'Rua dos Combatentes da Grande Guerra 14,3030-175,Coimbra' Only one address is allowed """ from gasp import unicode_to_str ADDRESS_SANITIZE = ','.join(address) if \ type(address) == list else address if \ type(address) == str or type(address) == unicode else None if not ADDRESS_SANITIZE: raise ValueError(('Given Address is not valid!')) ADDRESS_SANITIZE = unicode(ADDRESS_SANITIZE) if ' ' not in ADDRESS_SANITIZE \ else unicode(ADDRESS_SANITIZE.replace(' ', '+')) ADDRESS_SANITIZE = unicode_to_str(ADDRESS_SANITIZE) if country: try: country = unicode_to_str(country) except: pass if locality: try: locality = unicode_to_str(locality) except: pass COUNTRY = '' if not country else 'country:{}'.format(country) LOCALITY = '' if not locality else 'locality:{}'.format(locality) POS_CODE = '' if not postal_code else 'postal_code:{}'.format(postal_code) if COUNTRY != '' or LOCALITY != '' or POS_CODE != '': COMPONENTS = '&components={}'.format( '|'.join([x for x in [COUNTRY, LOCALITY, POS_CODE] if x != '']) ) else: COMPONENTS = '' LANGUAGE = '' if not language else '&language={}'.format(language) # Get Key to be used KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key() URL = '{url}address={addrs}&key={k}{lang}{comp}'.format( url=GOOGLE_GEOCODING_URL, k=GOOGLE_API_KEY, addrs=ADDRESS_SANITIZE, lang=LANGUAGE, comp=COMPONENTS ) # Record api utilization record_api_utilization(KEY_FID, NR_REQUESTS + 1) try: position = json_fm_httpget(URL) except: raise ValueError( 'Something went wrong. The URL was {}'.format(URL) ) return check_result(position, "GEOCODING")
def extract_from_url_and_next(url, Nnext=None, returnNext=None): """ Extract data from Facebook URL and from next URL's until fullfil Nnext """ import pandas from gasp.web import json_fm_httpget raw_data = json_fm_httpget(url) data = pandas.DataFrame(raw_data["data"]) if not Nnext: if not returnNext: return sanitizeData(data) else: if 'paging' in raw_data: if 'next' in raw_data['paging']: return sanitizeData(data, raw_data["paging"]["next"]) else: return sanitizeData(data, None) else: return sanitizeData(data, None) else: if 'paging' not in raw_data: if not returnNext: return sanitizeData(data) else: return data, None if 'next' not in raw_data['paging']: if not returnNext: return data else: return data, None for i in range(Nnext): new_URL = raw_data["paging"]["next"] moreRawData = json_fm_httpget(new_URL) data = data.append(pandas.DataFrame(moreRawData['data']), ignore_index=True) if 'paging' in moreRawData: if 'next' in moreRawData['paging']: new_URL = moreRawData['paging']['next'] else: break else: break if not returnNext: return sanitizeData(data) else: return sanitizeData(data), new_URL
def extract_by_page(face_page, data_type='posts', nposts=100, returnNext=None, apiKeyToUse=None): """ Extract data from one or several Facebook pages using the Facebook GRAPH API The data_type could be: * Posts * Photos * Videos * Locations Reference Doc: https://developers.facebook.com/docs/graph-api/reference/v3.1/post """ import pandas from gasp.web import json_fm_httpget if not apiKeyToUse: KEY_ID, KEY_SECRET = FACEBOOK_TOKEN['APP_ID'], FACEBOOK_TOKEN[ 'APP_SECRET'] else: KEY_ID, KEY_SECRET = apiKeyToUse FIELDS = [ 'message', 'story', 'created_time', 'description', 'full_picture', 'link', 'place', 'type' ] if data_type == 'posts' else None URL = ('{graph}{page}/{dt}/?key=value&access_token={_id}|{secret}' '&limit=100{flds}').format(graph=FACEBOOK_GRAPH_URL, page=face_page, _id=KEY_ID, secret=KEY_SECRET, dt=data_type, flds='' if not FIELDS else '&fields={}'.format(",".join(FIELDS))) try: raw_data = json_fm_httpget(URL) except: print URL return None, None data = pandas.DataFrame(raw_data["data"]) if nposts <= 100: if not returnNext: return sanitizeData(data, FACE_PAGE=face_page) else: if 'paging' in raw_data: if 'next' in raw_data['paging']: return sanitizeData( data, FACE_PAGE=face_page), raw_data["paging"]["next"] else: return sanitizeData(data, FACE_PAGE=face_page), None else: return sanitizeData(data, FACE_PAGE=face_page), None else: N = int(round(nposts / 100.0, 0)) new_URL = raw_data["paging"]["next"] for n in range(N - 1): try: moreRawData = json_fm_httpget(new_URL) except: return None, None data = data.append(pandas.DataFrame(moreRawData['data']), ignore_index=True) if 'paging' in moreRawData: if 'next' in moreRawData['paging']: new_URL = moreRawData["paging"]["next"] else: break else: break if not returnNext: return sanitizeData(data, FACE_PAGE=face_page) else: return sanitizeData(data, FACE_PAGE=face_page), new_URL
def by_query(search_type, keyword=None, x_center=None, y_center=None, dist=None, limit='100', face_fields=None): """ Search data on facebook based on: - Keyword; - search type (user, page, event, place, placetopic); - location (center and distance from center); - limit (maximum number of users/pages/etc. to be returned)*. * Our default is 100, but the Facebook default is 60. Returns an array with the id of the data in facebook """ import pandas from gasp import goToList, unicode_to_str from gasp.web import json_fm_httpget # Deal with spaces in the keyword expression and with special characters keyword = unicode_to_str(keyword) if type(keyword) == unicode \ else keyword keyword = keyword.replace(' ', '%20') if keyword and ' ' in keyword \ else keyword face_fields = goToList(face_fields) URL = ( '{graph}search?access_token={_id}|{scrt}' '{_q}{typ}{cnt}{dst}{lmt}{flds}' ).format( graph=FACEBOOK_GRAPH_URL, _id = FACEBOOK_TOKEN['APP_ID'], scrt = FACEBOOK_TOKEN['APP_SECRET'], _q = '' if not keyword else '&q={}'.format(keyword), typ = '&type={}'.format(search_type), cnt = '' if not x_center and not y_center else '¢er={},{}'.format( y_center, x_center ), dst = '' if not dist else '&distance={}'.format(dist), lmt = '' if not limit else '&limit={}'.format(str(limit)), flds = '' if not face_fields else '&fields={}'.format(','.join(face_fields)) ) face_table = pandas.DataFrame(json_fm_httpget(URL)['data']) if not face_table.shape[0]: return None face_table["url"] = "https://facebook.com//" + face_table["id"] if face_fields: if "location" in face_fields: face_table = pandas.concat([ face_table.drop(["location"], axis=1), face_table["location"].apply(pandas.Series) ], axis=1) return face_table
def dist_matrix(origins, destination, NORIGINS, NDEST, transport_mode=None, useKey=None): """ Get distance matrix considering several origins and destinations Avaiable transport modes: * driving (standard) - indica que a distancia deve ser calculada usando a rede de estradas. * walking - solicita o calculo de distancia para caminhadas por vias para pedestres e calcadas (quando disponiveis). * bicycling - solicita o calculo de distancia para bicicleta por ciclovias e ruas preferencias (quando disponiveis). * transit - solicita o calculo de distancia por rotas de transporte publico (quando disponiveis). O valor so podera ser especificado se a solicitacao incluir uma chave de API ou um ID de cliente do Google Maps APIs Premium Plan. Se voce definir o modo como transit, podera especificar um valor de departure_time ou arrival_time. Se nenhum desses valores for especificado, o valor padrao de departure_time sera "now" (ou seja, o horario de partida padrao e o atual). Ha tambem a opcao para incluir um transit_mode e/ou uma transit_routing_preference. """ def sanitize_coords(pair): if type(pair) == tuple: x, y = str(pair[0]), str(pair[1]) return '{},{}'.format(y if ',' not in y else y.replace(',', '.'), x if ',' not in x else x.replace(',', '.')) elif type(pair) == str: return pair else: raise ValueError("Values of origins/destinations are not valid") origins_str = str(origins) if type(origins) == str or \ type(origins) == unicode else \ '|'.join([sanitize_coords(x) for x in origins]) \ if type(origins) == list else None destination_str = str(destination) if type(destination) == str or \ type(destination) == unicode else \ '|'.join([sanitize_coords(x) for x in destination]) \ if type(destination) == list else None if not origins_str or not destination_str: raise ValueError('origins or destination value is not valid') if useKey: KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = None, useKey, None else: # Get Key to be used KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key() transport_mode = 'driving' if not transport_mode else transport_mode URL = ('{u}origins={o_str}&destinations={d_str}&' 'mode={m}&key={api_key}').format(u=GOOGLE_DISTMATRIX_URL, o_str=origins_str, d_str=destination_str, m=transport_mode, api_key=GOOGLE_API_KEY) # Record api utilization if not useKey: record_api_utilization(KEY_FID, NR_REQUESTS + NORIGINS + NDEST) try: matrix = json_fm_httpget(URL) except: raise ValueError('Something went wrong. The URL was {}'.format(URL)) return check_result(matrix, "DISTANCE_MATRIX")