def matrix_od(locations,
idxSources="all",
idxDestinations="all",
useKey=None,
modeTransportation='foot-walking'):
"""
Execute Matrix Service
"""
from gasp.web import json_fm_httpget
API_KEY_TO_USE = API_KEY if not useKey else useKey
URL = ("{_main}matrix?api_key={apik}&profile={transport}&"
"locations={locStr}&sources={idx_src}&destinations={idx_des}"
"&metrics=duration&units=m&optimized=true").format(
_main=MAIN_URL,
apik=API_KEY_TO_USE,
transport=modeTransportation,
locStr=locations,
idx_src=idxSources,
idx_des=idxDestinations)
data = json_fm_httpget(URL)
return data
def get_address(lat, lng):
"""
Get a address for a given point
"""
def sanitize_coord(coord):
if ',' in str(coord):
return str(coord).replace(',', '.')
else:
return str(coord)
# Get Key to be used
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key()
URL = '{url}latlng={__lat},{__long}&key={k}'.format(
url = GOOGLE_GEOCODING_URL,
k = GOOGLE_API_KEY,
__lat = sanitize_coord(lat),
__long = sanitize_coord(lng)
)
# Record api utilization
record_api_utilization(KEY_FID, NR_REQUESTS + 1)
try:
address = json_fm_httpget(URL)
except:
raise ValueError(
'Something went wrong. The URL was {}'.format(URL)
)
return check_result(address, "GEOCODING")
def matrix(locations,
idxSources="all",
idxDestinations="all",
useKey=None,
modeTransportation='driving'):
"""
modeTransportation Options:
* driving;
* walking;
* cycling;
* driving-traffic.
"""
from gasp.web import json_fm_httpget
API_KEY_TO_USE = API_KEY if not useKey else useKey
URL = ("{murl}directions-matrix/v1/mapbox/{mtrans}/{coord}?"
"sources={srcIdx}&destinations={desIdx}&access_token="
"{api_key}").format(murl=MAIN_URL,
mtrans=modeTransportation,
coord=locations,
srcIdx=idxSources,
desIdx=idxDestinations,
api_key=API_KEY_TO_USE)
data = json_fm_httpget(URL)
return data
def matrix(origins, destinations, mode='driving'):
"""
Retrieve a distance matrix from a group of origins and
destinations.
origins and destinations = [(x, y), ..., (x, y)]
or
origins and destinations = '-8.052,40.052;-8.053,40.055'
"""
from gasp.web import json_fm_httpget
def sanitize_coords(pair):
x, y = str(pair[0]), str(pair[1])
return '{},{}'.format(x if ',' not in x else x.replace(',', '.'),
y if ',' not in y else y.replace(',', '.'))
origins = str(origins) if type(origins) == str or \
type(origins) == unicode else \
';'.join([sanitize_coords(x) for x in origins]) \
if type(origins) == list else None
destinations = str(destinations) if type(destinations) == str \
or type(destinations) == unicode else \
';'.join([sanitize_coords(x) for x in destinations]) \
if type(destinations) == list else None
if not origins or not destinations:
raise ValueError('origins or destinations value is not valid')
cnt = 0
__origins = origins.split(';')
__dest = destinations.split(';')
for i in range(len(__origins)):
if not i:
src = str(cnt)
else:
src += ';' + str(cnt)
cnt += 1
for i in range(len(__dest)):
if not i:
dest = str(cnt)
else:
dest += ';' + str(cnt)
cnt += 1
URL = "{}/table/v1/{}/{};{}?source={}&destinations={}".format(
BASE_URL, mode, origins, destinations, src, dest)
data = json_fm_httpget(URL)
return data
def get_location(facebook_id):
"""
Return the absolute location (x, y) of some facebook id
"""
from gasp.web import json_fm_httpget
url = '{grph}{__id}?fields=location&access_token={t_id}|{scret}'.format(
grph=FACEBOOK_GRAPH_URL,
__id=str(facebook_id),
t_id=FACEBOOK_TOKEN['APP_ID'],
scret=FACEBOOK_TOKEN['APP_SECRET'])
data = json_fm_httpget(url)['location']
return data['longitude'], data['latitude']
def conditions_by_position(lat, lng):
"""
Get Climatic conditions for sensor in some position
"""
from gasp.web import json_fm_httpget
URL = (
"https://api.openweathermap.org/data/2.5/weather?lat={}&lon={}"
"&appid={}"
).format(str(lat), str(lng), API_KEY)
print URL
data = json_fm_httpget(URL)
return data
def nearest(lat, lng, number=1, mode='driving'):
"""
Snaps a coordinate to the street network and returns the nearest n matches.
"""
from gasp.web import json_fm_httpget
URL = "{base}/nearest/v1/{_mod_}/{_lat},{_lng}?number={nr}".format(
URL=BASE_URL,
_mod_=mode,
_lat=str(lat) if ',' not in str(lat) else lat.replace(',', '.'),
_lng=str(lng) if ',' not in str(lng) else lng.replace(',', '.'),
nr=str(number))
data = json_fm_httpget(URL)
return data
def get_all_fields_by_id(facebook_id, data_type):
"""
Return all data avaiable for a post, photo, video, etc.
"""
from gasp.web import json_fm_httpget
url = '{base}{_id_}/?fields={fld}&access_token={t_id}|{scret}'.format(
base=FACEBOOK_GRAPH_URL,
_id_=str(facebook_id),
fld=','.join(FACEBOOK_NODE_FIELDS[data_type]),
t_id=FACEBOOK_TOKEN['APP_ID'],
scret=FACEBOOK_TOKEN['APP_SECRET'])
data = json_fm_httpget(url)
return data
def directions(latOrigin,
lngOrigin,
latDestination,
lngDestination,
modeTransportation='foot-walking'):
"""
Get Shortest path between two points using Directions service
profile options:
* driving-car;
* driving-hgv;
* cycling-regular;
* cycling-road;
* cycling-safe;
* cycling-mountain;
* cycling-tour;
* cycling-electric;
* foot-walking;
* foot-hiking;
* wheelchair.
preference options:
* fastest, shortest, recommended
format options: geojson, gpx
DOC: https://openrouteservice.org/documentation/#/authentication/UserSecurity
"""
from gasp.web import json_fm_httpget
URL = ("{_url_}directions?api_key={apik}&"
"coordinates={ox},{oy}|{dx},{dy},&"
"profile={prof}&preferences=fastest&"
"format=geojson").format(_url_=MAIN_URL,
apik=API_KEY,
oy=latOrigin,
ox=lngOrigin,
dy=latDestination,
dx=lngDestination,
prof=modeTransportation)
data = json_fm_httpget(URL)
return data
def get_places_by_radius(lat, lng, radius, keyword=None, _type=None):
"""
Get Places
"""
def sanitize_coord(coord):
if ',' in str(coord):
return str(coord).replace(',', '.')
else:
return str(coord)
from gasp import unicode_to_str
# Get Google Maps Key
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key()
# Prepare URL
keyword = unicode_to_str(keyword) if type(keyword) == unicode else \
keyword
str_keyword = '' if not keyword else '&keyword={}'.format(keyword)
_type = unicode_to_str(_type) if type(_type) == unicode else \
_type
str_type = '' if not _type else '&type={}'.format(_type)
URL = '{url}location={lt},{lg}&radius={r}&key={apik}{kw}{typ}'.format(
url=GOOGLE_PLACES_URL,
apik=GOOGLE_API_KEY,
lt=sanitize_coord(lat),
lg=sanitize_coord(lng),
r=sanitize_coord(radius),
kw=str_keyword,
typ=str_type)
data = json_fm_httpget(URL)
# Record Key utilization
record_api_utilization(KEY_FID, NR_REQUESTS)
return data
def route(latOrig, lngOrig, latDest, lngDest, profile='driving'):
"""
Get route between two points
"""
"""
URL Parameters:
service - One of the following values: route,
nearest, table, match, trip, tile
version - Version of the protocol implemented by the service.
v1 for all OSRM 5.x installations
profile - Mode of transportation, is determined statically by
the Lua profile that is used to prepare the data using osrm-extract.
Typically car, bike or foot if using one of the supplied profiles.
coordinates - String of format
{longitude},{latitude};{longitude},{latitude}[;{longitude},{latitude} ...]
or polyline({polyline}) or polyline6({polyline6}).
format - Only json is supported at the moment.
This parameter is optional and defaults to json
"""
from gasp.web import json_fm_httpget
URL = "{base}/route/v1/{pro}/{lng_o},{lat_o};{lng_d},{lat_d}".format(
base=BASE_URL,
lng_o=str(lngOrig) if ',' not in str(lngOrig) else lngOrig.replace(
',', '.'),
lat_o=str(latOrig) if ',' not in str(latOrig) else latOrig.replace(
',', '.'),
lng_d=str(lngDest) if ',' not in str(lngDest) else lngDest.replace(
',', '.'),
lat_d=str(latDest) if ',' not in str(latDest) else latDest.replace(
',', ','),
pro=profile)
data = json_fm_httpget(URL)
return data
def pnts_elev(pnt_coords):
"""
Get Points Elevation
"""
# Get key
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key()
URL = '{url}locations={loc}&key={key}'.format(url=GOOGLE_ELEVATION_URL,
key=GOOGLE_API_KEY,
loc=pnt_coords)
record_api_utilization(KEY_FID, NR_REQUESTS + 1)
try:
elv = json_fm_httpget(URL)
except:
raise ValueError('Something went wrong. The URL was {}'.format(URL))
return elv
def pnt_elev(x, y):
"""
Get point elevation
"""
# Get key
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key()
# Record KEY utilization
record_api_utilization(KEY_FID, GOOGLE_API_KEY, NR_REQUESTS + 1)
try:
elev_array = json_fm_httpget(
'{url}locations={lat},{lng}&key={key}'.format(
url=GOOGLE_ELEVATION_URL,
lat=str(y),
lng=str(x),
key=GOOGLE_API_KEY))
except:
raise ValueError('Something went wrong. The URL was {}'.format(URL))
return check_result(elev_array, "ELEVATION")
def point_to_point(latA, lngA, latB, lngB, mode="driving"):
"""
Go from A to B with Google Maps Directions API
DRIVING OPTIONS: driving; walking; bicycling
"""
import polyline
# Get Key to be used
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key()
path = json_fm_httpget((
'{url}origin={lat},{lng}&'
'destination={__lat},{__lng}&'
'key={k}'
).format(
url=GOOGLE_GEOCODING_URL, lat=str(latA), lng=str(lngA),
__lat=str(latB), __lng=str(lngB), k=GOOGLE_API_KEY
))
# Record api utilization
record_api_utilization(KEY_FID, NR_REQUESTS + 1)
results = path['routes'][0]
results['polyline'] = polyline.decode(
results['overview_polyline']['points']
)
results['general_distance'] = results['legs'][0]['distance']
results['general_duration'] = results['legs'][0]['duration']
del results['overview_polyline']['points']
del results['legs'][0]['distance']
del results['legs'][0]['duration']
return results
def isochrones(locations,
range,
range_type='time',
modeTransportation='foot-walking',
intervals=None,
useKey=None):
"""
Obtain areas of reachability from given locations
The Isochrone Service supports time and distance analyses for one
single or multiple locations. You may also specify the isochrone
interval or provide multiple exact isochrone range values.
This service allows the same range of profile options listed in the
ORS Routing section which help you to further customize your request
to obtain a more detailed reachability area response.
"""
from gasp.web import json_fm_httpget
url_intervals = "&interval={}".format(str(intervals)) if intervals \
else ""
API_KEY_TO_USE = API_KEY if not useKey else useKey
URL = ("{_url_}isochrones?api_key={apik}&"
"locations={loc}&profile={transport}&range_type={rng_type}&"
"range={rng}{_int}").format(_url_=MAIN_URL,
apik=API_KEY_TO_USE,
loc=locations,
transport=modeTransportation,
rng_type=range_type,
rng=range,
_int=url_intervals)
data = json_fm_httpget(URL)
return data
def pnt_to_pnt_duration(latA, lngA, latB, lngB, mode="driving"):
"""
Return duration from going from A to B
DRIVING OPTIONS: driving; walking; bicycling
"""
# Get Key to be used
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key()
path = json_fm_httpget((
'{url}origin={lat},{lng}&'
'destination={__lat},{__lng}&'
'key={k}'
).format(
url=GOOGLE_GEOCODING_URL, lat=str(latA), lng=str(lngA),
__lat=str(latB), __lng=str(lngB), k=GOOGLE_API_KEY
))
# Record api utilization
record_api_utilization(KEY_FID, NR_REQUESTS + 1)
# Return result
return path["routes"][0]["legs"][0]["duration"]["value"]
def get_position(address, country=None, language=None, locality=None,
postal_code=None):
"""
Get position of an address
* Address examples
address = ['1600 Amphitheatre+Parkway', 'Mountain+View', 'CA']
address = 'Rua dos Combatentes da Grande Guerra 14,3030-175,Coimbra'
Only one address is allowed
"""
from gasp import unicode_to_str
ADDRESS_SANITIZE = ','.join(address) if \
type(address) == list else address if \
type(address) == str or type(address) == unicode else None
if not ADDRESS_SANITIZE:
raise ValueError(('Given Address is not valid!'))
ADDRESS_SANITIZE = unicode(ADDRESS_SANITIZE) if ' ' not in ADDRESS_SANITIZE \
else unicode(ADDRESS_SANITIZE.replace(' ', '+'))
ADDRESS_SANITIZE = unicode_to_str(ADDRESS_SANITIZE)
if country:
try: country = unicode_to_str(country)
except: pass
if locality:
try: locality = unicode_to_str(locality)
except: pass
COUNTRY = '' if not country else 'country:{}'.format(country)
LOCALITY = '' if not locality else 'locality:{}'.format(locality)
POS_CODE = '' if not postal_code else 'postal_code:{}'.format(postal_code)
if COUNTRY != '' or LOCALITY != '' or POS_CODE != '':
COMPONENTS = '&components={}'.format(
'|'.join([x for x in [COUNTRY, LOCALITY, POS_CODE] if x != ''])
)
else: COMPONENTS = ''
LANGUAGE = '' if not language else '&language={}'.format(language)
# Get Key to be used
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key()
URL = '{url}address={addrs}&key={k}{lang}{comp}'.format(
url=GOOGLE_GEOCODING_URL, k=GOOGLE_API_KEY,
addrs=ADDRESS_SANITIZE,
lang=LANGUAGE,
comp=COMPONENTS
)
# Record api utilization
record_api_utilization(KEY_FID, NR_REQUESTS + 1)
try:
position = json_fm_httpget(URL)
except:
raise ValueError(
'Something went wrong. The URL was {}'.format(URL)
)
return check_result(position, "GEOCODING")
def extract_from_url_and_next(url, Nnext=None, returnNext=None):
"""
Extract data from Facebook URL and from next URL's until fullfil Nnext
"""
import pandas
from gasp.web import json_fm_httpget
raw_data = json_fm_httpget(url)
data = pandas.DataFrame(raw_data["data"])
if not Nnext:
if not returnNext:
return sanitizeData(data)
else:
if 'paging' in raw_data:
if 'next' in raw_data['paging']:
return sanitizeData(data, raw_data["paging"]["next"])
else:
return sanitizeData(data, None)
else:
return sanitizeData(data, None)
else:
if 'paging' not in raw_data:
if not returnNext:
return sanitizeData(data)
else:
return data, None
if 'next' not in raw_data['paging']:
if not returnNext:
return data
else:
return data, None
for i in range(Nnext):
new_URL = raw_data["paging"]["next"]
moreRawData = json_fm_httpget(new_URL)
data = data.append(pandas.DataFrame(moreRawData['data']),
ignore_index=True)
if 'paging' in moreRawData:
if 'next' in moreRawData['paging']:
new_URL = moreRawData['paging']['next']
else:
break
else:
break
if not returnNext:
return sanitizeData(data)
else:
return sanitizeData(data), new_URL
def extract_by_page(face_page,
data_type='posts',
nposts=100,
returnNext=None,
apiKeyToUse=None):
"""
Extract data from one or several Facebook pages using the
Facebook GRAPH API
The data_type could be:
* Posts
* Photos
* Videos
* Locations
Reference Doc: https://developers.facebook.com/docs/graph-api/reference/v3.1/post
"""
import pandas
from gasp.web import json_fm_httpget
if not apiKeyToUse:
KEY_ID, KEY_SECRET = FACEBOOK_TOKEN['APP_ID'], FACEBOOK_TOKEN[
'APP_SECRET']
else:
KEY_ID, KEY_SECRET = apiKeyToUse
FIELDS = [
'message', 'story', 'created_time', 'description', 'full_picture',
'link', 'place', 'type'
] if data_type == 'posts' else None
URL = ('{graph}{page}/{dt}/?key=value&access_token={_id}|{secret}'
'&limit=100{flds}').format(graph=FACEBOOK_GRAPH_URL,
page=face_page,
_id=KEY_ID,
secret=KEY_SECRET,
dt=data_type,
flds='' if not FIELDS else
'&fields={}'.format(",".join(FIELDS)))
try:
raw_data = json_fm_httpget(URL)
except:
print URL
return None, None
data = pandas.DataFrame(raw_data["data"])
if nposts <= 100:
if not returnNext:
return sanitizeData(data, FACE_PAGE=face_page)
else:
if 'paging' in raw_data:
if 'next' in raw_data['paging']:
return sanitizeData(
data, FACE_PAGE=face_page), raw_data["paging"]["next"]
else:
return sanitizeData(data, FACE_PAGE=face_page), None
else:
return sanitizeData(data, FACE_PAGE=face_page), None
else:
N = int(round(nposts / 100.0, 0))
new_URL = raw_data["paging"]["next"]
for n in range(N - 1):
try:
moreRawData = json_fm_httpget(new_URL)
except:
return None, None
data = data.append(pandas.DataFrame(moreRawData['data']),
ignore_index=True)
if 'paging' in moreRawData:
if 'next' in moreRawData['paging']:
new_URL = moreRawData["paging"]["next"]
else:
break
else:
break
if not returnNext:
return sanitizeData(data, FACE_PAGE=face_page)
else:
return sanitizeData(data, FACE_PAGE=face_page), new_URL
def by_query(search_type,
keyword=None, x_center=None, y_center=None, dist=None,
limit='100', face_fields=None):
"""
Search data on facebook based on:
- Keyword;
- search type (user, page, event, place, placetopic);
- location (center and distance from center);
- limit (maximum number of users/pages/etc. to be returned)*.
* Our default is 100, but the Facebook default is 60.
Returns an array with the id of the data in facebook
"""
import pandas
from gasp import goToList, unicode_to_str
from gasp.web import json_fm_httpget
# Deal with spaces in the keyword expression and with special characters
keyword = unicode_to_str(keyword) if type(keyword) == unicode \
else keyword
keyword = keyword.replace(' ', '%20') if keyword and ' ' in keyword \
else keyword
face_fields = goToList(face_fields)
URL = (
'{graph}search?access_token={_id}|{scrt}'
'{_q}{typ}{cnt}{dst}{lmt}{flds}'
).format(
graph=FACEBOOK_GRAPH_URL,
_id = FACEBOOK_TOKEN['APP_ID'],
scrt = FACEBOOK_TOKEN['APP_SECRET'],
_q = '' if not keyword else '&q={}'.format(keyword),
typ = '&type={}'.format(search_type),
cnt = '' if not x_center and not y_center else '¢er={},{}'.format(
y_center, x_center
),
dst = '' if not dist else '&distance={}'.format(dist),
lmt = '' if not limit else '&limit={}'.format(str(limit)),
flds = '' if not face_fields else '&fields={}'.format(','.join(face_fields))
)
face_table = pandas.DataFrame(json_fm_httpget(URL)['data'])
if not face_table.shape[0]:
return None
face_table["url"] = "https://facebook.com//" + face_table["id"]
if face_fields:
if "location" in face_fields:
face_table = pandas.concat([
face_table.drop(["location"], axis=1),
face_table["location"].apply(pandas.Series)
], axis=1)
return face_table
def dist_matrix(origins,
destination,
NORIGINS,
NDEST,
transport_mode=None,
useKey=None):
"""
Get distance matrix considering several origins and destinations
Avaiable transport modes:
* driving (standard) - indica que a distancia deve ser calculada usando a
rede de estradas.
* walking - solicita o calculo de distancia para caminhadas por vias
para pedestres e calcadas (quando disponiveis).
* bicycling - solicita o calculo de distancia para bicicleta por
ciclovias e ruas preferencias (quando disponiveis).
* transit - solicita o calculo de distancia por rotas de transporte
publico (quando disponiveis). O valor so podera ser especificado se a
solicitacao incluir uma chave de API ou um ID de cliente do Google Maps
APIs Premium Plan. Se voce definir o modo como transit, podera especificar
um valor de departure_time ou arrival_time. Se nenhum desses valores
for especificado, o valor padrao de departure_time sera "now" (ou seja,
o horario de partida padrao e o atual). Ha tambem a opcao para
incluir um transit_mode e/ou uma transit_routing_preference.
"""
def sanitize_coords(pair):
if type(pair) == tuple:
x, y = str(pair[0]), str(pair[1])
return '{},{}'.format(y if ',' not in y else y.replace(',', '.'),
x if ',' not in x else x.replace(',', '.'))
elif type(pair) == str:
return pair
else:
raise ValueError("Values of origins/destinations are not valid")
origins_str = str(origins) if type(origins) == str or \
type(origins) == unicode else \
'|'.join([sanitize_coords(x) for x in origins]) \
if type(origins) == list else None
destination_str = str(destination) if type(destination) == str or \
type(destination) == unicode else \
'|'.join([sanitize_coords(x) for x in destination]) \
if type(destination) == list else None
if not origins_str or not destination_str:
raise ValueError('origins or destination value is not valid')
if useKey:
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = None, useKey, None
else:
# Get Key to be used
KEY_FID, GOOGLE_API_KEY, NR_REQUESTS = select_api_key()
transport_mode = 'driving' if not transport_mode else transport_mode
URL = ('{u}origins={o_str}&destinations={d_str}&'
'mode={m}&key={api_key}').format(u=GOOGLE_DISTMATRIX_URL,
o_str=origins_str,
d_str=destination_str,
m=transport_mode,
api_key=GOOGLE_API_KEY)
# Record api utilization
if not useKey:
record_api_utilization(KEY_FID, NR_REQUESTS + NORIGINS + NDEST)
try:
matrix = json_fm_httpget(URL)
except:
raise ValueError('Something went wrong. The URL was {}'.format(URL))
return check_result(matrix, "DISTANCE_MATRIX")