def lat_long(p):
long = []
lat = []
for index, row in p.iterrows():
long += [geocode(p.loc[index, 'Address'])[0]['location']['x']]
lat += [geocode(p.loc[index, 'Address'])[0]['location']['y']]
return long, lat
def extractAddress(df, indice, idDf, address):
gis = GIS()
geocode_result = geocode(address = df.iloc[indice][address], as_featureset=True) # Cambiar la dirección
for x in geocode_result.features:
gdpoint = x.as_dict["geometry"]
attr = x.as_dict["attributes"]
df = df.append(pd.Series([df.iloc[indice][idDf], "", "", gdpoint["x"], gdpoint["y"], attr["Match_addr"], attr["Score"]], index=df.columns ), ignore_index=True)
return df
def geocode_address_executor(address):
"""
Geocodes the given address and returns the first result back
:param address:
:return:
"""
return geocode(address)[0]
def geocode_address(m_address):
"""
:param m_address: address of interest in street form
:return: address in coordinate (X and Y) form
"""
m_address = m_address + ", City: Boston, State: MA"
m_location = geocode(address=m_address)[0]
return m_location['location']
def get_coords(address):
address = str(address)
geocode_result = geocode(address)[0]
location = geocode_result["location"]
coords = (location["x"], location["y"])
return coords
def arcgis_req(adr):
try:
address = adr
geocode_result = geocode(address , max_locations= 1 )
lat = geocode_result[0]['attributes']['Y']
lon = geocode_result[0]['attributes']['X']
return pd.Series([lat , lon])
except:
pass
def geocode_address(m_address):
"""
:param m_address: address of interest in street form
:return: address in coordinate (X and Y) form
"""
# TODO: Make it generic to handle more cities
m_address = m_address + ", City: Boston, State: MA"
m_location = geocode(address=m_address)[0]
return m_location['location']
def addressToLatLong(self, address):
"""
geocoding an address into Latitude and Longitudes
:param address:
:return: Shapely.geometry.POINT
"""
results = geocode(address=address)
loc = results[0]['location']
return Point(loc['x'], loc['y'])
def get_postcode_coordinates(postcode):
geocode_fs = geocode(address="{}, Catalunya".format(postcode),
out_fields="Place_addr",
max_locations=1,
as_featureset=False)
latitude = geocode_fs[0]['location']['y']
longitude = geocode_fs[0]['location']['x']
#print(latitude,longitude)
return latitude,longitude
def geo_sde():
gis = GIS()
map = gis.map("United States")
map
sde = pd.read_csv("jobs_sde.csv")
data3 = pd.DataFrame(columns=('company', 'city', 'state', 'longitude',
'latitude'))
for i in range(0, len(sde)):
dict = {}
if pd.isna(sde.iloc[i]['location']):
state == []
else:
state = geocode(sde.iloc[i]['location'])
if state == []:
geo = geocode(sde.iloc[i]['company'])
if geo == []:
dict['longitude'] = na
dict['latitude'] = na
dict['city'] = na
dict['state'] = na
else:
dict['longitude'] = geo[0]['location']['x']
dict['latitude'] = geo[0]['location']['y']
dict['city'] = geo[0]['attributes']['City']
dict['state'] = geo[0]['attributes']['RegionAbbr']
else:
geo = geocode(sde.iloc[i]['company'], state[0]['extent'])
if geo == []:
dict['longitude'] = state[0]['location']['x']
dict['latitude'] = state[0]['location']['y']
dict['city'] = state[0]['attributes']['City']
dict['state'] = state[0]['attributes']['RegionAbbr']
else:
dict['longitude'] = geo[0]['location']['x']
dict['latitude'] = geo[0]['location']['y']
dict['city'] = geo[0]['attributes']['City']
dict['state'] = geo[0]['attributes']['RegionAbbr']
dict['company'] = sde.iloc[i]['company']
data3.loc[i] = dict
data3.to_csv("geo_sde.csv", encoding='utf-8', index=False)
def get_geo_coordinates_from_arcgis_with_login(self, location_address: str,
connection_params: dict):
GIS("http://www.arcgis.com", connection_params['ARCGIS_USER'],
connection_params['ARCGIS_PASSWORD'])
arc_gis_loc = geocode(location_address)
return {
'longitude': arc_gis_loc[0]['location']['x'],
'latitude': arc_gis_loc[0]['location']['y'],
'all_results': arc_gis_loc
}
def get_data_from_arcGIS_api(zipcodes, keywords):
"""
##################################################################
### This code generates public addresses
### using ARCGIS's API.
### _________________________________________________________
### INPUT: array of zipcodes (numpy.ndarray)
### list of keyword to search by (list of strings)
### OUTPUT: Pandas DataFrame
###################################################################
"""
# Number of zipcodes
n = len(zipcodes)
# Initialize List and GIS service
listedList = []
gis = GIS()
# Loop through all zip codes in Cook County and save unique items with geocode
for id in range(n):
yourZip = geocode(str(zipcodes[id]))[0]
for word in range(len(keywords)):
searchedItem = geocode(keywords[word],
yourZip['extent'],
max_locations=1000)
for item2 in range(len(searchedItem)):
listedList.append({
"ADDRESS":
searchedItem[item2]['attributes']['Place_addr'],
"PHONE":
searchedItem[item2]['attributes']['Phone'],
"POSTAL":
searchedItem[item2]['attributes']['Postal'],
"LONGITUDE":
searchedItem[item2]['location']['x'],
"LATITUDE":
searchedItem[item2]['location']['y']
})
listedList = pd.DataFrame(listedList)
# Find if there are duplicates (by ADDRESS)
dup_index = listedList.duplicated(["ADDRESS"])
prop_dup = listedList[dup_index]
listedList.drop_duplicates(subset=['ADDRESS'], inplace=True)
return (listedList)
def geocode(self, locations):
'''Get coordinates from address or location name'''
coordinates = []
for location in locations:
coord = geocode(location)
coordinates.append(coord)
print(coordinates)
return coordinates
def extractAddress(self, df, indice):
zona_extent = geocode(self.ubicacion)[0]
geocode_result = geocode(address=df.iloc[indice][self.fieldAddress],
as_featureset=True,
max_locations=1,
search_extent=zona_extent["extent"])
for x in geocode_result.features:
gdpoint = x.as_dict["geometry"]
attr = x.as_dict["attributes"]
df = df.append(pd.Series([
df.iloc[indice]["ID"], df.iloc[indice][self.fieldIndice],
gdpoint["x"], gdpoint["y"], df.iloc[indice][self.fieldAddress],
attr["Match_addr"], attr["Score"]
],
index=[
"ID", self.fieldIndice, "x", "y",
self.fieldAddress, "Match_addr",
"Score"
]),
ignore_index=True)
return df
def autocomplete(search_query: str, current_location: str):
geocode_result = geocode(address=search_query,
as_featureset=True,
search_extent=current_location)
places = []
for place in geocode_result.features:
places.append({
'name': place.attributes['PlaceName'],
'address': place.attributes['Place_addr'],
'lat': place.geometry.y,
'long': place.geometry.x
})
return places
def extractAddress(data, df, indice, idDf, address):
geocode_result = geocode(address=df.iloc[indice][address],
as_featureset=True,
max_locations=1)
x = geocode_result.features[0]
gdpoint = x.as_dict["geometry"]
attr = x.as_dict["attributes"]
dataF = data.append(pd.Series(
[
df.iloc[indice][address], gdpoint["x"], gdpoint["y"],
attr["Match_addr"], attr["Score"]
],
index=[idDf, "x", "y", "Match_addr", "Score"]),
ignore_index=True)
return dataF
def get_lat_long(self, address) -> Tuple[Optional[float], Optional[float]]:
"""
Get the latitude and longitude for an address if the accuracy score is high enough
:param address: Street address to search. The more complete the address, the better.
"""
logger.debug(f'Looking up {address}')
address = self._standardize_address(address)
with warnings.catch_warnings(
): # https://github.com/Esri/arcgis-python-api/issues/1090
warnings.simplefilter("ignore")
geo_dict = geocode(f'{address}, Baltimore, MD')
lat = None
lng = None
if geo_dict and geo_dict[0]['score'] > 80:
lat = float(geo_dict[0]['location']['y'])
lng = float(geo_dict[0]['location']['x'])
return lat, lng
def getFullAdress(zoneDNS):
dns = zoneDNS
adresse = ''
commune = ''
latitude = 0
longitude = 0
codePostale = 0
csv_sortie = []
gis = GIS("http://www.arcgis.com", "Aziguy88", "LYNEpromotion2012")
if zoneDNS == '':
pass
else:
geocode_result = geocode(address=dns)
for x in geocode_result:
if 'FR' in x['attributes']['Country'] and 'Var' in (
x['attributes']['LongLabel'] or x['attributes']['Subregion']):
adresse = x['attributes']['LongLabel']
latitude = x['attributes']['Y']
longitude = x['attributes']['X']
codePostale = x['attributes']['Postal']
commune = x['attributes']['City']
with open('outputs/ZoneDNSFullAdresses.csv',
'a',
encoding='utf-16',
newline='') as fichierSortie:
csv_sortie = csv.writer(fichierSortie, delimiter='\t')
fichierVide = os.stat(
'outputs/ZoneDNSFullAdresses.csv').st_size == 0
if fichierVide:
csv_sortie.writerow([
'DNS Source', 'Adresses', 'Latitudes', 'Longitudes',
'Codes', 'Communes'
])
else:
csv_sortie.writerow([
dns, adresse, latitude, longitude, codePostale, commune
])
time.sleep(2)
else:
pass
# raise ValueError("{0} ne correspond pas aux critères".format(dns))
return csv_sortie
def get_coordinates(addresses_list, print_address=False):
'''
Muestra en pantalla las coordenadas "x,y" de cada dirección
Parámetros:
* addresses_list: Lista unidimensional de strings con las direcciones para buscar coordenadas.
* print_address: Boolean, por defecto False. En caso de ser True, imprime además dirección original.
Salida:
* Print output de coordenadas "x,y" para cada dirección
'''
for address in addresses_list:
geocode_result = geocode(address=address, as_featureset=True)
geometry = geocode_result.features[0].geometry
if print_address:
print("{},{},{}".format(geometry.x,geometry.y,address))
else:
print("{},{}".format(geometry.x,geometry.y))
def get_assignments_from_csv():
assignments_to_add = []
assignments_in_csv = []
dev_gis = GIS()
csvFile = os.path.abspath(arquivoCSV)
logger.info("Lendo CSV {}...".format(csvFile))
with open(csvFile, 'r') as file:
reader = csv.reader(file, delimiter=';')# o delimitador eh ponto-e-virgula
for row in reader:
assignments_in_csv.append(row)
for assignment in assignments_in_csv:
logger.info(assignment)#loga os campos
#!!! ATENCAO!!!
#o Geocode abaixo utiliza o geocode gratuito, ou seja, não consome créditos e faz chamadas individuais
#se fizer muitos geocodes, pode colocar o ip do solicitador na lista negra da esri
#para uso em producao, deve ser alterado para utilizar uma conta agol ou o geocode com locators locais, publicados no Enterprise
geocode_result = geocode(address=assignment[0], as_featureset=True)#Geocode
if len(geocode_result.features) > 0: #se o geocode encontrou pelo menos uma equivalencia, pega a primeira, se nao encontrou, passa para a proxima linha do CSV
geometry = geocode_result.features[0].geometry#teoricamenteo o lat long com maior assertividade
logger.info(geometry)#registra no,log
# Cria os outros atributos
# calcula a duedate em formato de milissegundos utc, adicionados do shift em horas da coluna [6]
horas = int(assignment[6])
prazo =(float(arrow.utcnow().shift(hours=horas).format('X')))*1000
#os atributos do dicionario devem ser criados conforme os fields do hosted service http://noteimg415.img.local/server/rest/services/Hosted/assignments_fadecb6e28be45f9b12048336a5a070b/FeatureServer/0
attributes = dict(location=assignment[0],
description=assignment[1],
priority=assignment[2],
assignmenttype=assignment[3],
status=assignment[4],
workorderid=assignment[5],
duedate=prazo,
notes=assignment[7],
assignmentread=None)
#adiciona a geometria e os atributos a um novo assignment
new_assignment = arcgis.features.Feature(geometry=geometry, attributes=attributes)
assignment_dict = (dict(assignment=new_assignment))
logger.info(assignment_dict)
assignments_to_add.append(assignment_dict)
#por fim, retorno todos os assignments que foram possiveis de geocodificar e preencher os atributos
return assignments_to_add
def crawl(file):
gis = GIS()
map = gis.map("United States")
map
# read all kinds of job files
job_df = pd.read_csv(Point_v1.CONSULTING_FILE).append(
pd.read_csv(Point_v1.DS_FILE)).append(
pd.read_csv(Point_v1.SDE_FILE))
company_loc_df = pd.DataFrame()
company_loc_df["company"] = job_df["company"].unique()
geo_info = company_loc_df["company"].apply(lambda company: geocode(company)[0] if geocode(company) else None)
company_loc_df['longitude'] = geo_info.apply(lambda info: info["location"]["x"] if info else None)
company_loc_df['latitude'] = geo_info.apply(lambda info: info["location"]["y"] if info else None)
company_loc_df['city'] = geo_info.apply(lambda info: info['attributes']['City'] if info else None)
company_loc_df['state'] = geo_info.apply(lambda info: info['attributes']['RegionAbbr'] if info else None)
company_loc_df.to_csv(file, encoding='utf-8', index=False)
def geocode_address(m_address, zipcode=None):
"""
:param m_address: address of interest in street form
:return: address in coordinate (X and Y) form
"""
WITH_ZIP = "{address}, State:Ma, Zone:{zipcode}"
WITHOUT_ZIP = "{address}, City:Boston, State:Ma"
m_address = WITH_ZIP.format(address=m_address, zipcode=zipcode) \
if zipcode is not None else WITHOUT_ZIP.format(address=m_address)
m_location = geocode(address=m_address)
confident = confidenceCheck(m_location)
if confident:
return m_location[0]['location']
if len(m_location) > 1 and not confident and zipcode is None:
zipcodes = set(
[address['address'].split(',')[3] for address in m_location])
raise MultipleAddressError(zipcodes)
elif len(m_location) > 1 and not confident and zipcode is not None:
raise BadAPIResponse
return m_location[0]['location']
def get_geo_coordinates_from_arcgis_with_login(self, location_address: str):
"""
purpose: Retrieve Latitude and Longitude to a Given Address/Location With Credentials
@param location_address: Latitude and Longitude needed Address/Location
@return: Dict
status: True or False based on success,
message: Error message if an error occurred
result:
{
'latitude': Latitude of the Address Provided
'longitude': Longitude of the Address Provided
'all_results': Whole Response Object Received From Arc GIS
}
"""
# If Username and Password Not Set, Return an error Object
# if self.username_password_flag:
# return self.__get_error_msg('Username or Password is not Set')
try:
# Establish Connection To ArcGIS Server Via GIS Library
GIS("http://www.arcgis.com", self.connection_params['username'], self.connection_params['password'])
arc_gis_loc = geocode(location_address)
if len(arc_gis_loc) > 0:
return {
'status': True,
'message': None,
'result': {
'longitude': arc_gis_loc[0]['location']['x'],
'latitude': arc_gis_loc[0]['location']['y'],
'all_results': arc_gis_loc
}
}
else:
return self.__get_error_msg('Unknown Location. No Results Found')
except ConnectionError:
return self.__get_error_msg('Connection Error')
except TypeError:
return self.__get_error_msg('Type Error')
except Exception as e:
return self.__get_error_msg('Unknown Error Occurred, Error: {}'.format(e))
def get_geocoded(address):
"""
get_geocode expects and address (i.e. 100 Seymour Ave, Utica, NY 13502) in the URL and will return the a json in a form as shown below.
{
"geometry": {
"x": -75.23401051692672,
"y": 43.08877505712876,
"spatialReference": {
"wkid": 4326,
"latestWkid": 4326
}
},
"attributes": {
"Loc_name": "World",
"Status": "M",
"Score": 95.18,
"Match_addr": "Seymour Ave, Utica, New York, 13501",
"LongLabel": "Seymour Ave, Utica, NY, 13501, USA",
"ShortLabel": "Seymour Ave",
"Addr_type": "StreetName",
"Type": "",
"PlaceName": "",
"Place_addr": "Seymour Ave, Utica, New York, 13501",
.
.
.
"X": -75.23401051692672,
"Y": 43.08877505712876,
"DisplayX": -75.23401051692672,
"DisplayY": 43.08877505712876,
"Xmin": -75.23501051692672,
"Xmax": -75.23301051692671,
"Ymin": 43.08777505712876,
"Ymax": 43.089775057128755,
"ExInfo": "100",
"OBJECTID": 1
}
"""
return json.dumps(
geocode(address=address, as_featureset=True).features[0].as_dict)
def main():
#create instance of GIS
gis = GIS()
#starting variables
score = ""
x = ""
y = ""
#get user input for the address
user_input = input("Please enter an address to geocode:\n")
#geocode address
geocode_result = geocode(address=user_input)
#split geocode by commas
geocode_result = str(geocode_result).split(',')
#take only first geocode result
geocode_result = geocode_result[:30]
#find each value in list, store in variable
y_next_line = False
for line in geocode_result:
#store 'y' as y
if (y_next_line):
y = line[6:len(line) - 1]
y_next_line = False
else:
#store 'Score' as score
if (line[:8] == " 'Score'"):
score = line[10:]
#store 'x' as x
if (line[:11] == " 'location'"):
x = line[19:]
#once x is found, y will be on the next line
y_next_line = True
#print latitide and longitude, as well as accuracy of the result
print("\nLongitude: " + x + "\nLatitude: " + y + "\nAccuracy: " + score +
"%")
def geocode_addr(addr, city):
"""
Given a string and a city, determine if the address is in Boston, MA
:param addr: string corresponding to the address
:param city: the city of interest
:return: boolean
"""
m_location = geocode(addr)
for location in m_location:
if location['score'] < 100:
continue
if location['attributes']['MetroArea'] \
and location['attributes']['MetroArea'] != city:
continue
if location['attributes']['City'] \
and location['attributes']['City'] not in NEIGHBORHOODS:
continue
return True
# No address found in the provided city
return False
def get_info(ad_url: str):
"""Get content of url and extract info from HTML."""
dic = {}
try:
r = requests.get(ad_url, verify=False)
soup = BeautifulSoup(r.text, "lxml")
data_price = soup.select_one('div[class^="wrap col-wrap date-cost"]')
p = str(data_price).split("Monat</strong>")[1].split("</p>")[0].strip()
assert "Agentur" not in p
address = soup.select_one('div[class^="wrap col-wrap adress-region"]')
ad = str(address).split("Adresse</strong>")[1].split("</p>")[0].strip()
loc = str(address).split("Ort</strong>")[1].split("</p>")[0].strip()
ad_coords = geocode(f"{ad}, {loc}")[0]["location"]
dic = {
"url": ad_url,
"loc": loc,
"address": ad,
"price": p,
"coords": ad_coords,
}
except IndexError as e:
print(f"Something f****d up: {e}")
return dic
def fetch_geocoding_data_with_caching(input_data, reverse=False):
global HIT_API_YET
if reverse == True:
# Convert longitude latitude pair into a string to serve as a key in the cache dictionary
input_string = str(input_data[0]) + ', ' + str(input_data[1])
else:
input_string = input_data
if input_string in CACHE_DICTION:
# print("** Pulling data from cache **")
return CACHE_DICTION[input_string]
else:
print("** Fetching new data from API **")
if not HIT_API_YET:
gis = GIS()
HIT_API_YET = True
if reverse == False:
data = geocode(input_string) # geocode() argument is a string
else:
data = reverse_geocode(input_data) # reverse_geocode() argument is a list
CACHE_DICTION[input_string] = data
cache_file_open = open(ARCGIS_CACHE_FILE_NAME, "w")
cache_file_open.write(json.dumps(CACHE_DICTION, indent=4))
cache_file_open.close()
return data
def main():
# ----------------- this section gets and updates estimated travel time data ------------------ #
account_key = constants.LTA_ACCOUNT_KEY # get variables from constants module
Est_Travel_Time_URL = constants.Est_Travel_Time_URL
Est_Travel_Time_Item_ID = constants.Est_Travel_Time_Item_ID
time.sleep(30) # wait for 30s to avoid exceeding rate limit
new_est_travel_time = CommonModules.request_from_LTA(
Est_Travel_Time_URL,
account_key) # request estimated travel time data from LTA
agol_connection = GIS(
constants.agol_home_url,
constants.agol_username,
constants.agol_password,
verify_cert=False) # connect to ArcGIS Online using your credentials
est_travel_time_layer = agol_connection.content.get(
Est_Travel_Time_Item_ID
).layers[
0] # get the first layer in the estimated travel time feature service
data_schema = list(new_est_travel_time[0].keys()
) # get list of dictionary keys as attribute fields
update_result = CommonModules.RemoveAndAppendNewData(
est_travel_time_layer,
new_est_travel_time,
data_schema,
location=False,
geometry="point"
) # remove existing records in feature service and apped new record
# -------------------- this section creates travel segments start-point features --------------------- #
highway_location_list = [
(f['Name'], f['StartPoint']) for f in new_est_travel_time
] # get a list of (expressway, location) tuple from StartPoint locations
highway_location_list += [
(f['Name'], f['EndPoint']) for f in new_est_travel_time
] # add the list of (expressway, location) tuple from EndPoint lcoations
highway_location_list = list(
set(highway_location_list)) # remove duplicates
highway_fullname = constants.highway_name_match # get the list of dicionaary of expressway short name - long name match
# %%
landmark_features = [] # initialize an empty list of ladmark features
for highway_location in highway_location_list: # iterate every (expressway, location) tuple
landmark_feature = {} # initialize an emtpy dictionary
landmark_feature["LocationName"] = highway_location[
1] # assign location value to LocationName field
landmark_feature["ExpresswayName"] = highway_location[
0] # assign expressway value to ExpresswayName field
search_string = highway_location[0] + " " + highway_location[
1] # create a search string for geocoding
try:
search_result = geocode(search_string)[
0] # get the geocoding result with the highest match point
except IndexError: # if geocoding returns no result, skip this record
continue
landmark_feature["Longitude"] = search_result["location"][
'x'] # assign coordinates of the geocoding result to Longitude/Latitude keys (to match to the expected format)
landmark_feature["Latitude"] = search_result["location"]['y']
landmark_features.append(
landmark_feature) # append new landmark feature to list
# %%
expressway_landmark_layer = agol_connection.content.get(
constants.Expressway_Landmark_Item_ID).layers[
0] # get the expressway landmark feature layer
landmark_schema = list(landmark_features[0].keys()
) # get the list of attribute fields as schema
update_result = CommonModules.RemoveAndAppendNewData(
expressway_landmark_layer,
landmark_features,
landmark_schema,
location=True,
geometry="point") # remove existing landmarks and append new landmarks
CommonModules.UpdateLastUpdatedTime(
agol_connection, "Estimated Travel Time"
) # update data last updated timestamp in feature service
# Find unique zip codes (219)
uniqueZip = prop_df['Zip Code'].unique()
n = len(uniqueZip)
# Print
print("Total number of rows: " + str(n) + "\n")
# Initialize List
listedList = []
# Initiate GIS service
gis = GIS()
# Loop through all zip codes in Cook County and save unique items with geocode
for id in range(n):
yourZip = geocode(str(uniqueZip[id]))[0]
searchedItem = geocode("Jewel Osco", yourZip['extent'], max_locations=1000)
print("ID - " + str(id), end=" : ")
print("ZIPCODE - " + str(uniqueZip[id]), end=" : ")
print("NUM - " + str(len(searchedItem)))
for item2 in range(len(searchedItem)):
listedList.append({
"ADDRESS":
searchedItem[item2]['attributes']['Place_addr'],
"PHONE":
searchedItem[item2]['attributes']['Phone'],
"LONGITUDE":
searchedItem[item2]['location']['x'],
"LATITUDE":
searchedItem[item2]['location']['y']
})
# set background
gis.admin.ux.set_background("style_files/background.jpg")
print(" Set org background")
# set banner
gis.admin.ux.set_banner("style_files/banner.jpg")
print(" Set org banner")
# set name
gis.admin.ux.name = args.name
gis.admin.ux.description = "Official WebGIS for first responders and public awareness"
print(" Set org name and description")
# set extent - geocode the county's extent
ventura_county = geocode('Ventura, USA', category='subregion')[0]
ext = {"type": "extent",
"xmin": ventura_county['extent']['xmin'],
"ymin": ventura_county['extent']['ymin'],
"xmax": ventura_county['extent']['xmax'],
"ymax": ventura_county['extent']['ymax'],
"spatialReference": {"wkid": 4326}
}
gis.update_properties({'defaultExtent': ext})
print(" Set org extent")
print("============================================================")
except Exception as style_ex:
print(str(style_ex))
