def interpret_paragraphs(ps):
toilets = []
for p in ps:
name = p[0]
address = ''
opening_hours = ''
for line in p[1:len(p)]:
if is_opening_hours_related(line):
opening_hours += line
else:
address += line.replace('Hgh','High').replace('Address: ','')
address = address.replace(' ',' ')
if len(address) > 0:
if address[0] == ' ':
address = address[1:len(address)]
latlng = Geocoder.geocode(address)
if latlng == "unavailable":
print(address + " Unavailable")
else:
toilet = {
'data_source': 'Extracted from https://www.newham.gov.uk/community-parks-leisure/public-toilets-newham/2 on 29/01/2021',
'borough': 'Newham',
'address': address,
'opening_hours': opening_hours,
'name': name,
'baby_change': True,
'latitude': float(latlng[0]),
'longitude': float(latlng[1]),
'wheelchair': True,
'covid': "Many community toilets have been affected by Covid restrictions and may not be currently operating"
}
toilets.append(toilet)
return toilets
def get_hounslow_toilets():
""" https://maps.hounslow.gov.uk/map/Aurora.svc/run?script=%5cAurora%5cFind_your_nearest_Public+Toilets
.AuroraScript%24&nocache=747441043&resize=always """
with open("Data/hounslow_raw.json") as dataFile:
jsonData = json.loads(dataFile.read())
htmlData = jsonData['Html']
lines = htmlData.split("\n")
lines = [cleanhtml(l).replace("\r", "") for l in lines]
toilets = []
t = {}
for line in lines:
print(line)
if "name" in line.lower():
t = {"name": line.replace("Name: ", "")}
if "address" in line.lower():
a = line.replace("Address: ", "")
t["address"] = clean_addr(a)
coords = Geocoder.geocode(a)
if coords != "unavailable":
t["data_source"] = "www.hounslow.gov.uk/publictoilets"
t["latitude"] = coords[0]
t["longitude"] = coords[1]
t["borough"] = "Hounslow"
toilets.append(t)
else:
print("UNAVAILABLE " + t["name"])
t = {}
with open("Data/processed_data_hounslow.json", "w") as dataFile:
json.dump(toilets, dataFile)
def healthmatic_excel_to_json():
"""Leads to geocoding errors as many addresses are not full"""
data = read_healthmatic_data()
boroughs = get_boroughs()
with open("Data/processed_data_healthmatic.json", 'w') as dataFile:
toilets = []
for t in data:
# Royal parks data was collected seperately
if t['Group'] != 'Royal Parks':
try:
full_address = Geocoder.reverse_geocode(
get_latitude(t), get_longitude(t))
borough = get_borough(boroughs, full_address)
toilet = {
'data_source':
"Dataset sent in by Healthmatic 26/10/20",
'address': get_address(t),
'opening_hours': get_opening_hours(t),
'name': get_name(t),
'baby_change': get_baby_change(t),
'latitude': get_latitude(t),
'longitude': get_longitude(t),
'wheelchair': get_disabled(t),
'fee': get_charge(t),
'borough': borough
}
toilets.append(toilet)
except:
print('Error reverse geocoding %s' % get_name(t))
json.dump(toilets, dataFile)
def geocode_address_database(connector, candidate_name):
cursor = connector.cursor()
relation_name = candidate_name + '_Addresses'
cursor.execute('SELECT max(Contributor_id) FROM ' + relation_name)
max_id = cursor.fetchone()[0]
for i in range(max_id):
cursor.execute(
'SELECT Name,Address FROM ' + relation_name +
' WHERE Contributor_id=? AND Geocoded=0', (i, ))
row = cursor.fetchone()
if row is None:
continue
else:
try:
address = row[1]
#coordinates = Geocoder.geocode_addresses_osm(address)
#coordinates = Geocoder.geocode_addresses_locationIQ(address, key=Hidden.locationIQ_api_key)
coordinates = Geocoder.geocode_addresses_mapbox(
address, key=Hidden.mapbox_api_token)
except:
connector.commit()
coordinates = [0, 0]
lat = coordinates[0]
lng = coordinates[1]
cursor.execute(
'UPDATE ' + relation_name +
' SET Latitude=?, Longitude=?, Geocoded=1 WHERE Contributor_id=?',
(lat, lng, i))
connector.commit()
connector.commit()
cursor.close()
return
def lewisham_excel_to_json():
'''Leads to a few geocoding errors as many addresses are not full - see web scraping solution below'''
data = read_lewisham_data()
print(data)
with open("Data/processed_data_lewisham.json", 'w') as dataFile:
toilets = []
for t in data:
name = get_name(t)
address = get_address(t)
disabled = get_disabled(t)
babychange = get_baby_change(t)
opening = get_opening_hours(t)
print(address)
latLng = Geocoder.geocode(address)
if latLng != "unavailable":
toilet = {
'data_source':"Lewisham Community Toilets dataset (updated for Covid)",
'borough':"Lewisham",
'address': address,
'opening_hours': opening,
'name' : name,
'baby_change' : babychange,
'latitude' : latLng[0],
'longitude' : latLng[1],
'wheelchair' : disabled
}
toilets.append(toilet)
else:
print("THERE WAS AN ERROR GEOCODING THIS TOILET'S ADDRESS")
json.dump(toilets, dataFile)
def geocode(request, txtLocation):
resultDict = Geocoder.geocode(txtLocation)
# resultDict = { txtLocation: a, longitude: b, latitude }
if resultDict["display_name"] == None:
resultDict["display_name"] = "Location Not Found"
JSON_Result = json.dumps(resultDict)
return HttpResponse(JSON_Result)
def geocode (request, txtLocation ):
resultDict = Geocoder.geocode ( txtLocation )
# resultDict = { txtLocation: a, longitude: b, latitude }
if resultDict["display_name"]== None:
resultDict["display_name"] = "Location Not Found"
JSON_Result = json.dumps ( resultDict )
return HttpResponse (JSON_Result)
def get_latLng(toilet):
longest_addr = toilet['Site Name'] + " " + toilet['Full Address']
medium_addr = toilet['Site Name'] + " " + get_postcode(toilet)
short_addr = toilet['Full Address']
shortest_addr = get_postcode(toilet)
for a in [longest_addr, medium_addr, short_addr, shortest_addr]:
latLng = Geocoder.geocode(a)
if latLng != "unavailable":
return latLng
return latLng
def keyboards_query(update: Update, context):
query = update.callback_query
data = query.data
current_text_i = update.effective_message.text.replace(
'\nБот: В этом месте вы можите узнать погоду', '')
if current_text_i.lower() == 'привет':
current_text_i = 'Hello city'
current_text_replace = current_text_i.replace(
'\n Бот: Что вам именно нужно узнать?', '')
if data == 'WEATHER':
query.edit_message_text(text=current_text_i +
'\n Бот: Что вам именно нужно узнать?\n',
parse_mode=ParseMode.MARKDOWN,
reply_markup=weather_list())
elif data == 'STATUS':
query.edit_message_text(
text=Geocoder.weather_stats(current_text_replace),
parse_mode=ParseMode.MARKDOWN,
)
elif data == 'TEMPERATURE':
query.edit_message_text(
text=Geocoder.weather_temperature(current_text_replace),
parse_mode=ParseMode.MARKDOWN,
)
elif data == 'SPEED_WIND':
query.edit_message_text(
text=Geocoder.weather_wind(current_text_replace),
parse_mode=ParseMode.MARKDOWN,
)
elif data == 'NOTHING':
query.edit_message_text(
text='Что тогда?',
parse_mode=ParseMode.MARKDOWN,
)
elif data == 'GEOLOCATION':
query.edit_message_text(text='Бот: Вот где это находится\n' +
current_text_replace + ' ' +
Geocoder.geo(current_text_replace),
parse_mode=ParseMode.MARKDOWN)
def get_data():
data = requests.get("https://content.tfl.gov.uk/lrad-v2.xml").text
root = ET.fromstring(data)
toilets = []
for station in root.findall("Station"):
latlngcoords = "NONE"
name = station.find("StationName").text
i = 0
for toilet in station.iter('Toilet'):
i += 1
if i > 0:
latlngcoords = latlng(name)
if latlngcoords != "NONE":
full_addr = Geocoder.reverse_geocode(latlngcoords[0],
latlngcoords[1])
borough = get_borough(full_addr)
FEE = ""
LOCATION = ""
INSIDEGATE = False
ACCESSIBLE = False
BABYCHANGE = False
for toilet in station.iter('Toilet'):
for child in toilet:
if child.tag == "FeeCharged":
FEE = "Paid" if "y" in child.text.lower() else ""
elif child.tag == "Location":
if LOCATION == "":
LOCATION = child.text
elif child.text != LOCATION:
LOCATION += child.text + ", "
elif child.tag == "InsideGateLine":
INSIDEGATE = INSIDEGATE and ("y" in child.text.lower())
elif child.tag == "Accessible":
ACCESSIBLE = ACCESSIBLE or ("y" in child.text.lower())
elif child.tag == "BabyChanging":
BABYCHANGE = BABYCHANGE or ("y" in child.text.lower())
fullname = name
if INSIDEGATE:
fullname += " Inside Gate"
toilets.append({
"name": fullname,
"baby_change": BABYCHANGE,
"wheelchair": ACCESSIBLE,
"address": name + " " + LOCATION,
"fee": FEE,
"latitude": latlngcoords[0],
"longitude": latlngcoords[1],
"borough": borough
})
with open("Data/processed_data_tfl2.json", "w") as datafile:
json.dump(toilets, datafile)
def run(self):
self.root.console.config(state='normal')
self.root.console.delete(1.0, 'end')
self.root.console.config(state='disable')
print('执行地址解析工具...')
cf = configparser.ConfigParser()
cf.read('property.ini')
cf.set('Geocoder', 'key', self.e_key.get())
cf.set('Geocoder', 'inputfilepath', self.e_inputFile.get())
cf.set('Geocoder', 'outdirpath', self.e_outputDir.get())
cf.write(open('property.ini', 'w'))
key = self.e_key.get()
inputFilePath = self.e_inputFile.get()
outDirPath = self.e_outputDir.get()
coder.Geocoder(key, inputFilePath, outDirPath)
def echo_message(update, context):
answer_unstnd = choice(text_know)
current_text_i = update.message.text
try:
print(Geocoder.weather_stats(current_text_i)) # Ошибка, если не город
context.bot.send_message(
chat_id=update.effective_chat.id,
text=' {}\nБот: В этом месте вы можите узнать погоду'.format(
current_text_i),
reply_markup=weather_city())
except:
context.bot.send_message(chat_id=update.effective_chat.id,
text=update.message.text + ' \nБот: ' +
answer_unstnd)
def redbridge_excel_to_json():
data = read_redbridge_data()
with open("Data/processed_data_redbridge.json", 'w') as dataFile:
toilets = []
for t in data:
latLng = Geocoder.geocode(get_address(t))
if latLng != "unavailable":
toilet = {
'data_source' : 'Redbridge Culture and Leisure sent in spreadsheet 6/10/2020',
'borough':'Redbridge',
'address': get_address(t),
'opening_hours': get_opening_hours(t),
'name' : get_name(t),
'baby_change' : False,
'latitude' : latLng[0],
'longitude' : latLng[1],
'wheelchair' : get_disabled(t)
}
toilets.append(toilet)
json.dump(toilets, dataFile)
def getLocationNamesAndTimeZones(self):
import Geocoder
cn = psycopg2.connect(secret.DB_CONNECT)
cr = cn.cursor()
g = Geocoder.PyGeoNames()
cr.execute("SELECT latitude,longitude,location_id FROM locations")
for latlong in cr.fetchall():
placename = g.reverse(latlong[0:2])
timezone = g.timezone(latlong[0:2])
print placename, timezone
if len(placename) > 100: placename = placename[0:99]
cr.execute("UPDATE locations\
SET placename = (%s),\
timezone = (%s)\
WHERE location_id = (%s);" ,\
(placename,timezone,latlong[2]))
cn.commit()
cn.close()
def enfield_data_to_json():
data = get_data()
rows = get_rows(data)[1:]
with open('Data/processed_data_enfield.json', 'w') as dataFile:
toilets = []
for toilet in rows:
t = {}
t['name'] = 'Community Toilet'
t['address'] = toilet[0]
t['opening_hours'] = toilet[1]
t['data_source'] = "https://new.enfield.gov.uk/services/leisure-and-culture/community-toilet-scheme/"
t['borough'] = 'Enfield'
ll = Geocoder.geocode(toilet[0])
if ll != "unavailable":
t['latitude'] = ll[0]
t['longitude'] = ll[1]
else:
t['latitude'] = 0
t['longitude'] = 0
toilets.append(t)
json.dump(toilets, dataFile)
def barnet_libraries_csv_to_json():
data = read_barnet_data()
with open("Data/processed_barnet_libraries.json", 'w') as dataFile:
toilets = []
for t in data:
ltlng = Geocoder.geocode(t[LOCATION])
if ltlng != "unavailable" and "library" in t[LOCATION].lower():
toilet = {
'data_source': 'Downloaded from https://data.gov.uk/',
'borough': 'Barnet',
'address': t[LOCATION].replace("opening_hours=", ""),
'opening_hours':
t[OPENING_HOURS].replace("opening_hours=", ""),
'name': "Library toilets",
'latitude': ltlng[0],
'longitude': ltlng[1],
'open': True
}
toilets.append(toilet)
else:
pass
json.dump(toilets, dataFile)
def filter_json_data():
"""Return a new list of dicts with cleaned and filtered data to match input into toilets4london backend"""
wcs = []
dict = load_all_json("Data/data.json")
toilets = dict['elements']
for t in toilets:
toilet_tags = t['tags']
filtered_dict = {}
address = Geocoder.reverse_geocode(t['lat'],t['lon'])
filtered_dict['data_source'] = get_source(toilet_tags)
filtered_dict['address'] = address
filtered_dict['latitude'] = t['lat']
filtered_dict['longitude'] = t['lon']
filtered_dict['borough'] = get_borough(address)
filtered_dict['wheelchair'] = is_wheelchair_accessible(toilet_tags)
filtered_dict['name'] = get_name(toilet_tags)
filtered_dict['opening_hours'] = get_hours(toilet_tags)
filtered_dict['baby_change'] = get_baby_change(toilet_tags)
wcs.append(filtered_dict)
return wcs
def get_tfl_toilets():
with open("Data/transport_for_london_raw.json", "r") as rawDataFile:
raw_data = json.loads(rawDataFile.read())
toilets = []
for point in raw_data:
if has_toilets(point):
full_addr = Geocoder.reverse_geocode(point[LATITUDE],
point[LONGITUDE])
borough = get_borough(full_addr)
if borough != "Other":
addr = full_addr.replace(", London, Greater London, England",
"").replace(", United Kingdom", "")
t = {
'name': point[NAME],
'latitude': point[LATITUDE],
'longitude': point[LONGITUDE],
'data_source': 'Transport For London Open Data API',
'address': addr,
'opening_hours': get_opening_hours(point),
'borough': borough
}
toilets.append(t)
with open("Data/processed_data_tfl.json", "w") as dataFile:
json.dump(toilets, dataFile)
def fix_broken_addresses(connector, candidate_name):
cursor = connector.cursor()
relation_name = candidate_name + '_Addresses'
compressed_relation_name = candidate_name + '_Contributions_compressed'
ids_to_be_fixed = dict()
ids_to_be_removed = list()
cursor.execute(
'SELECT Contributor_id, Address, Latitude, Longitude FROM ' +
relation_name + ' WHERE Latitude<25 OR Latitude>49 OR Longitude>-65')
for row in cursor:
contributor_id = row[0]
address = str(row[1])
lat = row[2]
lng = row[3]
if re.search('APO, AE', address) or re.search(
'APO, AA',
address) or re.search('APO, AP', address) or re.search(
'FPO, AP',
address) or re.search('FPO, AE', address) or re.search(
'FPO, AS', address) or re.search(
'FPO, AA', address) or re.search(
'DPO, AE', address) or re.search(
'nan, nan, nan nan', address) or re.search(
'INFORMATION REQUESTED', address):
ids_to_be_removed.append(contributor_id)
continue
if re.search(' AK ', address) or re.search(
' HI ', address) or re.search(' FL ', address) or re.search(
' PR ',
address) or re.search(' GU ', address) or re.search(
'SAIPAN',
address) or re.search(' VI ', address) or re.search(
' ZZ ', address) or re.search(' AS ', address):
if lat == 0 or lng == 0:
pass # needs to be re-geocoded
else:
continue
if re.search('PO BOX', address):
split_address = re.findall('.*, (.*,.*)', address)
address = ''
if split_address is not None:
for piece in split_address:
address += piece + ' '
address = address.rstrip()
if re.search('nan, .*', address):
split_address = re.findall('nan, (.*)', address)
address = split_address[0]
if re.search(' CT ', address) or re.search(
' MA ', address) or re.search(' ME ', address) or re.search(
' NH ',
address) or re.search(' NJ ', address) or re.search(
' RI ', address) or re.search(' VT ', address):
zipcode = re.findall('.* ([0-9]*)', address)
if zipcode is not None and len(zipcode[0]) == 4:
new_zip = '0' + zipcode[0]
address = address.replace(zipcode[0], new_zip)
ids_to_be_fixed[contributor_id] = address
print('TO BE REMOVED:', len(ids_to_be_removed))
for c_id in ids_to_be_removed:
print(c_id)
cursor.execute(
'DELETE FROM ' + relation_name + ' WHERE Contributor_id=?',
(c_id, ))
connector.commit()
print('TO BE FIXED:', len(ids_to_be_fixed))
for c_id, address in ids_to_be_fixed.items():
print(c_id, ' - ', address)
try:
coordinates = Geocoder.geocode_addresses_google(
address, Hidden.google_api_key)
except:
coordinates = [0, 0]
lat = coordinates[0]
lng = coordinates[1]
cursor.execute(
'UPDATE ' + relation_name +
' SET Latitude=?, Longitude=? WHERE Contributor_id=?',
(lat, lng, c_id))
connector.commit()
cursor.close()
return
result, added, removed, updates = Calls.merge(active_calls[src.id], calls)
active_calls[src.id] = result.values()
report = Reporting.SourceUpdateReport(src, result, added, removed, updates)
# Show the changes
if report.hasChanges():
report.printChanges()
# Update the server with new data
if c.UseRemoteServer and len(c.IngestUrl) > 0:
report.sendReport(c.IngestUrl, c.ClientKey)
# Handle geocode requests
if Geocoder.canHandleRequests(c):
requests = Geocoder.getRequests(c.DataUrl)
if len(requests) > 0:
# Resolve all of the requests
for request in requests:
if not request.resolved and not Geocoder.quota_exceeded:
if request.tryResolve(c.GeoApiUrl, c.GeoApiKey):
print("Geocode resolved: {0} -> {1}".format(request.location, request.getFormattedAddress()))
else:
print("Geocode failed: {0} -> {1}".format(request.location, request.getError()))
# Report the results
report = Reporting.GeocodeReport(requests)
if len(report.getData().items()) > 0:
if report.sendReport(c.IngestUrl, c.ClientKey):