def insert_many_locations(locations_list):
"""
Executes multiple inserts at once, to reduce amount of time the database is locked
:param locations_list:
:return:
"""
if len(locations_list) >= number_entries_before_action:
# with databaseSetup.database.atomic(): WOULD INCREASE SPEED, IMPORT PROBLEMS
Locations.insert_many(locations_list).execute()
print("Inserted " + str(len(locations_list)) + " Records")
del locations_list[:]
def nominatim_parser(nominatim_response, longitude, latitude):
'''
Look at Nomatimin.json as an example
:param nominatim_response:
:param longitude:
:param latitude:
:return:
'''
nominatim_data = nominatim_response.get("address")
building = nominatim_data.get("building")
city = nominatim_data.get("city")
country = nominatim_data.get("country")
county = nominatim_data.get("county")
street = nominatim_data.get("pedestrian")
zipcode = nominatim_data.get("postcode")
state = nominatim_data.get("state")
area = nominatim_data.get("suburb")
continent = continent_finder(latitude, longitude)
provider_type = "Nominatim"
northeast = [latitude, longitude]
southwest = [latitude, longitude]
location_to_dict(longitude_query=longitude, latitude_query=latitude, response=nominatim_response, type="Nominatim")
return [Locations.insert(date=converted_time_stamp, time=time_stamp, longitude=longitude, latitude=latitude,
continent=continent, country=country, state=state, zip=zipcode, area=area, county=county,
city=city, street=street, name=building, provider=provider_type, bound_north=northeast[0],
bound_east=northeast[1], bound_south=southwest[0], bound_west=southwest[1]), northeast,
southwest, {"date": converted_time_stamp, "time": time_stamp, "longitude": longitude, "latitude": latitude,
"continent": continent, "country": country, "state": state, "zip": zipcode, "area": area,
"county": county, "city": city, "street": street, "name": building, "provider": provider_type,
"bound_north": northeast[0], "bound_east": northeast[1], "bound_south": southwest[0],
"bound_west": southwest[1]}]
def googleV3_parser(google_response, longitude, latitude):
'''
Look at Google.json for an example
:param longitude:
:param latitude:
:return:
'''
google_data = google_response.get("address_components")
street = ""
area = ""
city = ""
county = ""
country = ""
state = ""
zipcode = ""
building = ""
for piece in google_data:
type_of_data = piece.get("types")[0]
name = piece.get("long_name")
if type_of_data == "route":
street = name
elif type_of_data == "neighborhood":
area = name
elif type_of_data == "locality":
city = name
elif type_of_data == "administrative_area_level_2":
county = name
elif type_of_data == "administrative_area_level_1":
state = name
elif type_of_data == "country":
country = name
elif type_of_data == "postal_code":
zipcode = name
elif type_of_data == "street_number":
building = name
continent = continent_finder(latitude, longitude)
provider_type = "Google"
bounds = google_response.get("geometry").get("viewport")
northeast = [bounds.get("northeast").get("lat"), bounds.get("northeast").get("lng")]
southwest = [bounds.get("southwest").get("lat"), bounds.get("southwest").get("lng")]
location_to_dict(longitude_query=longitude, latitude_query=latitude, response=google_response, type="Google")
return [Locations.insert(date=converted_time_stamp, time=time_stamp, longitude=longitude, latitude=latitude,
continent=continent, country=country, state=state, zip=zipcode, area=area, county=county,
city=city, street=street, name=building, provider=provider_type, bound_north=northeast[0],
bound_east=northeast[1], bound_south=southwest[0], bound_west=southwest[1]), northeast,
southwest, {"date": converted_time_stamp, "time": time_stamp, "longitude": longitude, "latitude": latitude,
"continent": continent, "country": country, "state": state, "zip": zipcode, "area": area,
"county": county, "city": city, "street": street, "name": building, "provider": provider_type,
"bound_north": northeast[0], "bound_east": northeast[1], "bound_south": southwest[0],
"bound_west": southwest[1]}]
def opencage_parser(opencage_response, longitude, latitude):
'''
Look at OpenCage.json for an example
:param opencage_response:
:return:
'''
opencage_data = opencage_response.get("components")
building = opencage_data.get("house")
if building is None:
building = opencage_data.get("building")
if building is None:
building = opencage_data.get("house_number")
city = opencage_data.get("city")
country = opencage_data.get("country")
county = opencage_data.get("county")
street = opencage_data.get("road")
if street is None:
street = opencage_data.get("pedestrian")
zipcode = opencage_data.get("postcode")
state = opencage_data.get("state")
area = opencage_data.get("suburb")
continent = continent_finder(latitude, longitude)
provider_type = "OpenCage"
bounds = opencage_response.get("bounds")
northeast = [bounds.get("northeast").get("lat"), bounds.get("northeast").get("lng")]
southwest = [bounds.get("southwest").get("lat"), bounds.get("southwest").get("lng")]
location_to_dict(longitude_query=longitude, latitude_query=latitude, response=opencage_response, type="OpenCage")
return [Locations.insert(date=converted_time_stamp, time=time_stamp, longitude=longitude, latitude=latitude,
continent=continent, country=country, state=state, zip=zipcode, area=area, county=county,
city=city, street=street, name=building, provider=provider_type, bound_north=northeast[0],
bound_east=northeast[1], bound_south=southwest[0], bound_west=southwest[1]), northeast,
southwest, {"date": converted_time_stamp, "time": time_stamp, "longitude": longitude, "latitude": latitude,
"continent": continent, "country": country, "state": state, "zip": zipcode, "area": area,
"county": county, "city": city, "street": street, "name": building, "provider": provider_type,
"bound_north": northeast[0], "bound_east": northeast[1], "bound_south": southwest[0],
"bound_west": southwest[1]}]
try:
print("OpenCage")
# Try OpenCage first
time.sleep(2)
address = opencage_geolocator.reverse(point, exactly_one=True)
provider = "OpenCage"
with open("OpenCage.json", "a") as output:
json.dump(address.raw, output, sort_keys=True, indent=4)
output.write(",\n")
response = opencage_parser(address.raw, longitude, latitude)
response[0].execute()
except GeocoderQuotaExceeded or GeocoderTimedOut or GeocoderServiceError:
print("Could not access geocoders for location: " + point_string)
break # Skips if cannot find locat
if len(location_bulk_insert_queries) != 0:
Locations.insert_many(location_bulk_insert_queries).execute()
print("Inserted " + str(len(location_bulk_insert_queries)) + " Records")
else:
with open(os.path.join(rootdir, "LocationHistory.json"), 'r') as source:
data = json.load(source)
locations = data.get('locations')
for key, location in enumerate(locations):
time_stamp = location.get('timestampMs')
converted_time_stamp = datetime.fromtimestamp(float(time_stamp) / 1000.0)
longitude = location.get('longitudeE7') / 10000000.0
latitude = location.get('latitudeE7') / 10000000.0
point_string = str(latitude) + ", " + str(longitude)
point = Point(latitude=latitude, longitude=longitude)
if (key % number_entries_before_action) == 0:
current_position_saver(key)
if location_from_dict(longitude_query=longitude, latitude_query=latitude, type_query="Google"):
