def insert_many_locations(locations_list): """ Executes multiple inserts at once, to reduce amount of time the database is locked :param locations_list: :return: """ if len(locations_list) >= number_entries_before_action: # with databaseSetup.database.atomic(): WOULD INCREASE SPEED, IMPORT PROBLEMS Locations.insert_many(locations_list).execute() print("Inserted " + str(len(locations_list)) + " Records") del locations_list[:]
def nominatim_parser(nominatim_response, longitude, latitude): ''' Look at Nomatimin.json as an example :param nominatim_response: :param longitude: :param latitude: :return: ''' nominatim_data = nominatim_response.get("address") building = nominatim_data.get("building") city = nominatim_data.get("city") country = nominatim_data.get("country") county = nominatim_data.get("county") street = nominatim_data.get("pedestrian") zipcode = nominatim_data.get("postcode") state = nominatim_data.get("state") area = nominatim_data.get("suburb") continent = continent_finder(latitude, longitude) provider_type = "Nominatim" northeast = [latitude, longitude] southwest = [latitude, longitude] location_to_dict(longitude_query=longitude, latitude_query=latitude, response=nominatim_response, type="Nominatim") return [Locations.insert(date=converted_time_stamp, time=time_stamp, longitude=longitude, latitude=latitude, continent=continent, country=country, state=state, zip=zipcode, area=area, county=county, city=city, street=street, name=building, provider=provider_type, bound_north=northeast[0], bound_east=northeast[1], bound_south=southwest[0], bound_west=southwest[1]), northeast, southwest, {"date": converted_time_stamp, "time": time_stamp, "longitude": longitude, "latitude": latitude, "continent": continent, "country": country, "state": state, "zip": zipcode, "area": area, "county": county, "city": city, "street": street, "name": building, "provider": provider_type, "bound_north": northeast[0], "bound_east": northeast[1], "bound_south": southwest[0], "bound_west": southwest[1]}]
def googleV3_parser(google_response, longitude, latitude): ''' Look at Google.json for an example :param longitude: :param latitude: :return: ''' google_data = google_response.get("address_components") street = "" area = "" city = "" county = "" country = "" state = "" zipcode = "" building = "" for piece in google_data: type_of_data = piece.get("types")[0] name = piece.get("long_name") if type_of_data == "route": street = name elif type_of_data == "neighborhood": area = name elif type_of_data == "locality": city = name elif type_of_data == "administrative_area_level_2": county = name elif type_of_data == "administrative_area_level_1": state = name elif type_of_data == "country": country = name elif type_of_data == "postal_code": zipcode = name elif type_of_data == "street_number": building = name continent = continent_finder(latitude, longitude) provider_type = "Google" bounds = google_response.get("geometry").get("viewport") northeast = [bounds.get("northeast").get("lat"), bounds.get("northeast").get("lng")] southwest = [bounds.get("southwest").get("lat"), bounds.get("southwest").get("lng")] location_to_dict(longitude_query=longitude, latitude_query=latitude, response=google_response, type="Google") return [Locations.insert(date=converted_time_stamp, time=time_stamp, longitude=longitude, latitude=latitude, continent=continent, country=country, state=state, zip=zipcode, area=area, county=county, city=city, street=street, name=building, provider=provider_type, bound_north=northeast[0], bound_east=northeast[1], bound_south=southwest[0], bound_west=southwest[1]), northeast, southwest, {"date": converted_time_stamp, "time": time_stamp, "longitude": longitude, "latitude": latitude, "continent": continent, "country": country, "state": state, "zip": zipcode, "area": area, "county": county, "city": city, "street": street, "name": building, "provider": provider_type, "bound_north": northeast[0], "bound_east": northeast[1], "bound_south": southwest[0], "bound_west": southwest[1]}]
def opencage_parser(opencage_response, longitude, latitude): ''' Look at OpenCage.json for an example :param opencage_response: :return: ''' opencage_data = opencage_response.get("components") building = opencage_data.get("house") if building is None: building = opencage_data.get("building") if building is None: building = opencage_data.get("house_number") city = opencage_data.get("city") country = opencage_data.get("country") county = opencage_data.get("county") street = opencage_data.get("road") if street is None: street = opencage_data.get("pedestrian") zipcode = opencage_data.get("postcode") state = opencage_data.get("state") area = opencage_data.get("suburb") continent = continent_finder(latitude, longitude) provider_type = "OpenCage" bounds = opencage_response.get("bounds") northeast = [bounds.get("northeast").get("lat"), bounds.get("northeast").get("lng")] southwest = [bounds.get("southwest").get("lat"), bounds.get("southwest").get("lng")] location_to_dict(longitude_query=longitude, latitude_query=latitude, response=opencage_response, type="OpenCage") return [Locations.insert(date=converted_time_stamp, time=time_stamp, longitude=longitude, latitude=latitude, continent=continent, country=country, state=state, zip=zipcode, area=area, county=county, city=city, street=street, name=building, provider=provider_type, bound_north=northeast[0], bound_east=northeast[1], bound_south=southwest[0], bound_west=southwest[1]), northeast, southwest, {"date": converted_time_stamp, "time": time_stamp, "longitude": longitude, "latitude": latitude, "continent": continent, "country": country, "state": state, "zip": zipcode, "area": area, "county": county, "city": city, "street": street, "name": building, "provider": provider_type, "bound_north": northeast[0], "bound_east": northeast[1], "bound_south": southwest[0], "bound_west": southwest[1]}]
try: print("OpenCage") # Try OpenCage first time.sleep(2) address = opencage_geolocator.reverse(point, exactly_one=True) provider = "OpenCage" with open("OpenCage.json", "a") as output: json.dump(address.raw, output, sort_keys=True, indent=4) output.write(",\n") response = opencage_parser(address.raw, longitude, latitude) response[0].execute() except GeocoderQuotaExceeded or GeocoderTimedOut or GeocoderServiceError: print("Could not access geocoders for location: " + point_string) break # Skips if cannot find locat if len(location_bulk_insert_queries) != 0: Locations.insert_many(location_bulk_insert_queries).execute() print("Inserted " + str(len(location_bulk_insert_queries)) + " Records") else: with open(os.path.join(rootdir, "LocationHistory.json"), 'r') as source: data = json.load(source) locations = data.get('locations') for key, location in enumerate(locations): time_stamp = location.get('timestampMs') converted_time_stamp = datetime.fromtimestamp(float(time_stamp) / 1000.0) longitude = location.get('longitudeE7') / 10000000.0 latitude = location.get('latitudeE7') / 10000000.0 point_string = str(latitude) + ", " + str(longitude) point = Point(latitude=latitude, longitude=longitude) if (key % number_entries_before_action) == 0: current_position_saver(key) if location_from_dict(longitude_query=longitude, latitude_query=latitude, type_query="Google"):