def CrawlPopularity(request):
popdata_library = populartimes.get(
"AIzaSyB0RALv_IqPawmS5eeLWfJBVtcAE8vCW98", ["library"],
(40.116193536286815, -88.2435300726317),
(40.08857770924527, -88.2047958483285))
popdata_cafe = populartimes.get("AIzaSyB0RALv_IqPawmS5eeLWfJBVtcAE8vCW98",
["cafe"],
(40.116193536286815, -88.2435300726317),
(40.08857770924527, -88.2047958483285))
# for item1 in popdata_library:
# facility = populartimes.get_id("AIzaSyB0RALv_IqPawmS5eeLWfJBVtcAE8vCW98",item1['id'])
# if 'current_popularity' not in facility:
# facility['current_popularity'] = None
# print(item1['name'], item1['coordinates']['lng'], item1['coordinates']['lat'], facility['current_popularity'])
to_weekday = {
1: "Monday",
2: "Tuesday",
3: "Wednesday",
4: "Thursday",
5: "Friday",
6: "Saturday",
7: "Sunday"
}
today = to_weekday[datetime.datetime.today().weekday()]
with connection.cursor() as cursor:
for item1, item2 in zip(popdata_library, popdata_cafe):
library = populartimes.get_id(
"AIzaSyB0RALv_IqPawmS5eeLWfJBVtcAE8vCW98", item1['id'])
cafe = populartimes.get_id(
"AIzaSyB0RALv_IqPawmS5eeLWfJBVtcAE8vCW98", item2['id'])
if 'current_popularity' not in library:
library['current_popularity'] = 0
if 'current_popularity' not in cafe:
cafe['current_popularity'] = 0
try:
cursor.execute("INSERT INTO Place(name, longitude, latitude, crowdedness, type) VALUE (%s, %s, %s, %s, %s)",\
[str(item1['name']), \
str(item1['coordinates']['lng']), str(item1['coordinates']['lat']), \
str(library['current_popularity']),\
"Library"])
except:
pass
try:
cursor.execute("INSERT INTO Place(name, longitude, latitude, crowdedness, type) VALUE (%s, %s, %s, %s, %s)",\
[str(item2['name']), \
str(item2['coordinates']['lng']), str(item2['coordinates']['lat']), \
str(cafe['current_popularity']),
"Cafe"])
except:
pass
return redirect("user-detail")
def get_places_ranked_by_popularity(api_key, location, radius, types):
req_url = "https://maps.googleapis.com/maps/api/place/nearbysearch/json"
search_params = {
"key": api_key,
"location": "{},{}".format(location[0], location[1]),
"radius": radius,
"type": types
}
response = requests.get(req_url, params=search_params)
response_json = json.loads(response.text)
results = response_json["results"]
places = []
for place in results:
places.append((place["name"], place["place_id"]))
popularity_ranked_list = []
for place in places:
place_data = populartimes.get_id(api_key, place[1])
if "populartimes" in place_data:
for day in place_data["populartimes"]:
if day["name"] == 'Friday':
popularity_ranked_list.append(
(place_data["name"], day["data"][0]))
popularity_ranked_list.sort(key=lambda x: x[1], reverse=True)
print(popularity_ranked_list)
return places
def get_popularity():
# print("hey")
current_day = datetime.datetime.today().weekday()
obj = request.get_json()
try:
place_id = obj['place']
print("got it")
place_data = populartimes.get_id(google_key, place_id)
pop_times = place_data['populartimes']
return jsonify(pop_times[current_day])
# if (len(pop_times) >= current_day):
# print("got it")
# return jsonify(pop_times[current_day])
except:
print("could not get id")
# place_id = json.loads(obj)['place']
# place_id = obj['place']
# place_data = populartimes.get_id(google_key, place_id)
# print(place_data)
# pop_times = place_data['populartimes']
# if (len(pop_times) >= current_day):
# return jsonify(pop_times[current_day])
# return jsonify({})
return jsonify({"1": "a"})
def create_popular_dataframe(GOOGLE_API_KEY, places_id_list, print_mode=False):
"""Dataframe for cleaned API rensponse (simpler dict) + timestamps."""
response_list = [
] # original responses as list, could be saved as pickle file if wanted
reponse_dict_list = []
for idx, place_id in enumerate(places_id_list):
#time.sleep(10)
dt_now = datetime.datetime.now()
str_now = dt_now.strftime('%Y-%m-%d_%H%M%S')
try:
response = populartimes.get_id(GOOGLE_API_KEY, place_id)
print('get place {}: {}'.format(idx, response.get('name', 'None')))
response_list.append(response)
clean_response = get_clean_response(response)
clean_response[
'timestamp'] = dt_now #str_now # Put datetime in response dict
clean_response = OrderedDict(
clean_response) # to set timestamp to beginning
clean_response.move_to_end('timestamp', last=False)
reponse_dict_list.append(clean_response)
if print_mode:
print('Response for Time {}:'.format(str_now))
for (key, val) in clean_response.items():
print('{}: {}'.format(key, val))
print(clean_response)
except:
print('not worked for this one')
df_response = pd.DataFrame(reponse_dict_list)
return df_response, response_list
def fetch_new_info(placeid):
response = populartimes.get_id(get_api_key(),placeid)
new_info = {
'id' : response['id'],
'name' : response['name'],
'lat' : response['coordinates']['lat'],
'lng': response['coordinates']['lng']
}
optfields = [
'rating',
'rating_n',
'types',
'address'
]
for field in optfields:
if field not in response:
print('no {}'.format(field))
else:
new_info[field] = response[field]
if 'time_spent' not in response:
print('no time spent')
else:
new_info['avg_time_spent'] = mean(response['time_spent'])
return new_info
def current_crowd(id):
try:
pop = populartimes.get_id(key, id)
current = pop["current_popularity"]
#popular_time = pop["populartimes"]
return current
except:
return -1
def get_pop_times_from_id(api_key, place_id):
try:
pop_times = populartimes.get_id(api_key, place_id)
except:
return None
if "populartimes" in pop_times:
return json.dumps(pop_times["populartimes"])
else:
return None
def main():
place_id = getID_Nearby(data)
placeData = populartimes.get_id(api_key, place_id)
nearbyPopVal = (placeData["populartimes"][2]) ["data"][9]
sys.stdout = open('nearINFO', 'w')
print(getName_Nearby(data), end=" - Popularity of: ")
print(nearbyPopVal)
def scrapeData(coords):
popularresults = []
for key in coords['ids']:
result = populartimes.get_id(config.api_key, key)
if "populartimes" in result:
popularresults.append(result)
filename = 'public/data/' + coords['name'] + '.json'
with open(filename, 'w') as outfile:
json.dump(popularresults, outfile)
def get_popular_times_helper(self, place_id):
data = []
counter = 0
try:
for i in dict(populartimes.get_id(API_KEY,
place_id))['populartimes']:
if (counter == self.day):
return i['data']
counter += 1
except:
return None
def get_place_detail(instance):
try:
place_detail = populartimes.get_id(API_KEY, instance.placeId)
except:
return None
popular_times = get_popular_times(place_detail)
lat, lon = get_lat_lon(place_detail)
rating = get_rating(place_detail)
return popular_times, lat, lon, rating
def writeToJSONFile(fileName, id_lst):
filePathNameWExt = './' + fileName + '.json'
item_dict = {"cafe_list": []}
count = 0
for loc_id in id_lst:
loc_info = populartimes.get_id(api_key, loc_id)
loc_info["distance"] = "0.5"
loc_info["status"] = "gray"
loc_info["number"] = count
item_dict["cafe_list"].append(loc_info)
count += 1
with open(filePathNameWExt, 'w') as fp:
json.dump(item_dict, fp, indent=2)
def get_airport_popular_times(airport):
"https://maps.googleapis.com/maps/api/js?key=YOUR_API_KEY"
print("getting", file=sys.stderr)
places = google_places.nearby_search(keyword="IATA: " + airport["IATA"],
lat_lng={
'lat': airport["Latitude"],
'lng': airport["Longitude"]
},
radius=10000,
types=[types.TYPE_AIRPORT]).places
if len(places):
# todo: cache ids
airport_id = places[0].place_id
return populartimes.get_id(API_KEY, airport_id)["populartimes"]
def execute(self, bot, update, args):
if len(args) != 1:
bot.send_message(
chat_id=update.message.chat_id,
text="This doesn't seem like correct usage of /popular.",
disable_notification=True)
return
gparams = {"query": args[0], "key": self.apikey}
req = requests.get(
"https://maps.googleapis.com/maps/api/place/textsearch/json",
params=gparams)
place_info = json.loads(req.text)["results"]
if len(place_info) == 0:
bot.send_message(
chat_id=update.message.chat_id,
text=
"Unfortunately, I got no results for the place you searched.",
disable_notification=True)
return
place_id = place_info[0]["place_id"]
poptimes = populartimes.get_id(self.apikey, place_id)
out = "Busy times for {}:\n".format(poptimes['name'])
if len(poptimes['populartimes']) == 0:
out += " - Unavailable!\n"
for day in poptimes['populartimes']:
out += " - {}: ".format(day['name'])
# Order so the busy time list is from 6am to 5am
data = [x >= 50 for x in day['data'][6:] + day['data'][:6]]
busy = []
for k, v in groupby(enumerate(data), key=lambda x: x[1]):
if k:
v = list(v)
start, end = divmod(v[0][0] + 6,
12), divmod(v[-1][0] + 6, 12)
tstr = "{}{}m".format(12 if end[1] == 0 else end[1],
'p' if end[0] == 1 else 'a')
if start != end:
tstr = "{}{}m-".format(
12 if start[1] == 0 else start[1],
'p' if start[0] == 1 else 'a') + tstr
busy.append(tstr)
out += "{}\n".format(', '.join(busy) if len(busy) > 0 else "None")
if 'current_popularity' in poptimes.keys():
cur = poptimes['current_popularity']
is_busy = 'Not busy' if cur < 50 else 'Busy'
out += "\nCurrent busyness: {} ({}%)".format(is_busy, cur)
bot.send_message(chat_id=update.message.chat_id,
text=out,
disable_notification=True)
def filter_popularity_available(place_ids):
result = []
#print(len(place_ids))
for x, place_id in enumerate(tqdm(place_ids)):
try:
data = populartimes.get_id(api_key, place_id)
except Exception as e:
print(e)
print("Error with " + str(place_id))
continue
#print(data)
if "current_popularity" in data:
#print("found")
result.append(place_id)
return result
def on_place_create(data, context):
path_parts = context.resource.split('/documents/')[1].split('/')
collection_path = path_parts[0]
document_path = '/'.join(path_parts[1:])
affected_doc = client.collection(collection_path).document(document_path)
place_id = data["value"]["fields"]["placeId"]["stringValue"]
print(f"Attempting to find popular times for {place_id}")
populartimes_data = populartimes.get_id(MAPSKEY, place_id) or {}
print(f"Received the following data {populartimes_data}")
affected_doc.update({
u'busyTimes': populartimes_data.get("populartimes"),
u'waitTimes': populartimes_data.get("time_wait"),
u'avgSpentTimes': populartimes_data.get("time_spent"),
})
def placebusyness():
data = json.loads(request.data)
place_id = data['placeID']
dayOfWeek = data['dayOfWeek'] # first capitalized day of week
place = populartimes.get_id("AIzaSyC5xsFHLdEzj_3Ce8l8A2eqxDB0EvAvyQ8",
place_id)
if "populartimes" not in place:
return jsonify({"response": "NOT_FOUND"})
val = []
for day in place["populartimes"]:
if (day["name"] == dayOfWeek):
val = day["data"]
break
lat = float(place["coordinates"]["lat"])
lng = float(place["coordinates"]["lng"])
response = {"latitude": lat, "longitude": lng, "percentage": val}
return jsonify(response)
def api_call():
#Enable Key Cycling
with open("api_keys.txt") as f:
api_keys = cycle([key.strip() for key in f.readlines()])
place_ids = [
"ChIJuVGxxf51nkcRwhxFwvIr7EM",
"ChIJ4dic-71RqEcRZIsmE3K6r-0",
"ChIJmY9JK6Ulv0cRkNik00YUhZU",
"ChIJmZ8cBbbCuEcRecCyfQofums",
"ChIJ4___HmbPCUcRN5Ub3P2ZfmQ",
"ChIJ25xRQB9OqEcRiDCQLqrmcbA",
"ChIJX34c7jLbmUcRhaEUxVf0H3w",
"ChIJraB7riH4pkcRGXSvqsL52lM",
"ChIJcaYkt1gXuUcRPhoGAkKO10k",
"ChIJe9nAXKZXn0cRGUlOPBMiDIo",
"ChIJDSw1eVUJvUcRWndDicZ7OLo",
"ChIJext9rehXn0cRUISYYjjnrnE"
]
session = Session()
for x, place_id in enumerate(place_ids):
time.sleep(80)
try:
key = next(api_keys)
data = populartimes.get_id(key, place_id)
except Exception as e:
print(e)
print("Error with key: " + key)
continue
print(data["name"])
if "current_popularity" in data:
entry = Entry(
data["name"],
data["id"],
data["populartimes"][datetime.now().weekday()]["data"][datetime.now().hour],
data["current_popularity"]
)
session.add(entry)
else:
print("No Popularity-Data for " + data["name"])
session.commit()
#Lets us know how many Calls we have made so far.
print("There are now " + str(session.query(Entry).count()) + " Entries in the Database")
session.close()
def returnPoptimes(day, hour, location):
if (returnPlaceType(location) != 'locality'
and returnPlaceType(location) != 'sublocality'
and returnPlaceType(location) != 'administrative_area_level_1'
and returnPlaceType(location) != 'administrative_area_level_2'
and returnPlaceType(location) != 'administrative_area_level_3'
and returnPlaceType(location) != 'administrative_area_level_4'
and returnPlaceType(location) != 'administrative_area_level_5'
and returnPlaceType(location) != 'sublocality_level_1'
and returnPlaceType(location) != 'sublocality_level_2'
and returnPlaceType(location) != 'sublocality_level_3'
and returnPlaceType(location) != 'sublocality_level_4'
and returnPlaceType(location) != 'intersection'):
# periodPos = hour.find('M') -1
# timePeriod = hour[periodPos:]
# semiPos = hour.find(':')
# hour = int(hour[:semiPos])
# if timePeriod == "PM": #converts to military time
# hour+=12
# if hour == 12: #changes 12 to 0
# hour = 0
# if hour == 24: #changes 24 to 12
# hour = 12
loc = hour.find(':')
hour = hour[0:loc]
print("time is: ", hour)
dayNum = {
'Monday': 0,
'Tuesday': 1,
'Wednesday': 2,
'Thursday': 3,
'Friday': 4,
'Saturday': 5,
'Sunday': 6
}
poptimes = populartimes.get_id(API_KEY, getPlaceID(location))
try:
dataPoint = (((poptimes['populartimes'])[dayNum[str(day)]])['data']
)[int(hour) - 1]
except KeyError:
dataPoint = 0
else:
dataPoint = 0
return dataPoint
def add_more_data_to_stores(stores: list):
"""
Add more information in the existing list of stores.
:param stores: a list of dictionaries representing stores from Google Places API
:precondition: the Populartimes library does not have version conflicts with the Google Places API
:postcondition: add all new items to stores in existing list
"""
key = get_api_key()
# SET USED HERE
desired_data = {'international_phone_number', 'current_popularity', 'time_spent'}
for store in stores:
response = populartimes.get_id(key, store['place_id'])
for datum in desired_data:
try:
store[datum] = response[datum]
except KeyError:
# Key error will occur if place does not have data available. We will skip the data.
continue
def log_supermarkets():
print("Started Data Collection")
api_key = ""
with open('supermarkets.json') as f:
supermarkets = json.load(f)
supermarkets = supermarkets[0]
#supermarkets = json.loads("supermarkets.json")[0]
for market_name in supermarkets:
google_key = supermarkets[market_name]
data = populartimes.get_id(api_key, google_key)
print(data)
file_object = open(market_name + '.txt', 'a')
if "current_popularity" in data:
current_popularity = data["current_popularity"]
print(market_name + ": " + str(current_popularity))
file_object.write(str(current_popularity) + "\n")
else:
print("No popularity record for " + market_name)
file_object.write("-1\n")
file_object.close()
def get_ptimes_data(location, name):
gmaps = googlemaps.Client(key=BaseConfig.GMAPS_API_KEY)
# We want a lat/long of the place
# This helps to generalize things
# `location` can be "NYC" or "395 Brittingham Dr" or "11030" etc
full_list_geocoded_locations = gmaps.geocode(
address=location) # latitude/longitude value
cur_coordinates = get_lat_lng(
full_list_geocoded_locations[0]) # take the first place (limitatino)
full_places_list = gmaps.places(
name, location=cur_coordinates) # maybe filter with arg "type" later
place_id = full_places_list["results"][0][
"place_id"] # ONLY get FIRST returned place's place ID (limitation)
results = ptimes.get_id(BaseConfig.GMAPS_API_KEY, place_id)
return results # only returning first rn, definitely a limitation
def get_popular_times(self, shop: Shop):
"""
Add docu
:param shop:
"""
init = time()
if shop.gmaps_id is None:
return
data = populartimes.get_id(self._api_key, shop.gmaps_id)
print(f'Populartimes query took {time() - init:.2f}s')
day, hour = utils.current_weekday(), utils.current_hour()
pct_people = data.get('current_popularity')
if pct_people is None:
if 'populartimes' in data:
for weekday_info in data['populartimes']:
if weekday_info['name'] == day:
day = weekday_info['name']
pct_people = weekday_info['data'][int(hour)]
else:
pct_people = 0
shop.set_pct_people(pct_people=pct_people)
def Results2(id="None"):
times = [
'12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am',
'10am', '11am', '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm',
'7pm', '8pm', '9pm', '10pm', '11pm'
]
place = google_places.get_place(id)
photos2 = []
try:
places_result = populartimes.get_id(API_KEY, id)['populartimes']
except:
places_result = []
for photo in place.photos:
photo.get(maxheight=50000, maxwidth=50000)
photos2.append(photo.url)
return render_template('view.html',
photos=photos2,
name=place.name,
results=places_result,
time=times,
len2=len(times))
def find_shop(self, id, info_list):
#init = time()
gmaps = googlemaps.Client(
key='AIzaSyDSbCbc_tuJC57KYxkUPiaxyG2tRkc3-Wk')
shopinfo = self.I.get_store_info(id, info_list)
#print(f'Data loaded in {time() - init:.2f}s')
string = ''
for item in shopinfo:
string = str(str(string) + str(shopinfo[item]) + str(' '))
#string = 'Netto city Harscampstraße Aachen'
search1 = gmaps.find_place(string,
'textquery',
fields=('place_id', 'opening_hours',
'rating'))
try:
shopid = search1['candidates'][0]['place_id']
open_closed = search1['candidates'][0]['opening_hours']['open_now']
rating = search1['candidates'][0]['rating']
#init = time()
data = pt.get_id('AIzaSyDSbCbc_tuJC57KYxkUPiaxyG2tRkc3-Wk', shopid)
#print(f'Data loaded in {time() - init:.2f}s')
day = utils.current_weekday()
hour = utils.current_hour()
people = -1
try:
for item in data:
if ((item == 'current_popularity')):
people = data['current_popularity']
if (people == -1):
for i in range(len(data['populartimes'])):
if (data['populartimes'][i]['name'] == day):
day = data['populartimes'][i]['name']
people = data['populartimes'][i]['data'][int(hour)]
except:
people = 0
relevant_info = [shopid, open_closed, rating, people]
print(relevant_info)
return (relevant_info)
except:
print('-')
def api_popularity():
result = None
status = 'Success'
statusCode = 200
if 'place_id' in request.args:
result = populartimes.get_id(apiKey, request.args.get('place_id'))
app.logger.info("Received popularity request and returned status %s (%s)", statusCode, status)
else:
status = 'Invalid request'
statusCode = 400
app.logger.info("Received invalid popularity request and returned status %s (%s)", statusCode, status)
response = {
'result': result,
'status': status
}
return jsonify(response), statusCode
def popular_time_id(id_):
"""
:param id_:
:return:
"""
data = populartimes.get_id('AIzaSyC5tGMwgvsyi5HZBZ_6bo__KLhBxVEOlGs', id_)
with open('popularTime.csv', 'a+', newline='', encoding='UTF-8') as csv_file:
csv_writer = csv.writer(csv_file)
column_title_row = ['Time', data['name']]
csv_writer.writerow(column_title_row)
for i in range(len(data['populartimes'])):
for j in range(24):
if j < 10:
time = data['populartimes'][i]['name'] + ' 0' + str(j) + ':00'
else:
time = data['populartimes'][i]['name'] + ' ' + str(j) + ':00'
row = [time, data['populartimes'][i]['data'][j]]
csv_writer.writerow(row)
time_spent = ['Time Spent', str(data['time_spent'][0]) + '~' + str(data['time_spent'][1])]
csv_writer.writerow(time_spent)
def relative_popularity(client, search):
locations = googlemaps.places.find_place(client, search,
"textquery")["candidates"]
for loc in locations:
# print(loc)
# print(loc["place_id"])
popular_times_data = populartimes.get_id(GOOGLE_API_KEY,
loc["place_id"])
# print(popular_times_data["name"])
# print(popular_times_data["coordinates"])
now = datetime.now()
hour = now.hour
day = now.strftime("%A")
busyRanges = {
range(0, 30): 1,
range(30, 50): 2,
range(50, 80): 3,
range(80, 100): 4
}
# print(popular_times_data)
if "current_popularity" in popular_times_data:
currPop = popular_times_data["current_popularity"]
if busyRanges[currPop] == 4:
return 0
else:
return popular_times_data["name"]
if "populartimes" in popular_times_data:
popular_times = popular_times_data["populartimes"][0]
for p in popular_times:
if p["name"] == day:
return popular_times["data"][day]
return 50
def get_current_popularity():
"""Get the current popularity, if it exists, for the location passed
in the query string parameters"""
place_id = request.args.get("place_id")
location_info = populartimes.get_id(os.environ["GMAPS_PLACES_API_KEY"],
place_id)
if "current_popularity" in location_info:
current_popularity = location_info["current_popularity"]
return jsonify(current_popularity=current_popularity)
day_of_week = datetime.strftime(datetime.now(), "%A")
hour_of_day = int(datetime.strftime(datetime.now(), "%H"))
expected_popularity = 0
for day in location_info["populartimes"]:
if day_of_week.lower() == day["name"].lower():
expected_popularity = day["data"][hour_of_day]
break
return jsonify(expected_popularity=expected_popularity)
def get_place(query: str):
"""
Takes a search query string and returns a Place object populated with
results of a populartimes.get request. The search query can be any kind
of Place data, like a name or address.
:param query: The text input specifying which place to search for
:return: Place object
"""
# Establish ~10 mile circular search bias around the Eugene area
bias = "circle:[email protected],-123.095051"
# Get place ID from a given search query using Places API
search = CLIENT.find_place(query, "textquery", location_bias=bias)
place_id = search["candidates"][0]["place_id"]
# Get place details using the populartimes library
info = populartimes.get_id(KEY, place_id)
place = Place(place_id, info["name"], info["address"], info["coordinates"],
info["types"], info["populartimes"], info["rating"],
info["rating_n"])
return place
