def store_google_reviews_and_processed_text(gtable, target_table, city):
#"matched_google_ams"
#"matched_google_reviews_ams"
gpoints = postgis_functions.get_rows_from_table(gtable)
session, RTable = pois_storing_functions.setup_db(target_table, "notused",
"reviews")
eng_stop, eng_exclude = setup_for_topic_modeling("en")
if city == "ams":
nl_stop, nl_exclude = setup_for_topic_modeling("nl")
elif city == "ath":
gr_stop, gr_exclude = setup_for_topic_modeling("el")
lemma = WordNetLemmatizer()
for g in gpoints:
rev = {}
ratings = []
gjson = json.loads(g["json"])
if "reviews" in gjson:
for review in gjson["reviews"]:
rev["id"] = g["id"] + "_" + review["author_name"]
rev["placesid"] = g["id"]
rev["name"] = g["name"]
rev["type"] = g["type"]
rev["point"] = g["point"]
rev["lang"] = review["language"]
print(rev["lang"])
rev["text"] = review["text"]
if rev["lang"] == "en":
rev["processedtextlda"] = clean(review["text"], eng_stop,
eng_exclude, lemma)
elif rev["lang"] == "nl" and city == "ams":
rev["processedtextlda"] = clean(review["text"], nl_stop,
nl_exclude, lemma)
elif rev["lang"] == "el" and city == "ath":
rev["processedtextlda"] = clean(review["text"], gr_stop,
gr_exclude, lemma)
# ratings.append(review["rating"])
# rev["avgrating"] = np.mean(ratings)
print(
"############################################################"
)
try:
session.add(RTable(**rev))
session.commit()
print(rev["name"], " INSERTED!")
except Exception as err:
session.rollback()
print("# NOT INSERTED: ", err)
print(
"############################################################"
)
if __name__ == "__main__":
try:
city = "ams"
source = "gsv"
CITY_FOLDER = "/home/bill/Desktop/thesis/code/UDS/google_street_view/images/" + city
#c, rad, logfile, errorfile = get_map_points_to_search.config_parameters_for_searching("gsv")
logfile = "/home/bill/Desktop/thesis/logfiles/" + source + "_" + city + "_whole_clipped_point_iterations.txt"
not_inserted_file = "/home/bill/Desktop/thesis/logfiles/" + source + "_" + city + "_whole_clipped_not_inserted.txt"
last_searched_id = pois_storing_functions.get_last_id_from_logfile(
logfile)
points = get_points_from_db("google_ams_whole_clipped_40",
last_searched_id)
with open(not_inserted_file, "r") as f:
lines = f.read().splitlines()
session, GSVTable = pois_storing_functions.setup_db(
"gsv_ams_whole_clipped_40", "gsv_ams_whole_clipped_count", "gsv")
# For each point --> search nearby in google
for row_number, places_id, point_id, point_lat, point_lng in points:
get_map_points_to_search.log_last_searched_point(
logfile, row_number)
found_flag = False
print("POINT: ", row_number, places_id, point_id, point_lat,
point_lng)
panoids = streetview.panoids(lat=point_lat, lon=point_lng)
if not panoids:
with open(not_inserted_file, "a") as text_file:
print(
f"Not found for point \n row number: {row_number}, point_id: {point_id}, "
f"place id: {places_id}",
file=text_file)
def add_pop_times_in_places(api_key, query_results):
# for each place gotten from google
for place in query_results.places:
place.get_details()
gid = place.details["place_id"]
place.details["poptimes"] = get_pop_times_from_id(api_key, gid)
return query_results.places
if __name__ == "__main__":
google_places, api_key = setup()
c, rad, logfile, errorfile = get_map_points_to_search.config_parameters_for_searching("google", "ath")
last_searched_id = pois_storing_functions.get_last_id_from_logfile(logfile)
# define which table
session, GTable, CTable = pois_storing_functions.setup_db("google_ath_whole_clipped_40",
"google_ath_whole_clipped_count", "google")
#define types we don't care about
google_not_wanted_types = ["route"]
# For each point --> search nearby in google
for ogc_fid, point_lat, point_lng in c:
#try:
count_places = 0
count_duplicates = 0
print("POINT: ", ogc_fid, point_lat, point_lng)
# keep the id of the last searched point
get_map_points_to_search.log_last_searched_point(logfile, ogc_fid)
ll = {"lat": str(point_lat), "lng": str(point_lng)}
query_results = google_places.nearby_search(lat_lng=ll, radius=rad)
# if I want to include popular times
places_extended = add_pop_times_in_places(api_key, query_results)
# for each place gotten from google
# define which table (fsq table, count table)
pp = pprint.PrettyPrinter(indent=4)
# get hierarchy categories tree
categories_tree = cl.venues.categories()
#print(get_fsq_data.find_key_path_from_value(categories_tree["categories"], "Cantonese Restaurant"))
# For each point --> search nearby in google
for ogc_fid, point_lat, point_lng in c:
count_places = 0
count_duplicates = 0
print("POINT: ", ogc_fid, point_lat, point_lng)
# keep the id of the last searched point
get_map_points_to_search.log_last_searched_point(logfile, ogc_fid)
ll = str(point_lat) + ", " + str(point_lng)
# define which table
session, FTable, CTable = pois_storing_functions.setup_db(
"fsq_" + city + "_whole__clipped_40",
"fsq_" + city + "_whole_clipped_40_count", "FSQ")
# specify which categories to search for!
# food = 4d4b7105d754a06374d81259
# arts and entertainment = 4d4b7104d754a06370d81259
# college & uni = 4d4b7105d754a06372d81259
# nightlife spot = 4d4b7105d754a06376d81259
# outdoor & recreation = 4d4b7105d754a06377d81259
# food and drink shop = 4bf58dd8d48988d1f9941735
# outdoors and recreation = 4d4b7105d754a06377d81259
# Bank = 4bf58dd8d48988d10a951735
# clοthing store = 4bf58dd8d48988d103951735
# drugstore = 5745c2e4498e11e7bccabdbd
# Bus station = 4bf58dd8d48988d1fe931735
# Bus stop = 52f2ab2ebcbc57f1066b8b4f
# hotel = 4bf58dd8d48988d1fa931735
category_index = label_map_util.create_category_index(categories)
print(category_index)
def load_image_into_numpy_array(image):
(im_width, im_height) = image.size
return np.array(image.getdata()).reshape(
(im_height, im_width, 3)).astype(np.uint8)
# # Detection
if __name__ == '__main__':
# Size, in inches, of the output images.
IMAGE_SIZE = (12, 8)
#source = "coco"
session, ODTable = pois_storing_functions.setup_db(TABLE, "notused", TYPE)
# selected categories' ids
# cc_categories = ["person", "bicycle", "car", "motorcycle", "bus", "train", "truck",
# "boat", "trafficlight", "firehydrant", "stopsign", "bench", "pottedplant"]
# this should change depending on the labels
sel_categories = {x: category_index[x] for x in category_ids}
#logfile = "/home/bill/Desktop/thesis/logfiles/" + source + "_" + city + "_matched.txt"
#last_searched_id = logging_functions.get_last_id_from_logfile(logfile)
#imgs = postgis_functions.get_photos_from_db("matched_gsv_ams", last_searched_id)#last_searched_id)
imgs = postgis_functions.get_photos_for_od("matched_places_gsv_" + city,
TABLE)
# for i in sel_categories:
# print('Column("{name}", Numeric),'.format(name=sel_categories[i]["name"].lower().replace(" ", "")))
# print('Column("{count}", Numeric),'.format(count=sel_categories[i]["name"].lower().replace(" ", "")+"count"))
# print('Column("{high}", Numeric),'.format(high=sel_categories[i]["name"].lower().replace(" ", "")+"highestprob"))
def add_similarities_to_db(session, STable, sim_dict):
try:
session.add(STable(**sim_dict))
session.commit()
#print("~~ Similarities INSERTED!")
except Exception as err:
# errcount+=1
#print("~~ similarities NOT INSERTED")
print(err)
session.rollback()
if __name__ == '__main__':
fpoints = postgis_functions.get_pois_for_matching(
"gt_fsq_ams_matched_table", 0)
session, STable = pois_storing_functions.setup_db("similarities_ams_table",
"", "similarities")
rad = 300
point = 0
for fpoint in fpoints:
print("Point: ", point)
point += 1
count_matched = 0
count_nmatched = 0
google_closest_points = postgis_functions.get_matching_attr_from_pois_within_radius\
("google_ams_whole_clipped_40", fpoint["lat"], fpoint["lng"], rad)
for gpoint in google_closest_points:
sim_dict = get_similarities_dict(fpoint, gpoint)
sim_dict["id"] = fpoint["id"] + "_" + gpoint["id"]
if matched(fpoint, gpoint):
count_matched += 1
sim_dict["match"] = 1
#print("TOTAL: ", count)#countaddr +countphone +countnamestreet + countwebstreet + countwebname + countnamedist)
print("COUNT (actual): ", count)
print("@@@@@@@@@@@@@@@@@@@@@")
if __name__ == "__main__":
city = "ath"
# choose radius
#for rad in [275]:
fpoints = postgis_functions.get_pois_for_matching(
"fsq_" + city + "_whole__clipped_40", 0)
#fpoints = postgis_functions.get_pois_for_matching("fsq_" + city + "_places", 0)
# Create matched tables
session, FTable = pois_storing_functions.setup_db(
"tmatched_places_fsq_" + city, "", "fsq_matched")
session.close()
session, GTable = pois_storing_functions.setup_db(
"tmatched_places_google_" + city, "", "google_matched")
session.close()
session, MTable = pois_storing_functions.setup_db(
"tmatched_places_google_fsq_" + city, "", "matching_table")
rad = 300
###############
# FOR EACH POI#
###############
countnamestreet = 0
countwebstreet = 0
countphone = 0
countaddr = 0
point = 0
return c.fetchall()
if __name__ == '__main__':
api = setup_api()
city = "ams"
source = "twitter"
count = 0
logfile = "/home/bill/Desktop/thesis/logfiles/" + source + "_" + city + "_matched.txt"
while count < 1:
try:
last_searched_id = get_last_id_from_logfile(logfile)
#print(last_searched_id)
#points = postgis_functions.get_pois_from_fsq_db("matched_fsq_" + city, last_searched_id)
session, TTable = setup_db("matched_places_twitter_" + city,
"twitter_" + city + "_places_count",
"twitter")
points = get_fsq_points_less_tweets_than(
1,
"matched_places_fsq_" + city,
"matched_places_twitter_" + city,
random=True)
rad = "0.05km"
since = "2017-01-01"
until = "2018-12-19"
# since = "2014-01-01"
# until = "2016-12-30"
count = 0
print(len(points))
for fsq in points:
count += 1
import ast
import json
import pois_storing_functions
import google_fsq_features_extraction
if __name__ == '__main__':
# initializations
count = 0
city = "ams"
#format_str = "%Y-%m-%d %H:%M:%S"
#session, GFTable = pois_storing_functions.setup_db("matched_gf_features_ams", "notused", "gf_features")
gtab = "matched_places_google_" + city
ftab = "matched_places_fsq_" + city
# get places
gfpoints = postgis_functions.get_google_fsq_features(gtab, ftab)
session, GFTable = pois_storing_functions.setup_db("matched_places_spatial_features_" + city + "2", "notused", "spatial")
#fpoints = postgis_functions.get_rows_from_id_not_in_table("matched_fsq_ams", "matched_text_features_ams", "id")
# selecting what to store in table from the general information from google and fsq
for gf in gfpoints:
gfdata = {}
gf["id"] = gf["fid"]
# for using the function id is fsq id
types_100 = postgis_functions.get_place_types_in_radius(gf, "matched_places_fsq_" + city, 100)
types_1000 = postgis_functions.get_place_types_in_radius(gf, "matched_places_fsq_" + city, 1000)
types_2000 = postgis_functions.get_place_types_in_radius(gf, "matched_places_fsq_" + city, 2000)
types_3000 = postgis_functions.get_place_types_in_radius(gf, "matched_places_fsq_" + city, 3000)
gfdata = google_fsq_features_extraction.get_nearby_places(gfdata, types_100, "100")
gfdata = google_fsq_features_extraction.get_nearby_places(gfdata, types_1000, "1000")
if __name__ == '__main__':
source = "gsv"
city = "ams"
# load the labels
classes, labels_IO, labels_attribute, W_attribute = load_labels()
# load the model/home/bill
features_blobs = []
model = load_model()
# load the transformer
tf = returnTF() # image transformer
# get the softmax weight
params = list(model.parameters())
weight_softmax = params[-2].data.numpy()
weight_softmax[weight_softmax < 0] = 0
# Code has been EDITED from this point on
session, STable = pois_storing_functions.setup_db(
"matched_scene_features_ams2", "notused", "scene_features")
# GET images
logfile = "/home/bill/Desktop/thesis/logfiles/" + source + "_" + city + "_scene_features.txt"
last_searched_id = logging_functions.get_last_id_from_logfile(logfile)
imgs = postgis_functions.get_photos_from_db("matched_gsv_ams",
last_searched_id)
#placesids = postgis_functions.get_id_not_in_table(table_source, table_target)
#imgs = postgis_functions.get_row_by_id("mat")
count = 0
for row in imgs:
logging_functions.log_last_searched_point(logfile, row["point"])
print(row["point"])
# remove row from dict
#row.pop("rn", None)
count += 1
print("COUNT = " + str(count))
for t in types:
d[t["type"] + "_" + rad] = t["count"]
return d
if __name__ == '__main__':
# initializations
count = 0
city = "ath"
#format_str = "%Y-%m-%d %H:%M:%S"
#session, GFTable = pois_storing_functions.setup_db("matched_gf_features_ams", "notused", "gf_features")
gtab = "matched_places_google_" + city
ftab = "matched_places_fsq_" + city
# get places
gfpoints = postgis_functions.get_google_fsq_features(gtab, ftab)
session, GFTable = pois_storing_functions.setup_db("matched_places_gf_features_" + city, "notused", "gf")
#fpoints = postgis_functions.get_rows_from_id_not_in_table("matched_fsq_ams", "matched_text_features_ams", "id")
# selecting what to store in table from the general information from google and fsq
for gf in gfpoints:
gfdata = {}
# for using the function id is fsq id
gf["id"] = gf["fid"]
types_100 = postgis_functions.get_place_types_in_radius(gf, "matched_places_fsq_" + city, 100)
types_1000 = postgis_functions.get_place_types_in_radius(gf, "matched_places_fsq_" + city, 1000)
types_5000 = postgis_functions.get_place_types_in_radius(gf, "matched_places_fsq_" + city, 5000)
gfdata = get_nearby_places(gfdata, types_100, "100")
gfdata = get_nearby_places(gfdata, types_1000, "1000")
gfdata = get_nearby_places(gfdata, types_5000, "5000")
# for k in gfdata: