sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from config import get_config from grid.create_grid import RegionGrid from model.RegionEncoder import RegionEncoder from model.utils import write_embeddings import torch import numpy as np if len(sys.argv) > 1: N_RUNS = int(sys.argv[1]) else: N_RUNS = 15 c = get_config() region_grid = RegionGrid(config=c) region_grid.load_img_data(std_img=True) region_grid.load_weighted_mtx() OUT_DIR = c['data_dir_main'] + c['hyperparams_dir'] + "/" # hyperparameters n_nodes = len(region_grid.regions) n_nodal_features = region_grid.feature_matrix.shape[1] h_dim_graph = 64 h_dim_img = 32 h_dim_size = int(c['hidden_dim_size']) context_gcn = 4 neg_samples_gcn = 10 epochs = 12 learning_rate = .1
else: task = sys.argv[1] estimator = sys.argv[2] try: n_epochs = int(sys.argv[3]) except IndexError: n_epochs = 25 assert(estimator in ['xgb', 'lasso', 'rf', 'mlp', 'ridge']) n_folds = 5 print("K-Fold Learning - {}".format(estimator)) c = get_config() region_grid = RegionGrid(config=c) region_grid.load_weighted_mtx() tmp = pd.DataFrame(region_grid.feature_matrix, index = region_grid.idx_coor_map.values()) autoencoder_embed = c['autoencoder_embedding_file'] regionencoder_embed = c['embedding_file'] gcn_all_embed = '{}gcn_all_embedding.txt'.format(c['data_dir_main']) gcn_sg_embed = '{}gcn_skipgram_embedding.txt'.format(c['data_dir_main']) gcn_flow_embed = '{}gcn_flow_embedding.txt'.format(c['data_dir_main']) concat_embed ='{}concat_global_embedding.txt'.format(c['data_dir_main']) if task == 'house_price': input_data = region_grid.load_housing_data(c['housing_data_file'])
# total counts for each time bucket sums = m[v].sum(axis=1) s = sums.sum() # create probabilities p = [x / s for x in sums] walk = numpy.random.choice(m[v].shape[0], size=1, p=p)[0] walk_sequence.append(walk) # set this vertex to walk from v = walk walks.append(walk_sequence) return walks c = get_config() # init region region_grid = RegionGrid(config=c) def e_distance(d): import math return math.exp(-d * 1.5) distance_matrix = region_grid.get_distance_mtx(transform=e_distance) W, t = region_grid.create_flow_matrix(c['raw_flow_file'], region_name=c['city_name'], time=8) size = region_grid.grid_size * region_grid.grid_size
from model.utils import write_embeddings from model.GraphConvNet import GCN from grid.create_grid import RegionGrid if len(sys.argv) > 1: epochs = int(sys.argv[1]) learning_rate = float(sys.argv[2]) else: epochs = 25 learning_rate = .1 context_gcn = 4 neg_samples_gcn = 10 c = get_config() region_grid = RegionGrid(config=c) region_grid.load_weighted_mtx() n_nodes = len(region_grid.regions) h_dim_size = int(c['hidden_dim_size']) n_nodal_features = region_grid.feature_matrix.shape[1] gcn = GCN(n_features=n_nodal_features, h_dim_size=h_dim_size) embedding = gcn.run_train_job(region_grid, n_epoch=epochs, learning_rate=learning_rate, penalty=(1, 1), n_neg_samples=context_gcn, n_pos_samples=neg_samples_gcn) if torch.cuda.is_available(): embedding = embedding.data.cpu().numpy() else:
import unittest import numpy from config import get_config from grid.create_grid import RegionGrid from image.image_retrieval import get_images_for_all_no_marker c = get_config() TEST_REGION_GRID = RegionGrid(50, poi_file=open(c["poi_file"], 'rb'), load_imgs=False) class CreateGridTest(unittest.TestCase): def test_image_creation(self): print("Get Images") # get_images_for_all_no_marker(TEST_REGION_GRID, "image_2") def test_basic_region(self): self.assertEqual(len(TEST_REGION_GRID.regions), 2500) def test_region_grid_has_proper_lat_long(self): region = TEST_REGION_GRID.regions['0,49'] self.assertIsNotNone(region.points['nw']) self.assertIsNotNone(region.points['sw']) self.assertIsNotNone(region.points['ne']) self.assertIsNotNone(region.points['se']) last_y = TEST_REGION_GRID.y_space[-1] first_x = TEST_REGION_GRID.x_space[0]
colors.append(c[cat]) plt.figure(figsize=(3, 3)) plt.subplots_adjust(bottom=.6) plt.bar(range(len(d)), list(d.values()), align='center', color=colors) plt.xticks(range(len(d)), list(d.keys()), rotation=90, fontsize=14) plt.yticks(fontsize=8) plt.ylim(y_lim) plt.legend(loc='best') plt.savefig(fname) plt.clf() plt.close() c = get_config() region_grid = RegionGrid(config=c) k = 5 H = load_embedding(c['embedding_file']) X = region_grid.feature_matrix nbrs = NearestNeighbors(n_neighbors=k, algorithm='ball_tree').fit(H) distances, indices = nbrs.kneighbors(H) for id, r in region_grid.regions.items(): r_latent_nbrs = indices[r.index, 1:] all_nbrs_disconnected = 0 for nbr in r_latent_nbrs: nbr_coor = region_grid.idx_coor_map[nbr] r_nbr = region_grid.regions[nbr_coor] #print("r: {}, nbr: {}".format(id, nbr_coor))
import sys import os sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from config import get_config from grid.create_grid import RegionGrid import numpy as np import matplotlib.pyplot as plt plt.rc('text', usetex=True) plt.rc('font', family='serif') import geopandas as gpd c = get_config() region_grid = RegionGrid(config=c) house_price = region_grid.load_housing_data(c['housing_data_file']) print(house_price.head()) checkin = region_grid.get_checkin_counts(metric="mean") trim = np.percentile(house_price['priceSqft'], 99) print(trim) house_price = house_price[house_price['priceSqft'] < trim] house_price.to_csv(c['data_dir_main'] + 'zillow_house_price_trim.csv', index=False) checkin.to_csv(c['data_dir_main'] + "checkin.csv", index=False) print(checkin.head()) #checkin.checkins.hist() #plt.show() #plt.clf()
if __name__ == "__main__": import numpy as np from grid.create_grid import RegionGrid if len(sys.argv) > 1: epochs = int(sys.argv[1]) learning_rate = float(sys.argv[2]) batch_size = int(sys.argv[3]) else: epochs = 25 learning_rate = .05 batch_size = 25 c = get_config() region_grid = RegionGrid(config=c) region_grid.load_img_data(std_img=True) img_tensor = torch.Tensor(region_grid.img_tensor) h_dim_size = int(c['hidden_dim_size']) auto_encoder = AutoEncoder(img_dims=(50, 50), h_dim_size=h_dim_size) embedding = auto_encoder.run_train_job(n_epoch=epochs, img_tensor=img_tensor, lr=learning_rate) if torch.cuda.is_available(): embedding = embedding.data.cpu().numpy() else: embedding = embedding.data.numpy() write_embeddings(arr=embedding, n_nodes=region_grid.n_regions, fname=c['tile2vec_file'])
import sys import os sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from model.RegionEncoder import RegionEncoder from grid.create_grid import RegionGrid, get_images_for_grid from config import get_config import pickle import torch from model.utils import write_embeddings # Main script if region area, or grid configuration changes c = get_config() # init region region_grid = RegionGrid(config=c) print("Initializing Region: Lat [{}, {}], Lon: [{}, {}]".format(region_grid.lat_min, region_grid.lat_max, region_grid.lon_min, region_grid.lat_max)) # pull images get_images_for_grid(region_grid, clear_dir=True) # load images region_grid.load_img_data(std_img=True) # Compute weighted edge matrix W W = region_grid.create_flow_matrix(c['raw_flow_file'], region_name=c['city_name']) with open(c['flow_mtx_file'], 'wb') as f: pickle.dump(W, f) region_grid.load_weighted_mtx() region_grid.load_img_data() # hyperparameters n_nodes = len(region_grid.regions)
self.input = input self.format = format self.undirected = undirected self.output = output self.number_walks = number_walks self.representation_size = representation_size self.walk_length = walk_length self.max_memory_data_size = 1000000000 self.seed = 1990 self.window_size = 5 self.workers = 1 # Configure project config = get_config() region_grid = RegionGrid(config=config) region_grid.write_adj_list(config['adj_list_file']) region_grid.write_edge_list(config['edge_list_file']) h_dim_size = int(config['hidden_dim_size']) args = DWparams(input=config['adj_list_file'], format='adjlist', undirected=True, output=config['deepwalk_file'], number_walks=100, representation_size=h_dim_size, walk_length=40) process(args)
mse = mean_squared_error(y_true=y_test, y_pred=y_hat) mae = mean_absolute_error(y_test, y_hat) errors[i, 0] = mse errors[i, 1] = mae mean_cv_err = np.round(np.mean(errors, axis=0), 4) std_cv_err = np.round(np.std(errors, axis=0), 4) return mean_cv_err[0], std_cv_err[0], mean_cv_err[1], std_cv_err[ 1], errors if __name__ == "__main__": c = get_config() region_grid = RegionGrid(config=c) region_grid.load_weighted_mtx() region_grid.load_housing_data(c['housing_data_file']) y_house = region_grid.get_target_var("house_price") results = [] # Adjacency Average Model mse, mse_std, mae, mae_std, err_adj = cv_adj_mean( region_grid.regions, region_grid.matrix_idx_map, y_house) results.append(['adjacent avg', mse, mse_std, mae, mae_std]) y_is_valid = np.where(~np.isnan(y_house))[0] y_house = y_house[y_is_valid] # Global Avg Model