def extract_features(directory): cities = [] filenames = [] input_features = [] output_features = [] files = glob.glob(directory) for name in files: with open(name, 'r') as f: city = cityiograph.City(f.read()) cities.append(city) input_features.append(get_features(city)) output_features.append(get_results(city)) input_features = np.array(input_features).astype('int32') output_features = np.array(output_features).astype('int32') return cities, input_features, output_features
shortest_paths = dijkstra.shortestPaths(road_graph, destination) for path in shortest_paths: traffic = traffic_between(pop_graph[path[-1]], pop_graph[destination]) for road in path: city.get_cell(road).data["traffic"] += traffic def traffic_between(source, destination): return source * destination if __name__ == "__main__": in_dir = "./data/input/" out_dir = "./data/output/" for directory in [x[0] for x in os.walk(in_dir)]: for filename in os.listdir(directory): if filename.endswith(".json"): json_input = open(directory + "/" + filename).read() city = cityiograph.City(json_input) road_graph = city.get_road_graph() traffic_sim(city) full_out_dir = out_dir + directory.split("/")[-1] + "/" if not os.path.exists(os.path.dirname(full_out_dir)): os.makedirs(os.path.dirname(full_out_dir)) outfile = open(full_out_dir + filename, 'w') outfile.write(city.to_json())
expected_vals.append(get_data(expected_cities[i])) predicted_vals.append(get_data(predicted_cities[i])) return R_squared(expected_vals, predicted_vals) if __name__ == "__main__": expected_dir = "./TrafficML/data/test/" predicted_dir = "./TrafficML/data/prediction-population-isroad-treesim/prediction/" expected_vals = [] print("Parsing expected citites") for filename in os.listdir(expected_dir): if filename.endswith(".json"): city = cityiograph.City(open(expected_dir + filename).read()) expected_vals.append(get_data(city)) expected_vals = np.array(expected_vals) print("Traversing predicted directory") for dirname, dirs, files in os.walk(predicted_dir): print("Parsing " + dirname) predicted_vals = [] for filename in files: if filename.endswith(".json"): city = cityiograph.City(open(dirname + "/" + filename).read()) predicted_vals.append(get_data(city)) predicted_vals = np.array(predicted_vals) if len(predicted_vals) == 0: continue
return np.mean(model_output.reshape(-1, 49), axis=1) def update_city(old_city, new_city, x, y): remove_old = deltas(get_5x5_block(old_city, x, y)) add_new = deltas(get_5x5_block(new_city, x, y)) change = np.subtract( add_new, remove_old) # Kevin - switched order for subtract operation... # print(change) push_5x5_deltas(old_city, change, x, y) return old_city if __name__ == "__main__": import sys sys.path.append("../global/") import cityiograph with open("./city_0_output_solar.json", 'r') as f: city = cityiograph.City(f.read()) # for cell in city.cells.values(): # cell.data["solar"] = 0 new_city = city.copy() for cell in new_city.cells.values(): cell.density = 0 update_city(new_city, city, 5, 5) # with open("city_0_output_solar_new.json", 'w') as o: # o.write(city.to_json())
def data(config): print(">>> Loading data...") config = DotMap(config) # Try to get pickled data try: if config.util.force_data_reload: raise print(">>> Trying to load pickled train/val sets.") data = pickle.load(open(config.root.pickle_fname, 'rb')) print(">>> Success on pickle file.") return data except Exception: print(">>> Loading cities for the first time.") train_X, train_Y, train_filename_list = [], [], [] val_X, val_Y, val_filename_list = [], [], [] for i, path in tqdm(enumerate(glob(config.root.dir_json))): with open(path, 'r') as f: # Extract city = cityiograph.City(f.read()) inp = cityiograph.get_features(city, mode='traffic') out = cityiograph.get_results(city, mode='traffic') # Reshape inp = inp.reshape(16, 16, 2) out = out.reshape(16, 16, 2) # Noramlize inp = (inp - inp.mean()) / inp.std() out = (out - out.mean()) / out.std() # Load if i < config.util.num_val_samples: # Add to train set train_X.append(inp) train_Y.append(out) train_filename_list.append(path) else: # Add to val set val_X.append(inp) val_Y.append(out) val_filename_list.append(path) # Reshape final arrays train_X = np.array(train_X) train_Y = np.array(train_Y) val_X = np.array(val_X) val_Y = np.array(val_Y) # Create dict output = { 'train_X': train_X, 'train_Y': train_Y, 'val_X': val_X, 'val_Y': val_Y, 'train_filename_list': train_filename_list, 'val_filename_list': val_filename_list } # Write to pickle pickle.dump(output, open(config.root.pickle_fname, 'wb')) return output
def run(config): # Save this source code to the local experiment directory for later use print(">>> Saving source code locally...") with open(__file__, 'r') as source_file_input: source_text = source_file_input.read() source_name = os.path.join(ex.observers[0].dir, 'SOURCE_' + __file__) with open(source_name, 'w') as source_file_output: source_file_output.write(source_text) # Get config config = DotMap(config) # Get data data_dict = DotMap(data()) # Get model m = model() if config.root.model_fname is None: # Only train if this is not a pre-loaded model run m.compile(optimizer=config.hyper.optimizer, loss=config.hyper.loss, metrics=config.hyper.metrics) # Init callbacks callbacks = [ TQDMCallback(), TensorBoard(log_dir=ex.observers[0].dir, histogram_freq=1, write_graph=True, write_images=False) ] # Train print(">>> Training...") m.fit(x=data_dict.train_X, y=data_dict.train_Y, batch_size=config.hyper.batch_size, epochs=config.hyper.nb_epochs, verbose=2, callbacks=callbacks, validation_data=(data_dict.val_X, data_dict.val_Y)) m.save(os.path.join(ex.observers[0].dir, 'final_model.hdf5')) # Test print(">>> Evaluating...") result = m.evaluate(x=data_dict.val_X, y=data_dict.val_Y, batch_size=config.hyper.batch_size, verbose=1, sample_weight=None) # Write result to file for later use - loss and R^2 with open(os.path.join(ex.observers[0].dir, 'test_metrics.txt'), 'w') as f: f.writelines([str(result), "loss, R^2"]) print(">>> Predicting cities...") # 1. Make output dir output_dir = os.path.join(ex.observers[0].dir, 'predicted_cities') os.mkdir(output_dir) # 2. Iterate for x, fname in tqdm( list(zip(data_dict.val_X, data_dict.val_filename_list))): try: # Reshape x x = np.expand_dims(x, axis=0) # Load city with open(fname, 'r') as city_file_object: city = cityiograph.City(city_file_object.read()) # Make prediction pred = m.predict(x) # Reshape pred = pred.squeeze().flatten().tolist() # Add features city.update_values(pred, mode='traffic') # Write to new file fname_base = os.path.splitext(os.path.basename(fname))[0] fname_output = os.path.join(output_dir, fname_base + '_predicted.json') with open(fname_output, 'w') as city_file_object_output: city_file_object_output.write(city.to_json()) except Exception: # Error parsing city or something else went wrong print("Missed city {}.".format(fname)) pass print("Process complete!!!")
def extract_data(directory, return_endpoints=False): print("Extracting data for directory {}.".format(directory)) # Get all JSON files # Recursive feature taken from https://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python files = glob.glob( directory, recursive=True) # Load list of filenames from the directory cities = [] # City objects list filenames = [] # List of raw filenames inp = [] # Input feature matrix out = [] # Output feature matrix # Create dictionary mapping of bounds for both traffic and wait # bounds = { 'low_traffic' : 0 , 'high_traffic' : 0 , 'low_wait' : 0 , 'high_wait' : 0 } # Create np array objects for traffic and wait values traffic = [] wait = [] for name in tqdm(files): # Using tqdm for loop analysis with open(name, 'r') as f: current_input = [] current_output = [] s = f.read() city = cityiograph.City(s) raw = os.path.splitext(os.path.basename(name))[0] d = json.loads(s) max_traffic = max([ cell['data']['traffic'] for cell in d['grid'] if 'data' in cell ]) max_wait = max( [cell['data']['wait'] for cell in d['grid'] if 'data' in cell]) # Check bounds if needed if return_endpoints: traffic.append(max_traffic) wait.append(max_wait) # Ignore zero cities - suprisingly high percentage if max_traffic == 0 or max_wait == 0: continue # Else, construct feature vector for city for i in range(city.width): for j in range(city.height): cell = city.cells.get((i, j)) current_input.append(cell.population) current_input.append(0) if ( cell.type_id == 6) else current_input.append(1) current_output.append(cell.data["traffic"]) current_output.append(cell.data["wait"]) cities.append(city) # Append city structure filenames.append(raw) # Append raw filename for later reference inp.append(np.array(current_input)) out.append(np.array(current_output)) # Return correct data based on return_endpoints parameter if return_endpoints: return np.array(traffic), np.array(wait) # Construct feature matrices and return inp = np.array(inp).astype(int) out = np.array(out).astype(int) return cities, filenames, inp, out
def unique_city_generator(city_directory, ai_only=False): """Generator instance to give us cities that are different based on equals(). Args: city_directory (str): directory (relative or absolute) with .json city *output* files ai_only (bool, optional): used for AI acceptance - if we only want the AI city option Yields: 3-tuple: unique City instance that can be used for analysis and corresponding dictionary object + filename """ prev_city = None good_count = 0 total_count = 0 files = glob.glob(city_directory + '*.json') files.sort() for city_path in tqdm(files): total_count += 1 fname = os.path.basename(city_path) # Validate time the_city_time = get_time(fname) if the_city_time < MIN_TIME: continue # Read file with open(city_path, 'r') as f: full_json_string = f.read() # Get only the city described by ai or predict keys d = json.loads(full_json_string)['ai'] if d is None: # null AI -> use predict instead if not ai_only: d = json.loads(full_json_string)['predict'] else: # Just ignore yield None, None, fname continue j_string = json.dumps(d) current_city = cityiograph.City(j_string) if prev_city is None: # First time, just yield prev_city = current_city good_count += 1 yield current_city, d, fname elif not prev_city.equals(current_city): # Different cities - yield new prev_city = current_city good_count += 1 yield current_city, d, fname else: # Equal continue print("Yield = {:.2%}. {} / {}.".format( float(good_count) / total_count, good_count, total_count))