def extract_features(directory):
cities = []
filenames = []
input_features = []
output_features = []
files = glob.glob(directory)
for name in files:
with open(name, 'r') as f:
city = cityiograph.City(f.read())
cities.append(city)
input_features.append(get_features(city))
output_features.append(get_results(city))
input_features = np.array(input_features).astype('int32')
output_features = np.array(output_features).astype('int32')
return cities, input_features, output_features
shortest_paths = dijkstra.shortestPaths(road_graph, destination)
for path in shortest_paths:
traffic = traffic_between(pop_graph[path[-1]], pop_graph[destination])
for road in path:
city.get_cell(road).data["traffic"] += traffic
def traffic_between(source, destination):
return source * destination
if __name__ == "__main__":
in_dir = "./data/input/"
out_dir = "./data/output/"
for directory in [x[0] for x in os.walk(in_dir)]:
for filename in os.listdir(directory):
if filename.endswith(".json"):
json_input = open(directory + "/" + filename).read()
city = cityiograph.City(json_input)
road_graph = city.get_road_graph()
traffic_sim(city)
full_out_dir = out_dir + directory.split("/")[-1] + "/"
if not os.path.exists(os.path.dirname(full_out_dir)):
os.makedirs(os.path.dirname(full_out_dir))
outfile = open(full_out_dir + filename, 'w')
outfile.write(city.to_json())
expected_vals.append(get_data(expected_cities[i]))
predicted_vals.append(get_data(predicted_cities[i]))
return R_squared(expected_vals, predicted_vals)
if __name__ == "__main__":
expected_dir = "./TrafficML/data/test/"
predicted_dir = "./TrafficML/data/prediction-population-isroad-treesim/prediction/"
expected_vals = []
print("Parsing expected citites")
for filename in os.listdir(expected_dir):
if filename.endswith(".json"):
city = cityiograph.City(open(expected_dir + filename).read())
expected_vals.append(get_data(city))
expected_vals = np.array(expected_vals)
print("Traversing predicted directory")
for dirname, dirs, files in os.walk(predicted_dir):
print("Parsing " + dirname)
predicted_vals = []
for filename in files:
if filename.endswith(".json"):
city = cityiograph.City(open(dirname + "/" + filename).read())
predicted_vals.append(get_data(city))
predicted_vals = np.array(predicted_vals)
if len(predicted_vals) == 0:
continue
return np.mean(model_output.reshape(-1, 49), axis=1)
def update_city(old_city, new_city, x, y):
remove_old = deltas(get_5x5_block(old_city, x, y))
add_new = deltas(get_5x5_block(new_city, x, y))
change = np.subtract(
add_new,
remove_old) # Kevin - switched order for subtract operation...
# print(change)
push_5x5_deltas(old_city, change, x, y)
return old_city
if __name__ == "__main__":
import sys
sys.path.append("../global/")
import cityiograph
with open("./city_0_output_solar.json", 'r') as f:
city = cityiograph.City(f.read())
# for cell in city.cells.values():
# cell.data["solar"] = 0
new_city = city.copy()
for cell in new_city.cells.values():
cell.density = 0
update_city(new_city, city, 5, 5)
# with open("city_0_output_solar_new.json", 'w') as o:
# o.write(city.to_json())
def data(config):
print(">>> Loading data...")
config = DotMap(config)
# Try to get pickled data
try:
if config.util.force_data_reload:
raise
print(">>> Trying to load pickled train/val sets.")
data = pickle.load(open(config.root.pickle_fname, 'rb'))
print(">>> Success on pickle file.")
return data
except Exception:
print(">>> Loading cities for the first time.")
train_X, train_Y, train_filename_list = [], [], []
val_X, val_Y, val_filename_list = [], [], []
for i, path in tqdm(enumerate(glob(config.root.dir_json))):
with open(path, 'r') as f:
# Extract
city = cityiograph.City(f.read())
inp = cityiograph.get_features(city, mode='traffic')
out = cityiograph.get_results(city, mode='traffic')
# Reshape
inp = inp.reshape(16, 16, 2)
out = out.reshape(16, 16, 2)
# Noramlize
inp = (inp - inp.mean()) / inp.std()
out = (out - out.mean()) / out.std()
# Load
if i < config.util.num_val_samples:
# Add to train set
train_X.append(inp)
train_Y.append(out)
train_filename_list.append(path)
else:
# Add to val set
val_X.append(inp)
val_Y.append(out)
val_filename_list.append(path)
# Reshape final arrays
train_X = np.array(train_X)
train_Y = np.array(train_Y)
val_X = np.array(val_X)
val_Y = np.array(val_Y)
# Create dict
output = {
'train_X': train_X,
'train_Y': train_Y,
'val_X': val_X,
'val_Y': val_Y,
'train_filename_list': train_filename_list,
'val_filename_list': val_filename_list
}
# Write to pickle
pickle.dump(output, open(config.root.pickle_fname, 'wb'))
return output
def run(config):
# Save this source code to the local experiment directory for later use
print(">>> Saving source code locally...")
with open(__file__, 'r') as source_file_input:
source_text = source_file_input.read()
source_name = os.path.join(ex.observers[0].dir, 'SOURCE_' + __file__)
with open(source_name, 'w') as source_file_output:
source_file_output.write(source_text)
# Get config
config = DotMap(config)
# Get data
data_dict = DotMap(data())
# Get model
m = model()
if config.root.model_fname is None:
# Only train if this is not a pre-loaded model run
m.compile(optimizer=config.hyper.optimizer,
loss=config.hyper.loss,
metrics=config.hyper.metrics)
# Init callbacks
callbacks = [
TQDMCallback(),
TensorBoard(log_dir=ex.observers[0].dir,
histogram_freq=1,
write_graph=True,
write_images=False)
]
# Train
print(">>> Training...")
m.fit(x=data_dict.train_X,
y=data_dict.train_Y,
batch_size=config.hyper.batch_size,
epochs=config.hyper.nb_epochs,
verbose=2,
callbacks=callbacks,
validation_data=(data_dict.val_X, data_dict.val_Y))
m.save(os.path.join(ex.observers[0].dir, 'final_model.hdf5'))
# Test
print(">>> Evaluating...")
result = m.evaluate(x=data_dict.val_X,
y=data_dict.val_Y,
batch_size=config.hyper.batch_size,
verbose=1,
sample_weight=None)
# Write result to file for later use - loss and R^2
with open(os.path.join(ex.observers[0].dir, 'test_metrics.txt'),
'w') as f:
f.writelines([str(result), "loss, R^2"])
print(">>> Predicting cities...")
# 1. Make output dir
output_dir = os.path.join(ex.observers[0].dir, 'predicted_cities')
os.mkdir(output_dir)
# 2. Iterate
for x, fname in tqdm(
list(zip(data_dict.val_X, data_dict.val_filename_list))):
try:
# Reshape x
x = np.expand_dims(x, axis=0)
# Load city
with open(fname, 'r') as city_file_object:
city = cityiograph.City(city_file_object.read())
# Make prediction
pred = m.predict(x)
# Reshape
pred = pred.squeeze().flatten().tolist()
# Add features
city.update_values(pred, mode='traffic')
# Write to new file
fname_base = os.path.splitext(os.path.basename(fname))[0]
fname_output = os.path.join(output_dir,
fname_base + '_predicted.json')
with open(fname_output, 'w') as city_file_object_output:
city_file_object_output.write(city.to_json())
except Exception:
# Error parsing city or something else went wrong
print("Missed city {}.".format(fname))
pass
print("Process complete!!!")
def extract_data(directory, return_endpoints=False):
print("Extracting data for directory {}.".format(directory))
# Get all JSON files
# Recursive feature taken from https://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python
files = glob.glob(
directory, recursive=True) # Load list of filenames from the directory
cities = [] # City objects list
filenames = [] # List of raw filenames
inp = [] # Input feature matrix
out = [] # Output feature matrix
# Create dictionary mapping of bounds for both traffic and wait
# bounds = { 'low_traffic' : 0 , 'high_traffic' : 0 , 'low_wait' : 0 , 'high_wait' : 0 }
# Create np array objects for traffic and wait values
traffic = []
wait = []
for name in tqdm(files): # Using tqdm for loop analysis
with open(name, 'r') as f:
current_input = []
current_output = []
s = f.read()
city = cityiograph.City(s)
raw = os.path.splitext(os.path.basename(name))[0]
d = json.loads(s)
max_traffic = max([
cell['data']['traffic'] for cell in d['grid'] if 'data' in cell
])
max_wait = max(
[cell['data']['wait'] for cell in d['grid'] if 'data' in cell])
# Check bounds if needed
if return_endpoints:
traffic.append(max_traffic)
wait.append(max_wait)
# Ignore zero cities - suprisingly high percentage
if max_traffic == 0 or max_wait == 0:
continue
# Else, construct feature vector for city
for i in range(city.width):
for j in range(city.height):
cell = city.cells.get((i, j))
current_input.append(cell.population)
current_input.append(0) if (
cell.type_id == 6) else current_input.append(1)
current_output.append(cell.data["traffic"])
current_output.append(cell.data["wait"])
cities.append(city) # Append city structure
filenames.append(raw) # Append raw filename for later reference
inp.append(np.array(current_input))
out.append(np.array(current_output))
# Return correct data based on return_endpoints parameter
if return_endpoints:
return np.array(traffic), np.array(wait)
# Construct feature matrices and return
inp = np.array(inp).astype(int)
out = np.array(out).astype(int)
return cities, filenames, inp, out
def unique_city_generator(city_directory, ai_only=False):
"""Generator instance to give us cities that are different based on equals().
Args:
city_directory (str): directory (relative or absolute) with
.json city *output* files
ai_only (bool, optional): used for AI acceptance - if we only want the
AI city option
Yields:
3-tuple: unique City instance that can be used for analysis
and corresponding dictionary object + filename
"""
prev_city = None
good_count = 0
total_count = 0
files = glob.glob(city_directory + '*.json')
files.sort()
for city_path in tqdm(files):
total_count += 1
fname = os.path.basename(city_path)
# Validate time
the_city_time = get_time(fname)
if the_city_time < MIN_TIME:
continue
# Read file
with open(city_path, 'r') as f:
full_json_string = f.read()
# Get only the city described by ai or predict keys
d = json.loads(full_json_string)['ai']
if d is None:
# null AI -> use predict instead
if not ai_only:
d = json.loads(full_json_string)['predict']
else:
# Just ignore
yield None, None, fname
continue
j_string = json.dumps(d)
current_city = cityiograph.City(j_string)
if prev_city is None:
# First time, just yield
prev_city = current_city
good_count += 1
yield current_city, d, fname
elif not prev_city.equals(current_city):
# Different cities - yield new
prev_city = current_city
good_count += 1
yield current_city, d, fname
else:
# Equal
continue
print("Yield = {:.2%}. {} / {}.".format(
float(good_count) / total_count, good_count, total_count))
