def __init__(self):
self.log("Streets4Serial!")
# set random seed based on process rank
random_seed = settings["random_seed"] + (37)
seed(random_seed)
self.log("Reading OpenStreetMap data...")
data = GraphBuilder(settings["osm_file"])
self.log("Building street network...")
street_network = data.build_street_network()
if settings["persist_traffic_load"]:
self.log_indent("Saving street network to disk...")
persist_write("street_network_1.s4serial", street_network)
self.log("Locating area types...")
data.find_node_categories()
self.log("Generating trips...")
trip_generator = TripGenerator()
# distribute residents over processes
number_of_residents = settings["number_of_residents"]
if settings["use_residential_origins"]:
potential_origins = data.connected_residential_nodes
else:
potential_origins = street_network.get_nodes()
potential_goals = data.connected_commercial_nodes | data.connected_industrial_nodes
trips = trip_generator.generate_trips(number_of_residents, potential_origins, potential_goals)
# set traffic jam tolerance for this process and its trips
jam_tolerance = random()
self.log("Setting traffic jam tolerance to", str(round(jam_tolerance, 2)) + "...")
# run simulation
simulation = Simulation(street_network, trips, jam_tolerance, self.log_indent)
for step in range(settings["max_simulation_steps"]):
if step > 0 and step % settings["steps_between_street_construction"] == 0:
self.log_indent("Road construction taking place...")
simulation.road_construction()
if settings["persist_traffic_load"]:
persist_write("street_network_" + str(step + 1) + ".s4serial", simulation.street_network)
self.log("Running simulation step", step + 1, "of", str(settings["max_simulation_steps"]) + "...")
simulation.step()
total_traffic_load = simulation.traffic_load
simulation.cumulative_traffic_load = merge_arrays((total_traffic_load, simulation.cumulative_traffic_load))
if settings["persist_traffic_load"]:
self.log_indent("Saving traffic load to disk...")
persist_write("traffic_load_" + str(step + 1) + ".s4serial", total_traffic_load, is_array=True)
print(len(total_traffic_load))
del total_traffic_load
self.log("Done!")
def make_row(info):
formatters = (
TitleFormatter,
VideoFormatter,
AudioFormatter,
)
data = (info.title, info.video, info.audio)
return merge_arrays(f(d).make() for f, d in zip(formatters, data))
def __init__(self):
# get process info from mpi
communicator = MPI.COMM_WORLD
self.process_rank = communicator.Get_rank()
number_of_processes = communicator.Get_size()
self.log("Welcome to Streets4MPI!")
# set random seed based on process rank
random_seed = settings["random_seed"] + (37 * self.process_rank)
seed(random_seed)
self.log("Reading OpenStreetMap data...")
data = GraphBuilder(settings["osm_file"])
self.log("Building street network...")
street_network = data.build_street_network()
if self.process_rank == 0 and settings["persist_traffic_load"]:
self.log_indent("Saving street network to disk...")
persist_write("street_network_1.s4mpi", street_network)
self.log("Locating area types...")
data.find_node_categories()
self.log("Generating trips...")
trip_generator = TripGenerator()
# distribute residents over processes
number_of_residents = settings["number_of_residents"] / number_of_processes
if settings["use_residential_origins"]:
potential_origins = data.connected_residential_nodes
else:
potential_origins = street_network.get_nodes()
potential_goals = data.connected_commercial_nodes | data.connected_industrial_nodes
trips = trip_generator.generate_trips(number_of_residents, potential_origins, potential_goals)
# set traffic jam tolerance for this process and its trips
jam_tolerance = random()
self.log("Setting traffic jam tolerance to", str(round(jam_tolerance, 2)) + "...")
# run simulation
simulation = Simulation(street_network, trips, jam_tolerance, self.log_indent)
for step in range(settings["max_simulation_steps"]):
if step > 0 and step % settings["steps_between_street_construction"] == 0:
self.log_indent("Road construction taking place...")
simulation.road_construction()
if self.process_rank == 0 and settings["persist_traffic_load"]:
persist_write("street_network_" + str(step + 1) + ".s4mpi", simulation.street_network)
self.log("Running simulation step", step + 1, "of", str(settings["max_simulation_steps"]) + "...")
simulation.step()
# gather local traffic loads from all other processes
self.log("Exchanging traffic load data between nodes...")
total_traffic_load = array("I", repeat(0, len(simulation.traffic_load)))
communicator.Allreduce(simulation.traffic_load, total_traffic_load, MPI.SUM)
simulation.traffic_load = total_traffic_load
simulation.cumulative_traffic_load = merge_arrays((total_traffic_load, simulation.cumulative_traffic_load))
if self.process_rank == 0 and settings["persist_traffic_load"]:
self.log_indent("Saving traffic load to disk...")
persist_write("traffic_load_" + str(step + 1) + ".s4mpi", total_traffic_load, is_array = True)
del total_traffic_load
self.log("Done!")
def train(model, x, y, x_adv, aux_adv, batch_size=None, epochs=1, verbose=1, callbacks=None,
validation_split=0., shuffle=True, class_weight=None, sample_weight=None):
# Validate user data.
x, y, sample_weights = model._standardize_user_data(
x, y,
sample_weight=sample_weight,
class_weight=class_weight,
batch_size=batch_size)
ins = x + y + sample_weights
# Prepare validation data.
do_validation = False
if validation_split and 0. < validation_split < 1.:
do_validation = True
if hasattr(x[0], 'shape'):
split_at = int(x[0].shape[0] * (1. - validation_split))
else:
split_at = int(len(x[0]) * (1. - validation_split))
x, val_x = (slice_arrays(x, 0, split_at),
slice_arrays(x, split_at))
y, val_y = (slice_arrays(y, 0, split_at),
slice_arrays(y, split_at))
sample_weights, val_sample_weights = (
slice_arrays(sample_weights, 0, split_at),
slice_arrays(sample_weights, split_at))
val_ins = val_x + val_y + val_sample_weights
else:
val_ins = []
# ____________________________________________________________________________
# Fit
num_train_samples = x[0].shape[0]
index_array = np.arange(num_train_samples)
#sess = K.get_session()
#tf_x = K.placeholder(shape=(None, x[0].shape[1]))
#tf_y = K.placeholder(shape=(None, y[0].shape[1]))
#sess.run(tf.initialize_all_variables())
# Loop over epochs
for epoch in xrange(epochs):
print('Epoch {0}'.format(epoch))
if shuffle:
np.random.shuffle(index_array)
batches = make_batches(num_train_samples, batch_size)
# Loop over batches
for batch_index, (batch_start, batch_end) in enumerate(batches):
print('.. batch {0}'.format(batch_index))
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_arrays(ins, batch_ids)
assert isinstance(ins_batch, list) and len(ins_batch) == 1 + 2 + 2
# Add noise
if add_noise:
noise = x_adv[np.random.randint(0, x_adv.shape[0], ins_batch[0].shape[0])]
noise_reg = np.zeros_like(ins_batch[1]) + 100. # mask_value is set to 100
noise_discr = np.zeros_like(ins_batch[2])
noise_reg_w = np.ones_like(ins_batch[3])
noise_discr_w = np.ones_like(ins_batch[3])
ins_noise = [noise, noise_reg, noise_discr, noise_reg_w, noise_discr_w]
ins_batch = merge_arrays(ins_batch, ins_noise)
model._make_train_function()
f = model.train_function
outs = f(ins_batch)
def __init__(self):
# get process info from mpi
communicator = MPI.COMM_WORLD
self.process_rank = communicator.Get_rank()
number_of_processes = communicator.Get_size()
self.log("Welcome to Streets4MPI!")
# set random seed based on process rank
random_seed = settings["random_seed"] + (37 * self.process_rank)
seed(random_seed)
self.log("Reading OpenStreetMap data...")
data = GraphBuilder(settings["osm_file"])
self.log("Building street network...")
street_network = data.build_street_network()
if self.process_rank == 0 and settings["persist_traffic_load"]:
self.log_indent("Saving street network to disk...")
persist_write("street_network_1.s4mpi", street_network)
self.log("Locating area types...")
data.find_node_categories()
self.log("Generating trips...")
trip_generator = TripGenerator()
# distribute residents over processes
number_of_residents = settings[
"number_of_residents"] / number_of_processes
if settings["use_residential_origins"]:
potential_origins = data.connected_residential_nodes
else:
potential_origins = street_network.get_nodes()
potential_goals = data.connected_commercial_nodes | data.connected_industrial_nodes
trips = trip_generator.generate_trips(number_of_residents,
potential_origins,
potential_goals)
# set traffic jam tolerance for this process and its trips
jam_tolerance = random()
self.log("Setting traffic jam tolerance to",
str(round(jam_tolerance, 2)) + "...")
# run simulation
simulation = Simulation(street_network, trips, jam_tolerance,
self.log_indent)
for step in range(settings["max_simulation_steps"]):
if step > 0 and step % settings[
"steps_between_street_construction"] == 0:
self.log_indent("Road construction taking place...")
simulation.road_construction()
if self.process_rank == 0 and settings["persist_traffic_load"]:
persist_write("street_network_" + str(step + 1) + ".s4mpi",
simulation.street_network)
self.log("Running simulation step", step + 1, "of",
str(settings["max_simulation_steps"]) + "...")
simulation.step()
# gather local traffic loads from all other processes
self.log("Exchanging traffic load data between nodes...")
total_traffic_load = array("I",
repeat(0, len(simulation.traffic_load)))
communicator.Allreduce(simulation.traffic_load, total_traffic_load,
MPI.SUM)
simulation.traffic_load = total_traffic_load
simulation.cumulative_traffic_load = merge_arrays(
(total_traffic_load, simulation.cumulative_traffic_load))
if self.process_rank == 0 and settings["persist_traffic_load"]:
self.log_indent("Saving traffic load to disk...")
persist_write("traffic_load_" + str(step + 1) + ".s4mpi",
total_traffic_load,
is_array=True)
del total_traffic_load
self.log("Done!")