def main():
""" Main method """
window = GameWindow(SCREEN_WIDTH, SCREEN_HEIGHT, SCREEN_TITLE)
simulation_parameters = SimulationParameters(
population_count=650,
person_size=7,
frames_to_result=200,
chance_of_infection=0.6,
chance_for_immunity=1.0,
chance_for_death=0.05,
initial_top_speed=250,
reporting_interval=10,
initial_infected_people=1,
initial_immune_people=0,
vertical_walls=4,
door_size=100,
)
# for chance_of_infection in [0.4, 0.7]:
# simulation_parameters.chance_of_infection = chance_of_infection
window.setup(simulation_parameters)
arcade.run()
plot_results(window)
window.close()
def puertoPredictions(x_train, y_train, x_validation, validation_ids):
x_train, x_test, y_train, y_test = train_test_split(
x_train, y_train, test_size=.15,
random_state=42) #Splits to 10% test data
model = Sequential()
addNeuralNetworkLayers(model, len(x_train[0]), 1)
model.compile(loss='binary_crossentropy',
optimizer=optimizers.Adam(lr=0.001),
metrics=["binary_accuracy"])
nEpochs = 60
batchSize = 512
print('Training!')
#trainNeuralNetwork(model, batchSize, nEpochs, x_train, y_train)
roc_auc = trainNeuralNetwork(model,
batchSize,
nEpochs,
x_train,
y_train,
x_test=x_test,
y_test=y_test)
plot_results(roc_auc.gini, roc_auc.gini_val)
nn_output = model.predict(x_validation)
export_csv_file("output.csv", validation_ids, nn_output)
def fetch_model(train_gen, val_gen, metadata):
model_path = "models\\model" +\
"_R"+str(metadata['Nrows']) +\
"_C"+str(metadata['Ncols']) +\
"_fs"+str(metadata['FILTER_SIZE'][0]) +\
"_ep"+str(metadata['NUM_EPOCHS']) +\
".h5"
try:
print("\nLoading saved model")
model = tf.keras.models.load_model(model_path)
print("\nmodel loaded")
except:
print("\nModel not found. Training new model...")
pre_trained_model = InceptionV3(input_shape=(metadata['Nrows'],
metadata['Ncols'], 3),
include_top=False,
weights='imagenet')
for layer in pre_trained_model.layers:
layer.trainable = False
last_layer = pre_trained_model.get_layer('mixed7')
last_output = last_layer.output
x = tf.keras.layers.Conv2D(256,
metadata['FILTER_SIZE'],
activation='relu')(last_output)
x = tf.keras.layers.MaxPooling2D(2, 2)(x)
x = tf.keras.layers.Flatten()(x)
x = tf.keras.layers.Dense(1024, activation='relu')(x)
x = tf.keras.layers.Dense(1, activation='sigmoid')(x)
model = tf.keras.models.Model(pre_trained_model.input, x)
## compile model
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
model.summary()
## fit model to data - training
history = model.fit_generator(train_gen,
epochs=metadata['NUM_EPOCHS'],
validation_data=val_gen,
verbose=1,
callbacks=[callback])
print("\nNew model trained")
## save model to file
print("\nSaving model for later use...")
model.save(model_path)
print("\nModel Successfully saved")
## plot results
print("\nPlotting results...")
pltres.plot_results(history)
print("\n........................")
return model
def main():
c = Central(300)
params = Parameters(c)
config = Configuration(
initial_state=params.initial_state(),
partial_state_update_blocks=params.partial_state_update_blocks,
sim_config=simulation_parameters)
exec_mode = ExecutionMode()
exec_context = ExecutionContext(exec_mode.single_proc)
executor = Executor(exec_context, [config])
raw_result, tensor = executor.execute()
raw_result = [d for d in raw_result]
plot_results(raw_result)
def main(run_simulations=True, parallel=False):
"""Main function that runs all the exercises."""
save = save_plots()
pylog.info('Running network')
run_network(plot=not save)
pylog.info('Running simulation exercises')
arguments = []
arguments = (['example', '8b', '8c', '8d1', '8d2', '8e', '8f']
if run_simulations else [])
if parallel:
pool = Pool(processes=4)
pool.map(exercise_all, [[arg] for arg in arguments])
else:
exercise_all(arguments=arguments)
pylog.info('Plotting simulation results')
plot_results(plot=not save)
def launch_experiments(args):
# create Theano variables for input and target minibatch
input_var = T.fmatrix('X')
# Create tang function
# need abs to avoid nan with pow on the GPU
f0 = 0.5 * (T.pow(abs(input_var), 4) - 16 * T.pow(abs(input_var), 2) + 5 * input_var)
tang_output = f0.sum(axis=-1)
# Create and train network normally
network = create_student_model(input_var)
out = standard_train(args, input_var, network, tang_output)
# Create and train network with Sobolev
network_sobolev = create_student_model(input_var)
out_sobolev = sobolev_train(args, input_var, network_sobolev, tang_output)
# Now plot and compare outputs
plot_results.plot_results(args, out, out_sobolev)
def launch_experiments(args):
# create Theano variables for input and target minibatch
input_var = T.fmatrix('X')
# Create tang function
# need abs to avoid nan with pow on the GPU
f0 = 0.5 * (T.pow(abs(input_var), 4) - 16 * T.pow(abs(input_var), 2) +
5 * input_var)
tang_output = f0.sum(axis=-1)
# Create and train network normally
network = create_student_model(input_var)
out = standard_train(args, input_var, network, tang_output)
# Create and train network with Sobolev
network_sobolev = create_student_model(input_var)
out_sobolev = sobolev_train(args, input_var, network_sobolev, tang_output)
# Now plot and compare outputs
plot_results.plot_results(args, out, out_sobolev)
def run_results(results, exp_name, n_epochs):
# extract results data
train_losses = results[0]
valid_losses = results[1]
train_accs = results[2]
valid_accs = results[3]
test_acc = results[4]
epoch_times = results[5]
train_time = results[6]
res_file_name = exp_name + '_test_acc_' + f'{test_acc:.1f}'
# save the results
with open(res_file_name + '.pkl', 'wb') as f:
pl.dump(results, f)
print('Saved results data file: ', res_file_name + '.pkl')
pic_name = res_file_name + '.png'
pr.plot_results([train_losses, valid_losses],
n_epochs, pic_name)
df.columns.get_loc(attr_2)]].values
print("Data is ready")
# Normalization
X = normalize(X)
print("Normalize finished")
# MACHINE LEARNING MODELLING
y_kmeans, kmeans, n_cluster = KMeansIMP(X)
print("KMeans applied")
# RESULT EXTRACTION#
# arr contains number of FFP per cluster
customerWithFFP = []
arr = findNumOfCustomerInCluster(y_kmeans)
# Adding Cluster ID to map with FFP_ID
a = np.array(y_kmeans)[np.newaxis]
df = appendWithClusterID(df, a.T)
print("Cluster ID has been added to dataFrame")
# Result Extraction
attributeArray = result_extraction(df, y_kmeans, n_cluster)
print("Result Extraction finished")
showResultsOnGrid(n_cluster, attributeArray)
print("Plotting")
# PLOTTING RESULTS
plot_results(X, y_kmeans, kmeans, n_cluster)
map(lambda x: x['duration'],
run_data['kernel_executions'])))
buffer_reads_hhal[idx].append(
list(
map(
lambda x: {
'size': x['size'],
'duration': x['duration']
}, run_data['buffer_reads'])))
buffer_writes_hhal[idx].append(
list(
map(
lambda x: {
'size': x['size'],
'duration': x['duration']
}, run_data['buffer_writes'])))
if mango:
return exp_sizes, total_durations, buffer_reads, buffer_writes, kernel_execs, resource_allocations, buffer_reads_hhal, buffer_writes_hhal, kernel_execs_hhal
else:
return exp_sizes, total_durations, buffer_reads, buffer_writes, kernel_execs
plot_results(exp_nvidia_dir,
exp_mango_dir,
exp_opencl_dir,
get_data,
'pathfinder',
buffer_write_unit='megabytes',
kernel_executions_time_unit='μs',
buffer_reads_time_unit='μs')
# Resolution in pc:
cli.resolution = 100.0
# Photons per grid cell:
cli.photons_temperature = 1
cli.photons_raytracing = 1
cli.photons_imaging = 1
for cli.case in [1, 2]:
for cli.opticaldepth in [0.256434, 1.28217, 6.41084]:
for cli.mode in ["temperature", "images", "seds"]:
# Setup step:
setup_model(cli);
# Compute step:
file = filename(cli, cli.mode)
model = Model.read(file+".rtin")
model.write("temp.rtin")
model.run(filename=file+".rtout", logfile=file+".hyout", overwrite=True)
# Plotting step:
#TODO: if(cli.mode == "temperature"): visualize_setup + visualize_results => plot_setup
if(cli.mode != "temperature"):
plot_results(cli);
# Export to fits format:
if(cli.mode == "images"):
export_to_fits(cli);
def GIANT(model,
worker,
device,
max_iterations=100,
max_communication_rounds=200,
gradient_norm_tolerance=1e-8,
line_search_rho=1e-4,
line_search_max_iterations=50):
"""Run the GIANT algorithm.
Arguments:
model (torch.nn.Module): A model to optimize.
worker (Worker): A Worker class instance. The worker on the driver is
used to record test accuracy.
device (torch.device): The device tensors will be allocated to.
max_iterations (int, optional): The maximum number of iterations.
max_communication_rounds (int, optional): The maximum number of
communication rounds.
gradient_norm_tolerance (float, optional): The smallest the norm of the
full gradient can be before the algorithm stops.
line_search_rho (float, optional): Armijo line search parameter.
line_search_max_iterations (int, optional): The maximum number of line
search iterations.
"""
rank = dist.get_rank()
dimension = sum([p.numel() for p in model.parameters()])
num_workers = dist.get_world_size() - 1
cpu = torch.device("cpu")
if rank == 0:
print("\n{:-^106s}\n".format(" GIANT "))
# Results are added to these lists and then plotted.
cumulative_communication_rounds_list = [0]
cumulative_time_list = [0]
gradient_norm_list = []
loss_list = []
step_size_list = []
test_accuracy_list = []
# We will store a message about why the algorithm stopped.
end_message = "max_iterations reached"
iteration = 0
total_communication_rounds = 0
subproblem_failed = False
while iteration < max_iterations:
if total_communication_rounds >= max_communication_rounds:
end_message = 'max_communication_rounds reached'
break
#-------------------------- GIANT iteration ---------------------------
if rank == 0:
# The driver will record how long each iteration takes.
iteration_start_time = time()
if iteration == 0:
# This iteration doesn't require initial update to workers' model.
if rank > 0:
local_loss = worker.get_local_loss(model).cpu()
local_gradient = worker.get_local_gradient(model).cpu()
local_loss_and_gradient = torch.cat(
[local_loss.reshape(1, 1), local_gradient], dim=0)
# Workers send local objective value and gradient to driver.
dist.reduce(local_loss_and_gradient, 0)
total_communication_rounds += 1
if rank == 0:
loss_and_gradient = torch.zeros(1 + dimension, 1, device=cpu)
dist.reduce(loss_and_gradient, 0)
total_communication_rounds += 1
loss_and_gradient = loss_and_gradient.to(device)
loss_and_gradient *= 1.0 / num_workers
loss = loss_and_gradient[0, 0]
gradient = loss_and_gradient[1:]
else:
# Broadcast step_size to update workers' model and compute gradient.
if rank == 0:
# step_size is from previous line search.
step_size = torch.tensor(float(step_size),
device=cpu).reshape(1, 1)
dist.broadcast(step_size, 0)
total_communication_rounds += 1
if rank > 0:
step_size = torch.zeros(1, 1, device=cpu)
dist.broadcast(step_size, 0)
total_communication_rounds += 1
step_size = step_size.item()
# This worker node has direction from previous line search.
model = get_updated_model(model, direction, step_size)
local_gradient = worker.get_local_gradient(model).cpu()
dist.reduce(local_gradient, 0)
total_communication_rounds += 1
if rank == 0:
gradient = torch.zeros(dimension, 1, device=cpu)
dist.reduce(gradient, 0)
total_communication_rounds += 1
gradient = gradient.to(device)
gradient *= 1.0 / num_workers
if rank == 0:
dist.broadcast(gradient.cpu(), 0)
total_communication_rounds += 1
gradient_norm = torch.norm(gradient)
if gradient_norm <= gradient_norm_tolerance:
end_message = 'gradient_norm_tolerance reached'
break
if rank > 0:
gradient = torch.zeros(dimension, 1, device=cpu)
dist.broadcast(gradient, 0)
total_communication_rounds += 1
gradient = gradient.to(device)
gradient_norm = torch.norm(gradient)
if gradient_norm <= gradient_norm_tolerance:
break
local_hessian_inverse_times_gradient \
= worker.get_local_hessian_inverse_times_vector(model,
gradient, "CG")
# CG could have failed.
if local_hessian_inverse_times_gradient is None:
v = torch.cat([
torch.ones((1, 1), device=cpu),
torch.zeros((dimension, 1), device=cpu)
],
dim=0)
# The 1 indicates that CG failed.
else:
v = torch.cat([
torch.zeros((1, 1), device=cpu),
local_hessian_inverse_times_gradient.cpu()
],
dim=0)
# The 0 indicates that CG passed.
dist.reduce(v, 0)
total_communication_rounds += 1
if rank == 0:
direction = torch.zeros(1 + dimension, 1, device=cpu)
dist.reduce(direction, 0)
total_communication_rounds += 1
dist.broadcast(direction, 0)
total_communication_rounds += 1
if int(direction[0, 0]) > 0: # CG failed on at least 1 worker node
end_message = 'CG failed on {} worker nodes'.format(
int(direction[0, 0]))
subproblem_failed = True
break
direction = direction.to(device)
direction = direction[1:]
direction *= -1.0 / num_workers
if rank > 0:
direction = torch.zeros(1 + dimension, 1, device=cpu)
dist.broadcast(direction, 0)
total_communication_rounds += 1
if int(direction[0, 0]) > 0: # CG failed on at least 1 worker node
break
direction = direction.to(device)
direction = direction[1:]
direction *= -1.0 / num_workers
local_line_search_values = get_local_line_search_values(
model, worker, direction, line_search_max_iterations).cpu()
dist.reduce(local_line_search_values, 0)
total_communication_rounds += 1
if rank == 0:
line_search_values = torch.zeros(1,
line_search_max_iterations,
device=cpu)
dist.reduce(line_search_values, 0)
total_communication_rounds += 1
line_search_values = line_search_values.to(device)
line_search_values *= 1.0 / num_workers
new_model, new_loss, line_search_exp, step_size = line_search(
model, line_search_values, loss, gradient, direction,
line_search_rho)
iteration_time = time() - iteration_start_time
#------------------------------ Printing ------------------------------
# This time is not recorded.
if rank == 0:
loss_list.append(loss)
gradient_norm_list.append(gradient_norm)
step_size_list.append(step_size)
# Recall that the driver stores the test dataset in its worker.
test_accuracy = worker.get_accuracy(model)
test_accuracy_list.append(test_accuracy)
cumulative_communication_rounds_list.append(
total_communication_rounds)
cumulative_time_list.append(cumulative_time_list[-1] +
iteration_time)
print_row(iteration, total_communication_rounds,
iteration_time, line_search_exp, step_size,
torch.norm(direction), loss, gradient_norm,
test_accuracy)
loss = new_loss
model = new_model
iteration += 1
# Print final row.
if (iteration == max_iterations
or total_communication_rounds >= max_communication_rounds):
# Need to first get gradient norm on the final model.
if rank == 0:
step_size = torch.tensor(float(step_size),
device=cpu).reshape(1, 1)
dist.broadcast(step_size, 0)
if rank > 0:
step_size = torch.zeros(1, 1, device=cpu)
dist.broadcast(step_size, 0)
step_size = step_size.item()
model = get_updated_model(model, direction, step_size)
local_gradient = worker.get_local_gradient(model).cpu()
dist.reduce(local_gradient, 0)
if rank == 0:
gradient = torch.zeros(dimension, 1, device=cpu)
dist.reduce(gradient, 0)
gradient = gradient.to(device)
gradient *= 1.0 / num_workers
gradient_norm = torch.norm(gradient)
if rank == 0:
loss_list.append(loss)
gradient_norm_list.append(gradient_norm)
test_accuracy = worker.get_accuracy(model)
test_accuracy_list.append(test_accuracy)
print_row(iteration=iteration,
objective_value=loss,
gradient_norm=gradient_norm,
test_accuracy=test_accuracy,
is_final_row=True)
print("\n{} after {:.2f} seconds\n".format(end_message,
cumulative_time_list[-1]))
plot_results(loss_list,
gradient_norm_list,
test_accuracy_list,
cumulative_communication_rounds_list,
step_size_list,
label="GIANT",
max_x=max_communication_rounds,
failed=subproblem_failed)
def ran_simulation():
"""
Main ran_simulation
"""
# define sim_param and inside RB pool: all available Resources list
sim_param = SimParam()
no_of_slices = sim_param.no_of_slices
no_of_users_per_slice = sim_param.no_of_users_per_slice
# create result directories
create_dir(sim_param)
# create logfile and write SimParameters
results_dir = "results/" + sim_param.timestamp
log_file = open(results_dir + "/logfile.txt", "wt")
log_file.write('no_of_slices: %d\nno_of_users_per_slice: %d\n\n' % (no_of_slices, no_of_users_per_slice))
attrs = vars(sim_param)
log_file.write('SimParam\n'+''.join("%s: %s\n" % item for item in attrs.items()))
# log_file.close()
# initialize SD_RAN_Controller
SD_RAN_Controller = Controller(sim_param)
# Each slice has different users
slices = []
slice_results = []
# initialize all slices
for i in range(no_of_slices):
slice_param_tmp = SliceParam(sim_param)
slice_param_tmp.SLICE_ID = i
slices.append(SliceSimulation(slice_param_tmp))
slice_results.append([])
# initialize all users with traffics and distances
tmp_users = []
seed_dist = 0 # users in all slices have identical distance distributions
#rng_dist = RNG(ExponentialRNS(lambda_x=1. / float(sim_param.MEAN_Dist)), s_type='dist') # , the_seed=seed_dist
rng_dist = RNG(UniformRNS(sim_param.DIST_MIN,sim_param.DIST_MAX, the_seed=seed_dist), s_type='dist') #
dist_arr = [10, 100 ]#[30, 30, 100, 100, 100, 100, 100, 100, 100, 100] # 10*(1+user_id%no_of_users_per_slice)**2
for j in range(no_of_users_per_slice):
user_id = i*no_of_users_per_slice + j
#tmp_users.append(User(user_id, rng_dist.get_dist(), slice_list=[slices[i]], sim_param=sim_param))
tmp_users.append(User(user_id, dist_arr[j], slice_list=[slices[i]], sim_param=sim_param))
# insert user to slices
slices[i].insert_users(tmp_users)
# Choose Slice Manager Algorithm, 'PF': prop fair, 'MCQI': Max Channel Quality Index, 'RR': round-robin
slices[0].slice_param.SM_ALGO = 'RR'
slices[1].slice_param.SM_ALGO = 'MCQI'
slices[2].slice_param.SM_ALGO = 'PF'
# log Slice Parameters
for i in range(no_of_slices):
attrs = vars(slices[i].slice_param)
log_file.write('\nSliceParam\n' + ''.join("%s: %s\n" % item for item in attrs.items()))
#log_file.close()
# loop rounds for each slice
for i in range(int(sim_param.T_FINAL/sim_param.T_C)):
RB_mapping = SD_RAN_Controller.RB_allocate_to_slices(slices[0].sim_state.now, slices)
for j in range(len(slices)):
slices[j].prep_next_round(RB_mapping[j,:,:])
slice_results[j].append(slices[j].simulate_one_round())
# Store Simulation Results
# user results
parent_dir = "results/" + sim_param.timestamp + "/user_results"
path = parent_dir + "/tp"
for i in range(len(slice_results)):
user_count = len(slice_results[i][-1].server_results) # choose latest result for data
for k in range(user_count):
common_name = "/slice%d_user%d_" % (i, slice_results[i][-1].server_results[k].server.user.user_id)
cc_temp = slice_results[i][-1].server_results[k].server.counter_collection
# tp
filename = parent_dir + "/tp" + common_name + "sum_power_two.csv"
savetxt(filename, cc_temp.cnt_tp.sum_power_two, delimiter=',')
filename = parent_dir + "/tp" + common_name + "values.csv"
savetxt(filename, cc_temp.cnt_tp.values, delimiter=',')
filename = parent_dir + "/tp" + common_name + "timestamps.csv"
savetxt(filename, cc_temp.cnt_tp.timestamps, delimiter=',')
filename = parent_dir + "/tp" + common_name + "all_data.csv"
#savetxt(filename, np.transpose(np.array([cc_temp.cnt_tp.values,cc_temp.cnt_tp.timestamps])), delimiter=',')
df = DataFrame(np.transpose(np.array([cc_temp.cnt_tp.values,cc_temp.cnt_tp.timestamps])), columns=['Values', 'Timestamps'])
export_csv = df.to_csv(filename, index=None, header=True)
# tp2
filename = parent_dir + "/tp2" + common_name + "sum_power_two.csv"
savetxt(filename, cc_temp.cnt_tp2.sum_power_two, delimiter=',')
filename = parent_dir + "/tp2" + common_name + "values.csv"
savetxt(filename, cc_temp.cnt_tp2.values, delimiter=',')
filename = parent_dir + "/tp2" + common_name + "timestamps.csv"
savetxt(filename, cc_temp.cnt_tp2.timestamps, delimiter=',')
# ql
filename = parent_dir + "/ql" + common_name + "sum_power_two.csv"
savetxt(filename, cc_temp.cnt_ql.sum_power_two, delimiter=',')
filename = parent_dir + "/ql" + common_name + "values.csv"
savetxt(filename, cc_temp.cnt_ql.values, delimiter=',')
filename = parent_dir + "/ql" + common_name + "timestamps.csv"
savetxt(filename, cc_temp.cnt_ql.timestamps, delimiter=',')
# system time (delay)
filename = parent_dir + "/delay" + common_name + "values.csv"
savetxt(filename, cc_temp.cnt_syst.values, delimiter=',')
filename = parent_dir + "/delay" + common_name + "timestamps.csv"
savetxt(filename, cc_temp.cnt_syst.timestamps, delimiter=',')
# Find how to insert histograms
# plot results
parent_dir = "results/" + sim_param.timestamp
plot_results(parent_dir, no_of_slices, no_of_users_per_slice, sim_param, slices)
# rb dist printing
filename = "results/" + sim_param.timestamp + "/summary"
rb_total = 0
rb_dist = []
for s in slices:
rb_dist_slice = []
for u in s.server_list:
rb_dist_slice.append(u.RB_counter)
slicesum = np.nansum(rb_dist_slice)
print("Slice %d dist: " % s.slice_param.SLICE_ID, *np.round(np.divide(rb_dist_slice,slicesum/100), 1))
# write these to file savetxt(filename, cc_temp.cnt_ql.sum_power_two, delimiter=',')
rb_dist.append(slicesum)
totalsum = np.nansum(rb_dist)
print("rb dist (RR MCQI PF): ", *np.round(np.divide(rb_dist, totalsum / 100), 1))
def DINGO(model,
worker,
device,
theta=1e-4,
phi=1e-6,
max_iterations=100,
max_communication_rounds=200,
gradient_norm_tolerance=1e-8,
line_search_rho=1e-4,
line_search_max_iterations=50):
"""Run the DINGO algorithm.
Arguments:
model (torch.nn.Module): A model to optimize.
worker (Worker): A Worker class instance.
device (torch.device): The device tensors will be allocated to.
theta (float, optional): The hyperparameter theta in the DINGO
algorithm.
phi (float, optional): The hyperparameter phi in the DINGO algorithm.
max_iterations (int, optional): The maximum number of iterations.
max_communication_rounds (int, optional): The maximum number of
communication rounds.
gradient_norm_tolerance (float, optional): The smallest the norm of the
full gradient can be before the algorithm stops.
line_search_rho (float, optional): Armijo line search parameter.
line_search_max_iterations (int, optional): The maximum number of line
search iterations.
"""
rank = dist.get_rank()
dimension = sum([p.numel() for p in model.parameters()])
num_workers = dist.get_world_size() - 1
cpu = torch.device("cpu")
if rank == 0:
print("\n{:-^116s}\n".format(" DINGO "))
# Results are added to these lists and then plotted.
cumulative_communication_rounds_list = [0]
cumulative_time_list = [0]
gradient_norm_list = []
loss_list = []
step_size_list = []
test_accuracy_list = []
# We will store a message about why the algorithm stopped.
end_message = "max_iterations reached"
iteration = 0
total_communication_rounds = 0
subproblem_failed = False
while iteration < max_iterations:
if total_communication_rounds >= max_communication_rounds:
end_message = 'max_communication_rounds reached'
break
#-------------------------- DINGO iteration ---------------------------
if rank == 0:
# The driver will record how long each iteration takes.
iteration_start_time = time()
if iteration == 0:
# This iteration requires an additional communication round.
'''
In subsequent iterations, the driver will broadcast the step-size
to update workers' model.
'''
if rank > 0:
local_loss = worker.get_local_loss(model).cpu()
local_gradient = worker.get_local_gradient(model).cpu()
local_loss_and_gradient = torch.cat(
[local_loss.reshape(1, 1), local_gradient], dim=0)
# Workers send local objective value and gradient to driver.
dist.reduce(local_loss_and_gradient, 0)
total_communication_rounds += 1
if rank == 0:
loss_and_gradient = torch.zeros(1 + dimension, 1, device=cpu)
dist.reduce(loss_and_gradient, 0)
total_communication_rounds += 1
loss_and_gradient = loss_and_gradient.to(device)
loss_and_gradient *= 1.0 / num_workers
loss = loss_and_gradient[0, 0]
gradient = loss_and_gradient[1:]
step_size = 0.0
if rank == 0:
# The driver broadcasts the step-size and gradient to all workers.
# step_size is from previous line search.
alpha_and_gradient = torch.cat([
torch.tensor(float(step_size), device=cpu).reshape(1, 1),
gradient.cpu()
],
dim=0)
dist.broadcast(alpha_and_gradient, 0)
total_communication_rounds += 1
gradient_norm = torch.norm(gradient)
if gradient_norm <= gradient_norm_tolerance:
end_message = 'gradient_norm_tolerance reached'
break
gradient_norm_squared = gradient_norm.pow(2)
if rank > 0:
alpha_and_gradient = torch.zeros(1 + dimension, 1, device=cpu)
dist.broadcast(alpha_and_gradient, 0)
total_communication_rounds += 1
alpha_and_gradient = alpha_and_gradient.to(device)
step_size = alpha_and_gradient[0, 0]
gradient = alpha_and_gradient[1:]
if iteration > 0:
# All workers update to current model.
# This worker node has direction from previous line search.
model = get_updated_model(model, direction, step_size)
gradient_norm = torch.norm(gradient)
if gradient_norm <= gradient_norm_tolerance:
break
# Update Direction.
temp = get_update_direction(model, worker, device, gradient, theta,
phi)
if rank == 0:
if temp is None:
end_message = 'CG failed'
subproblem_failed = True
break
direction, hessian_times_gradient, case, rounds = temp
total_communication_rounds += rounds
if rank > 0:
if temp is None:
break
direction, rounds = temp
total_communication_rounds += rounds
# Line Search.
if rank > 0:
local_line_search_matrix = get_local_line_search_matrix(
model, worker, direction, line_search_max_iterations)
dist.reduce(local_line_search_matrix, 0)
total_communication_rounds += 1
if rank == 0:
# The driver averages all workers' local line-search matrix.
line_search_matrix = torch.zeros(1 + dimension,
line_search_max_iterations,
device=cpu)
dist.reduce(line_search_matrix, 0)
total_communication_rounds += 1
line_search_matrix = line_search_matrix.to(device)
line_search_matrix *= 1.0 / num_workers
new_model, new_loss, new_gradient, line_search_exp, step_size \
= line_search(model, line_search_matrix, gradient_norm_squared,
direction, hessian_times_gradient,
line_search_rho)
iteration_time = time() - iteration_start_time
#------------------------------ Printing ------------------------------
# This time is not recorded.
if rank == 0:
loss_list.append(loss)
gradient_norm_list.append(gradient_norm)
step_size_list.append(step_size)
# Recall that the driver stores the test dataset in its worker.
test_accuracy = worker.get_accuracy(model)
test_accuracy_list.append(test_accuracy)
cumulative_communication_rounds_list.append(
total_communication_rounds)
cumulative_time_list.append(cumulative_time_list[-1] +
iteration_time)
print_row(iteration, total_communication_rounds,
iteration_time, case, line_search_exp, step_size,
torch.norm(direction), loss, gradient_norm,
test_accuracy)
loss = new_loss
gradient = new_gradient
model = new_model
iteration += 1
# Print final row.
if rank == 0:
loss_list.append(loss)
gradient_norm = torch.norm(gradient)
gradient_norm_list.append(gradient_norm)
test_accuracy = worker.get_accuracy(model)
test_accuracy_list.append(test_accuracy)
print_row(iteration=iteration,
objective_value=loss,
gradient_norm=gradient_norm,
test_accuracy=test_accuracy,
is_final_row=True)
print("\n{} after {:.2f} seconds\n".format(end_message,
cumulative_time_list[-1]))
plot_results(loss_list,
gradient_norm_list,
test_accuracy_list,
cumulative_communication_rounds_list,
step_size_list,
label="DINGO",
max_x=max_communication_rounds,
failed=subproblem_failed)
cov = cov + np.eye(J * N) * variance
covi = np.linalg.inv(cov)
likelihood_list = []
for i in range(I):
likelihood = [e.T.dot(covi).dot(e) for e in error[:, i, :]]
likelihood = np.reshape(likelihood, (n_configuration, len(theta)))
likelihood_list.append(likelihood)
likelihood = np.array(likelihood_list)
argmin = np.nanargmin(likelihood, axis=2).T
estimate = theta[argmin, :]
estimate_error = estimate - transmitters_coordinates[None, :, :]
# Compute CRLB
K = 10 * gamma / np.log(10)
dYb = K * (vector / d[:, :, None]**2)
dYb = np.repeat(np.moveaxis(dYb, 1, 0), N, axis=1)
fisher = (i.T.dot(covi).dot(i) for i in dYb)
crlb = np.array([np.linalg.inv(i) for i in fisher])
with lzma.open(cache_file, "wb") as fp:
dill.dump((transmitters_coordinates, estimate, ap_coordinates, crlb),
fp)
if __name__ == '__main__':
# Plotting results
p = plot_results(transmitters_coordinates, estimate, ap_coordinates, crlb)
show(row(*p))
def main():
try:
all_data = pickle.load(
open(settings.data_folder + settings.input_file, "rb"))
print('Data loaded from pickle')
except FileNotFoundError:
print('Start serializing data')
participant_list = serialize_data()
print('Start processing data')
all_dataframes = create_dataframes(participant_list)
print('Start processing commands')
all_dataframes = analyse_commands(all_dataframes)
# print('Start processing conflicts')
# all_dataframes = analyse_conflicts(participant_list)
print('Start loading SSDs')
all_data = ssd_loader(all_dataframes)
print('Start plotting')
plot_commands(all_data)
# plot_traffic()
print('Start training the neural network')
# measure training time
start_time = time.time()
metrics_all_df = pd.DataFrame()
for target_type in settings.target_types:
settings.target_type = target_type
for participant in settings.participants:
settings.current_participant = participant
participant_ids = [participant]
for ssd_condition in settings.ssd_conditions:
settings.ssd = ssd_condition
settings.iteration_name = '{}_{}_{}_{}'.format(
target_type, participant, settings.experiment_name,
ssd_condition)
print('------------------------------------------------')
print('-- Start training:', settings.iteration_name)
print('------------------------------------------------')
""" TRAIN MODEL """
metrics_iteration_df = ssd_trainer(all_data, participant_ids)
""" SAVE TO DISK """
if not metrics_iteration_df.empty:
metrics_all_df = metrics_iteration_df if metrics_all_df.empty else metrics_all_df.append(
metrics_iteration_df)
metrics_all_df.to_csv(
settings.output_dir +
'/metrics_{}.csv'.format(settings.experiment_name))
if settings.callback_save_model:
remove_redundant_weights.main()
print('Train time: {} min'.format(
round(int(time.time() - start_time) / 60), 1))
plot_results(settings.experiment_name)
if settings.callback_save_model and settings.participants != ['all']:
remove_redundant_weights.main(
) # only keeps the models with the best validation MCC.
auto_validator.main()
compare_performance_consistency()
def AIDE(model,
worker,
device,
eta=1.0,
mu=0.0,
tau=0.0,
subproblem_step_size=1.0,
max_iterations=100,
max_communication_rounds=200,
gradient_norm_tolerance=1e-8):
"""Run the AIDE algorithm.
Arguments:
model (torch.nn.Module): A model to optimize.
worker (Worker): A Worker class instance.
device (torch.device): The device tensors will be allocated to.
eta (float, optional): The hyperparameter eta in the InexactDANE
algorithm.
mu (float, optional): The hyperparameter mu in the InexactDANE
algorithm.
tau (float, optional): The hyperparameter tau in the AIDE algorithm.
subproblem_step_size (float, optional): The step size used by the
subproblem solver.
max_iterations (int, optional): The maximum number of iterations.
max_communication_rounds (int, optional): The maximum number of
communication rounds.
gradient_norm_tolerance (float, optional): The smallest the norm of the
full gradient can be before the algorithm stops.
"""
rank = dist.get_rank()
dimension = sum([p.numel() for p in model.parameters()])
num_workers = dist.get_world_size() - 1
cpu = torch.device("cpu")
if rank == 0:
print("\n{:-^76s}\n".format(" AIDE "))
# Results are added to these lists and then plotted.
cumulative_communication_rounds_list = [0]
cumulative_time_list = [0]
gradient_norm_list = []
loss_list = []
test_accuracy_list = []
# We will store a message about why the algorithm stopped.
end_message = "max_iterations reached"
else:
AIDE_y = get_model_weights(model)
zeta = 0.5
iteration = 0
total_communication_rounds = 0
while iteration < max_iterations:
if total_communication_rounds >= max_communication_rounds:
end_message = 'max_communication_rounds reached'
break
#-------------------------- AIDE iteration ----------------------------
if rank == 0:
# The driver will record how long each iteration takes.
iteration_start_time = time()
if iteration > 0:
# This iteration requires an initial update to workers' model.
if rank == 0:
# new_weights is from previous iteration.
model = replace_model_weights(model, new_weights)
dist.broadcast(new_weights.cpu(), 0)
total_communication_rounds += 1
if rank > 0:
old_weights = get_model_weights(model)
new_weights = torch.zeros(dimension, 1, device=cpu)
dist.broadcast(new_weights, 0)
total_communication_rounds += 1
new_weights = new_weights.to(device)
model = replace_model_weights(model, new_weights)
AIDE_y, zeta = get_new_y_and_new_zeta(old_weights, new_weights,
tau, zeta)
if rank > 0:
local_loss = worker.get_local_loss(model).cpu()
local_gradient = worker.get_local_gradient(model).cpu()
local_loss_and_gradient = torch.cat(
[local_loss.reshape(1, 1), local_gradient], dim=0)
# All workers send local objective value and gradient to driver.
dist.reduce(local_loss_and_gradient, 0)
total_communication_rounds += 1
if rank == 0:
loss_and_gradient = torch.zeros(1 + dimension, 1, device=cpu)
dist.reduce(loss_and_gradient, 0)
total_communication_rounds += 1
loss_and_gradient = loss_and_gradient.to(device)
loss_and_gradient *= 1.0 / num_workers
loss = loss_and_gradient[0, 0]
gradient = loss_and_gradient[1:]
dist.broadcast(gradient.cpu(), 0)
total_communication_rounds += 1
gradient_norm = torch.norm(gradient)
if gradient_norm <= gradient_norm_tolerance:
end_message = 'gradient_norm_tolerance reached'
break
if rank > 0:
gradient = torch.zeros(dimension, 1, device=cpu)
dist.broadcast(gradient, 0)
total_communication_rounds += 1
gradient = gradient.to(device)
gradient_norm = torch.norm(gradient)
if gradient_norm <= gradient_norm_tolerance:
break
local_solution = worker.get_InexactDANE_subproblem_solution(
model, gradient, subproblem_step_size, eta, mu, tau,
AIDE_y).cpu()
dist.reduce(local_solution, 0)
total_communication_rounds += 1
if rank == 0:
new_weights = torch.zeros(dimension, 1, device=cpu)
dist.reduce(new_weights, 0)
total_communication_rounds += 1
new_weights = new_weights.to(device)
new_weights *= 1.0 / num_workers
iteration_time = time() - iteration_start_time
#------------------------------ Printing ------------------------------
# This time is not recorded.
if rank == 0:
loss_list.append(loss)
gradient_norm_list.append(gradient_norm)
# Recall that the driver stores the test dataset in its worker.
test_accuracy = worker.get_accuracy(model)
test_accuracy_list.append(test_accuracy)
cumulative_communication_rounds_list.append(
total_communication_rounds)
cumulative_time_list.append(cumulative_time_list[-1] +
iteration_time)
print_row(iteration, total_communication_rounds, iteration_time,
loss, gradient_norm, test_accuracy)
iteration += 1
# Print final row.
if (iteration == max_iterations
or total_communication_rounds >= max_communication_rounds):
# Need to first get objective value and gradient norm on final model.
if rank == 0:
model = replace_model_weights(model, new_weights)
dist.broadcast(new_weights.cpu(), 0)
if rank > 0:
new_weights = torch.zeros(dimension, 1, device=cpu)
dist.broadcast(new_weights, 0)
new_weights = new_weights.to(device)
model = replace_model_weights(model, new_weights)
local_loss = worker.get_local_loss(model).cpu()
local_gradient = worker.get_local_gradient(model).cpu()
local_loss_and_gradient = torch.cat(
[local_loss.reshape(1, 1), local_gradient], dim=0)
dist.reduce(local_loss_and_gradient, 0)
if rank == 0:
loss_and_gradient = torch.zeros(1 + dimension, 1, device=cpu)
dist.reduce(loss_and_gradient, 0)
loss_and_gradient = loss_and_gradient.to(device)
loss_and_gradient *= 1.0 / num_workers
loss = loss_and_gradient[0, 0]
gradient = loss_and_gradient[1:]
gradient_norm = torch.norm(gradient)
if rank == 0:
loss_list.append(loss)
gradient_norm_list.append(gradient_norm)
test_accuracy = worker.get_accuracy(model)
test_accuracy_list.append(test_accuracy)
print_row(iteration=iteration,
objective_value=loss,
gradient_norm=gradient_norm,
test_accuracy=test_accuracy,
is_final_row=True)
print("\n{} after {:.2f} seconds\n".format(end_message,
cumulative_time_list[-1]))
plot_results(loss_list,
gradient_norm_list,
test_accuracy_list,
cumulative_communication_rounds_list,
label="AIDE",
max_x=max_communication_rounds)
def multi_eval(algos,LOGS_DIR, num_eval=5, num_workers=12):
"""
evaluating 12 jobs with 1 workers took: 415.78 seconds
evaluating 12 jobs with 3 workers took: 154.78 seconds
evaluating 12 jobs with 6 workers took: 91.88 seconds
evaluating 12 jobs with 12 workers took: 70.68 seconds
on gunther one gets cuda out of mem error with num_workers>12
"""
task = PlatoScoreTask(LOGS_DIR = LOGS_DIR)
jobs = [
Experiment(
job_id=get_id(),
name=build_name(algo, error_sim, two_slots),
config=build_config(algo, error_sim=error_sim, two_slots=two_slots),
train_dialogues=td,
eval_dialogues=1000,
num_warmup_dialogues=warmupd
)
for _ in range(num_eval)
for error_sim in [False,True]
for two_slots in [False,True]
for td in [40000]
for warmupd in [4000]
for algo in algos
]
start = time()
outfile = LOGS_DIR+"/results.jsonl"
mode = "wb"
if os.path.isdir(LOGS_DIR):
results = list(data_io.read_jsonl(outfile))
done_ids = [e['job_id'] for e in results]
jobs = [e for e in jobs if e.job_id not in done_ids]
print('only got %d jobs to do'%len(jobs))
print([e.job_id for e in jobs])
mode = "ab"
else:
os.makedirs(LOGS_DIR)
if num_workers > 0:
num_workers = min(len(jobs),num_workers)
with WorkerPool(processes=num_workers, task=task, daemons=False) as p:
processed_jobs = p.process_unordered(jobs)
data_io.write_jsonl(outfile, processed_jobs, mode=mode)
else:
with task as t:
processed_jobs = [t(job) for job in jobs]
data_io.write_jsonl(outfile, processed_jobs, mode=mode)
scoring_runs = list(data_io.read_jsonl(outfile))
plot_results(scoring_runs,LOGS_DIR)
print(
"evaluating %d jobs with %d workers took: %0.2f seconds"
% (len(jobs), num_workers, time() - start)
)
run_data['kernel_executions'])))
buffer_reads_hhal[idx].append(
list(
map(
lambda x: {
'size': x['size'],
'duration': x['duration']
}, run_data['buffer_reads'])))
buffer_writes_hhal[idx].append(
list(
map(
lambda x: {
'size': x['size'],
'duration': x['duration']
}, run_data['buffer_writes'])))
if mango:
return exp_sizes, total_durations, buffer_reads, buffer_writes, kernel_execs, resource_allocations, buffer_reads_hhal, buffer_writes_hhal, kernel_execs_hhal
else:
return exp_sizes, total_durations, buffer_reads, buffer_writes, kernel_execs
return exp_sizes, total_durations, buffer_reads, buffer_writes, kernel_execs
plot_results(exp_nvidia_dir,
exp_mango_dir,
exp_opencl_dir,
get_data,
'hotspot',
buffer_write_unit='megabytes',
buffer_read_unit='megabytes')
def train_model():
batch_size = 64
nb_epoch = 300
depth = 40
nb_layers = 12
nb_dense_block = 3
nb_filter = 16
growth = 12
dropout_rate = 0.2
learning_rate = 0.1
weight_decay = 1e-4
nb_classes = 100
rl_up_rate = 0.1
rl_epoch = 5 # number of interval epoches between 2 reversely learning session
rl_loop_epoch = 1 # number of loops in each reversely learning session
dim_fc1 = 448 # the dimension of the bottleneck layer of densenet-40
## BP-based learning setting
print('Building the model...\n')
model = densenet_model(nb_classes, nb_dense_block, nb_layers, growth, dropout=dropout_rate)
model.summary()
sgd = SGD(lr=learning_rate, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
print('Model compiled.')
## Reversely learning setting
with tf.device('/gpu:0'):
with tf.name_scope('config_para'):
omega = 2e-1
weights_elem_keep_rate = 0.5
rl_ur_ph = tf.placeholder(tf.float32, name='rl_ur_ph')
with tf.name_scope('forward_pass_rl'):
y_ = tf.placeholder(tf.float32, [None, nb_classes], name='y_ph')
w_fc12_ph = tf.placeholder(tf.float32, [dim_fc1, nb_classes], name='w_fc12_ph')
fc1_ph = tf.placeholder(tf.float32, [None, dim_fc1], name='fc1_ph')
fc2_ts = tf.nn.relu(tf.matmul(fc1_ph, w_fc12_ph), name='fc2_ts')
with tf.name_scope('accuracy_forward_rl'):
correct_prediction_forward_rl = tf.equal(tf.argmax(fc2_ts, 1), tf.argmax(y_, 1))
correct_prediction_forward_rl = tf.cast(correct_prediction_forward_rl, tf.float32)
accuracy_forward_rl = tf.reduce_mean(correct_prediction_forward_rl)
with tf.name_scope('reverse_learning_rl'):
# fc3 reverse learning
w_fc12_rl_tmp_ts = tf.matmul(pinv_func2(fc1_ph), y_,
name='w_fc12_rl_diff_ts') # use fc1_ts and y_ to compute the w_fc12
w_fc12_rl_dif_dp_ts = rand_weight_drop(w_fc12_rl_tmp_ts, shape=[dim_fc1, nb_classes],
keep_rate=weights_elem_keep_rate)
w_fc12_rl_ts = tf.add(w_fc12_ph, w_fc12_rl_dif_dp_ts * rl_ur_ph, name='w_fc12_rl_ts')
## Loading data
print('Loading data...')
(x_train, y_train), (x_test, y_test) = cifar100.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train = preprocess_data(x_train)
x_test = preprocess_data(x_test)
y_train = keras.utils.to_categorical(y_train, nb_classes)
y_test = keras.utils.to_categorical(y_test, nb_classes)
print('Data loaded.\n\n')
## Callbacks and history
model_checkpoint = ModelCheckpoint(weights_file, monitor="val_acc", save_best_only=True,
save_weights_only=True, verbose=1)
callbacks = [model_checkpoint]
# hist holder
#
list_bp_train_loss = []
list_bp_test_loss = []
list_bp_train_acc = []
list_bp_test_acc = []
#
list_w12_test_acc = []
#
hist_final = {}
#########################
# Main loop #
#########################
for epoch in range(1, nb_epoch + 1):
#########################
# learning rate control #
#########################
if (epoch - 1) == int(0.5 * nb_epoch):
learning_rate = 0.01
K.set_value(model.optimizer.lr, learning_rate)
print('Epoch %d: Set learning rate to %s, and model is compiled' % (epoch, learning_rate))
rl_up_rate = rl_up_rate / 10.
if (epoch - 1) == int(0.75 * nb_epoch):
learning_rate = 0.001
K.set_value(model.optimizer.lr, learning_rate)
print('Epoch %d: Set learning rate to %s, and model is compiled' % (epoch, learning_rate))
rl_up_rate = rl_up_rate / 10.
print('Epoch %d: BP learning rate: %s; Reversely learning update rate: %s. ' % (
epoch, K.get_value(model.optimizer.lr), rl_up_rate))
hist = model.fit(x_train, y_train, batch_size=batch_size,
epochs=epoch, verbose=2,
validation_data=(x_test, y_test),
callbacks=callbacks, shuffle=True, initial_epoch=epoch-1)
# eval_loss, eval_acc = model.evaluate(x_test, y_test, verbose=0, batch_size=32)
# print('Epoch %d: Evaluate acc before reverse learning: %s.\n ' % (epoch, eval_acc))
########################
# hist and plot #
########################
list_bp_train_loss.append(hist.history['loss'])
list_bp_test_loss.append(hist.history['val_loss'])
list_bp_train_acc.append(hist.history['acc'])
list_bp_test_acc.append(hist.history['val_acc'])
############################
# REVERSE LEARNING #
############################
if ( epoch-1 ) % rl_epoch ==0:
print('Epoch %d: Begin reversely learning...' % epoch)
model_ft = Model(model.input, outputs=model.get_layer(name='fc1').output)
fc1_tr_np = model_ft.predict(x_train)
fc1_te_np = model_ft.predict(x_test)
for idx_RLloop in range(rl_loop_epoch):
# print('Epoch %d, idx_RLloop %d...' % (epoch, idx_RLloop))
w_fc12_np = model.get_layer(name='fc2').get_weights()[0]
## Execute reversely learning
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
start_time = time.time()
w_fc12_rl_np = w_fc12_rl_ts.eval(
feed_dict={
fc1_ph: fc1_tr_np,
y_: y_train,
w_fc12_ph: w_fc12_np,
rl_ur_ph: rl_up_rate
})
acc_tf = accuracy_forward_rl.eval(
feed_dict={
fc1_ph: fc1_te_np,
y_: y_test,
w_fc12_ph: w_fc12_rl_np
})
print('Epoch {}, idx_RLloop {} Reversely learning: acc with updated weight: {}, time duration: {}.\n'.format(epoch, idx_RLloop, acc_tf, time.time() - start_time))
if idx_RLloop == (rl_loop_epoch - 1):
list_w12_test_acc.append([acc_tf.item()])
#### update the reversely learnt weight to the model
w_fc12_list = [None]
w_fc12_list[0] = w_fc12_np
model.get_layer(name='fc2').set_weights(w_fc12_list)
# write the hist
hist_final['loss'] = list_bp_train_loss
hist_final['val_loss'] = list_bp_test_loss
hist_final['acc'] = list_bp_train_acc
hist_final['val_acc'] = list_bp_test_acc
hist_final['w12_acc'] = list_w12_test_acc
with open(hist_json_file, 'w') as file_json:
json.dump(hist_final, file_json, indent=4, sort_keys=True)
with open(hist_pi_file, 'wb') as file_pi:
pickle.dump(hist_final, file_pi)
# after the all the loop, plot the
plot_results(hist_json_file, save_dir=HIST_DIR, mode='json')
return
def select_the_best_delta_using_the_best_strategy_HMM(k=10, shuffle=True):
'''
'''
address_to_read = r"E:\pgmpy\Seq of sensor events_based_on_activity_and_no_overlap_delta\delta_{delta}min.csv"
deltas = [
15, 30, 45, 60, 75, 90, 100, 120, 150, 180, 200, 240, 300, 400, 500,
600, 700, 800, 900, 1000
]
best_score = 0
best_delta = 0
best_train_set = 0
best_test_set = 0
best_train_set_person_labels = 0
best_test_set_person_labels = 0
list_of_deltas = []
list_of_f1_micros = []
for d in deltas:
print("delta:", d)
list_of_data, list_of_persons, _ = read_sequence_based_CSV_file_with_activity(
file_address=address_to_read.format(delta=d),
has_header=True,
separate_data_based_on_persons=True)
number_of_persons = len(list_of_data)
train_set = np.zeros(number_of_persons, dtype=np.ndarray)
test_set = np.zeros(number_of_persons, dtype=np.ndarray)
train_set_person_labels = np.zeros(number_of_persons, dtype=np.ndarray)
test_set_person_labels = np.zeros(number_of_persons, dtype=np.ndarray)
k_splitted_train_set = np.ndarray(shape=(2, k), dtype=np.ndarray)
k_splitted_train_set_person_labels = np.ndarray(shape=(2, k),
dtype=np.ndarray)
for per in range(number_of_persons):
if shuffle:
list_of_data[per], list_of_persons[
per] = unison_shuffled_copies(list_of_data[per],
list_of_persons[per])
# repeat person tags
#is_one_person = true because i just send the data of one person in each iteration
list_of_persons[per] = repaet_person_tags_as_much_as_seq_length(
list_of_data[per], list_of_persons[per], is_one_person=True)
number_of_train_set_data = int(0.8 * len(list_of_data[per]))
train_set[per] = list_of_data[per][0:number_of_train_set_data]
train_set_person_labels[per] = list_of_persons[per][
0:number_of_train_set_data]
test_set[per] = list_of_data[per][number_of_train_set_data:]
test_set_person_labels[per] = list_of_persons[per][
number_of_train_set_data:]
#split both train_set and test_set to k=10 groups
print("len(train_set[per]):", len(train_set[per]),
"number_of_train_set_data:", number_of_train_set_data)
number_of_each_group_of_data = int(len(train_set[per]) / k)
start = 0
for i in range(k - 1):
end = (i + 1) * number_of_each_group_of_data
k_splitted_train_set[per][i] = train_set[per][start:end]
k_splitted_train_set_person_labels[per][
i] = train_set_person_labels[per][start:end]
start = end
k_splitted_train_set[per][k - 1] = train_set[per][start:]
k_splitted_train_set_person_labels[per][
k - 1] = train_set_person_labels[per][start:]
train_set_k = np.zeros(number_of_persons, dtype=np.ndarray)
train_set_labels_k = np.zeros(number_of_persons, dtype=np.ndarray)
test_set_k = np.zeros(number_of_persons, dtype=np.ndarray)
test_set_labels_k = np.zeros(number_of_persons, dtype=np.ndarray)
scores = np.zeros(k, dtype=dict)
for i in range(k):
for per in range(number_of_persons):
train_set_k[per], train_set_labels_k[per], test_set_k[
per], test_set_labels_k[
per] = prepare_train_and_test_based_on_a_list_of_k_data(
k_splitted_train_set[per],
k_splitted_train_set_person_labels[per], i)
scores[
i] = create_casas7_HMM_with_prepared_train_and_test_based_on_seq_of_activities(
train_set=train_set_k,
list_of_persons_in_train=train_set_labels_k,
test_set=test_set_k,
list_of_persons_in_test=test_set_labels_k)
print("**************************\nscores:", scores)
scores_avg = calculate_f1_scoreaverage(scores, k)
print("scores_avg", scores_avg)
list_of_deltas.append(d)
list_of_f1_micros.append(scores_avg)
if scores_avg > best_score:
best_score = scores_avg
best_delta = d
best_train_set = train_set
best_test_set = test_set
best_train_set_person_labels = train_set_person_labels
best_test_set_person_labels = test_set_person_labels
print("Validation Scores:")
print("best_delta:", best_delta, "best_validation_score:", best_score)
#test_score = create_casas7_HMM_with_prepared_train_and_test_based_on_seq_of_activities(train_set = best_train_set , list_of_persons_in_train= best_train_set_person_labels, test_set= best_test_set , list_of_persons_in_test= best_test_set_person_labels)
#print("test_score:" , test_score)
plot_results(list_of_deltas,
list_of_f1_micros,
'Seq of events based on activities and no overlap delta',
y_label="f1 score micro")
plotname = "Eugene, OR, surface slope: " + str(
slope_deg) + orient + "Temperature dependency"
labels = ('Ra', 'Rso')
colors = ('r--', 'r', 'b--', 'b', 'g--', 'g', 'k--', 'k')
labloc = ("upper left", "lower center", "upper center")
fig1, ax1 = plt.subplots(figsize=(7, 5))
temperature_array = [-40.0, -30.0, -10.0, 0.0, 10.0, 20.0, 30.0, 40.0]
i = 0
for temperature in temperature_array:
result = run_radiation.run_radiation(latitude_deg, slope_deg, aspect_deg,
elevation, albedo, turbidity,
temperature, rhumidity, '1-hour')
plot_results.plot_results(result[0], result[2], fig1,
ax1, ymax, xname, yname, plotname,
str(temperature), colors[i], labloc[1])
i += 1
plt.show()
ymax = 120
ymin = -10
yname = 'NetLW, [W/m^2]'
xname = 'DOY'
i = 0
fig2, ax2 = plt.subplots(figsize=(7, 5))
for temperature in temperature_array:
result = run_radiation.run_radiation(latitude_deg, slope_deg, aspect_deg,
elevation, albedo, turbidity,
temperature, rhumidity, '3-hour',
'asce-ewri')
def DiSCO(model, worker, device, max_iterations=100,
max_communication_rounds=200, gradient_norm_tolerance=1e-8,
subproblem_tolerance=1e-4, subproblem_maximum_iterations=50):
"""Run the DiSCO algorithm.
Arguments:
model (torch.nn.Module): A model to optimize.
worker (Worker): A Worker class instance. The worker on the driver is
used to record test accuracy.
device (torch.device): The device tensors will be allocated to.
max_iterations (int, optional): The maximum number of iterations.
max_communication_rounds (int, optional): The maximum number of
communication rounds.
gradient_norm_tolerance (float, optional): The smallest the norm of the
full gradient can be before the algorithm stops.
subproblem_tolerance (float, optional): The tolerance used by the
subproblem solvers.
subproblem_maximum_iterations (int, optional): The maximum number of
iterations used by the subproblem solver.
"""
rank = dist.get_rank()
dimension = sum([p.numel() for p in model.parameters()])
num_workers = dist.get_world_size()-1
cpu = torch.device("cpu")
if rank == 0:
print("\n{:-^86s}\n".format(" DiSCO "))
# Results are added to these lists and then plotted.
cumulative_communication_rounds_list = [0]
cumulative_time_list = [0]
gradient_norm_list = []
loss_list = []
test_accuracy_list = []
# We will store a message about why the algorithm stopped.
end_message = "max_iterations reached"
iteration = 0
total_communication_rounds = 0
subproblem_failed = False
while iteration < max_iterations:
if total_communication_rounds >= max_communication_rounds:
end_message = 'max_communication_rounds reached'
break
#-------------------------- DiSCO iteration ---------------------------
if rank == 0:
# The driver will record how long each iteration takes.
iteration_start_time = time()
if iteration > 0:
# This iteration requires an initial update to workers' model.
if rank == 0:
# direction is from previous iteration.
dist.broadcast(direction.cpu(), 0)
total_communication_rounds += 1
if rank > 0:
direction = torch.zeros(dimension, 1, device=cpu)
dist.broadcast(direction, 0)
total_communication_rounds += 1
direction = direction.to(device)
model = get_updated_model(model, direction)
if rank > 0:
local_loss = worker.get_local_loss(model).cpu()
local_gradient = worker.get_local_gradient(model).cpu()
local_loss_and_gradient = torch.cat([local_loss.reshape(1,1),
local_gradient], dim=0)
# All workers send local objective value and gradient to driver.
dist.reduce(local_loss_and_gradient, 0)
total_communication_rounds += 1
if rank == 0:
loss_and_gradient = torch.zeros(1+dimension, 1, device=cpu)
dist.reduce(loss_and_gradient, 0)
total_communication_rounds += 1
loss_and_gradient = loss_and_gradient.to(device)
loss_and_gradient *= 1.0/num_workers
loss = loss_and_gradient[0,0]
gradient = loss_and_gradient[1:]
dist.broadcast(gradient.cpu(), 0)
total_communication_rounds += 1
gradient_norm = torch.norm(gradient)
if gradient_norm <= gradient_norm_tolerance:
end_message = 'gradient_norm_tolerance reached'
break
if rank > 0:
gradient = torch.zeros(dimension, 1, device=cpu)
dist.broadcast(gradient, 0)
total_communication_rounds += 1
gradient = gradient.to(device)
gradient_norm = torch.norm(gradient)
if gradient_norm <= gradient_norm_tolerance:
break
PCG_result = distributed_PCG_algorithm(model, worker, device, gradient,
subproblem_tolerance, subproblem_maximum_iterations)
if rank > 0:
if PCG_result is None: # PCG failed
break
total_communication_rounds += PCG_result
if rank == 0:
if PCG_result is None: # PCG failed
end_message = 'PCG failed'
subproblem_failed = True
break
v, delta, PCG_communication_rounds = PCG_result
total_communication_rounds += PCG_communication_rounds
direction = (-1.0 / (1 + delta)) * v
new_model = get_updated_model(model, direction)
iteration_time = time() - iteration_start_time
#------------------------------ Printing ------------------------------
# This time is not recorded.
if rank == 0:
loss_list.append(loss)
gradient_norm_list.append(gradient_norm)
# Recall that the driver stores the test dataset in its worker.
test_accuracy = worker.get_accuracy(model)
test_accuracy_list.append(test_accuracy)
cumulative_communication_rounds_list.append(
total_communication_rounds)
cumulative_time_list.append(
cumulative_time_list[-1] + iteration_time)
print_row(iteration, total_communication_rounds, iteration_time,
torch.norm(direction), loss, gradient_norm,
test_accuracy)
model = new_model
iteration += 1
# Print final row.
if (iteration == max_iterations
or total_communication_rounds >= max_communication_rounds):
# Need to first get objective value and gradient norm on final model.
if rank == 0:
dist.broadcast(direction.cpu(), 0)
if rank > 0:
direction = torch.zeros(dimension, 1, device=cpu)
dist.broadcast(direction, 0)
direction = direction.to(device)
model = get_updated_model(model, direction)
local_loss = worker.get_local_loss(model).cpu()
local_gradient = worker.get_local_gradient(model).cpu()
local_loss_and_gradient = torch.cat([local_loss.reshape(1,1),
local_gradient], dim=0)
dist.reduce(local_loss_and_gradient, 0)
if rank == 0:
loss_and_gradient = torch.zeros(1+dimension, 1, device=cpu)
dist.reduce(loss_and_gradient, 0)
loss_and_gradient = loss_and_gradient.to(device)
loss_and_gradient *= 1.0/num_workers
loss = loss_and_gradient[0,0]
gradient = loss_and_gradient[1:]
gradient_norm = torch.norm(gradient)
if rank == 0:
loss_list.append(loss)
gradient_norm_list.append(gradient_norm)
test_accuracy = worker.get_accuracy(model)
test_accuracy_list.append(test_accuracy)
print_row(iteration=iteration, objective_value=loss,
gradient_norm=gradient_norm, test_accuracy=test_accuracy,
is_final_row=True)
print("\n{} after {:.2f} seconds\n".format(end_message,
cumulative_time_list[-1]))
plot_results(loss_list, gradient_norm_list, test_accuracy_list,
cumulative_communication_rounds_list, label="DiSCO",
max_x=max_communication_rounds, failed=subproblem_failed)
for i in range(200):
x_train, y_train = next(train_generator)
#x_train = np.expand_dims(x_train[...,0],axis=-1)
class_weights = class_weight.compute_class_weight('balanced', np.unique(np.argmax(y_train,axis=-1)),np.argmax(y_train,axis=-1))
#cw = (class_weights,[])
print('num labels is ',y_train.shape)
print('the mean is ',np.mean(x_train))
print('the number of nans is ',np.count_nonzero(np.isnan(x_train)))
x_test, y_test = next(validation_generator)
#x_test = np.expand_dims(x_test[...,0],axis=-1)
z_dummy = np.zeros((x_train.shape[0],512,2))
z_dummy_test = np.zeros((x_test.shape[0],512,2))
print('the learning rate is ',K.eval(encoder_decoder_model.optimizer.lr))
reduce_lr = tf.keras.callbacks.LearningRateScheduler(schedule_lr(200,K.eval(encoder_decoder_model.optimizer.lr)))
encoder_decoder_model.fit(x_train, [x_train,z_dummy], epochs=10, batch_size=batch_size, validation_data=[x_test,[x_test,z_dummy_test]],shuffle=True, callbacks = [reduce_lr])
#encoder_decoder_model.fit_generator(train_generator,steps_per_epoch=2000,epochs=20,validation_data=validation_generator,validation_steps=800,verbose=1)
encoder_decoder_model.save_weights('vae_chest.h5')
decoder_model.save_weights('vae_decoder_chest.h5')
encoder_model.save_weights('vae_encoder_chest.h5')
plot_results((encoder_model,decoder_model),(x_test, np.argmax(y_test,axis=-1), x_train, np.argmax(y_train,axis=-1)),batch_size=128,model_name="vae_mlp",epoch=i)