def train(
main_config,
model_config,
model_name,
experiment_name,
dataset_name,
):
main_cfg = MainConfig(main_config)
model = MODELS[model_name]
dataset = dataset_type.get_dataset(dataset_name)
train_data = dataset.train_set_pairs()
vectorizer = DatasetVectorizer(main_cfg.model_dir,
raw_sentence_pairs=train_data)
dataset_helper = Dataset(vectorizer, dataset, main_cfg.batch_size)
max_sentence_len = vectorizer.max_sentence_len
vocabulary_size = vectorizer.vocabulary_size
train_mini_sen1, train_mini_sen2, train_mini_labels = dataset_helper.pick_train_mini_batch(
)
train_mini_labels = train_mini_labels.reshape(-1, 1)
test_sentence1, test_sentence2 = dataset_helper.test_instances()
test_labels = dataset_helper.test_labels()
test_labels = test_labels.reshape(-1, 1)
num_batches = dataset_helper.num_batches
model = model(
max_sentence_len,
vocabulary_size,
main_config,
model_config,
)
model_saver = ModelSaver(
main_cfg.model_dir,
experiment_name,
main_cfg.checkpoints_to_keep,
)
config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=main_cfg.log_device_placement,
)
with tf.Session(config=config) as session:
global_step = 0
init = tf.global_variables_initializer()
session.run(init)
log_saver = LogSaver(
main_cfg.logs_path,
experiment_name,
dataset_name,
session.graph,
)
model_evaluator = ModelEvaluator(model, session)
metrics = {'acc': 0.0}
time_per_epoch = []
log('Training model for {} epochs'.format(main_cfg.num_epochs))
for epoch in tqdm(range(main_cfg.num_epochs), desc='Epochs'):
start_time = time.time()
train_sentence1, train_sentence2 = dataset_helper.train_instances(
shuffle=True)
train_labels = dataset_helper.train_labels()
train_batch_helper = BatchHelper(
train_sentence1,
train_sentence2,
train_labels,
main_cfg.batch_size,
)
# small eval set for measuring dev accuracy
dev_sentence1, dev_sentence2, dev_labels = dataset_helper.dev_instances(
)
dev_labels = dev_labels.reshape(-1, 1)
tqdm_iter = tqdm(range(num_batches),
total=num_batches,
desc="Batches",
leave=False,
postfix=metrics)
for batch in tqdm_iter:
global_step += 1
sentence1_batch, sentence2_batch, labels_batch = train_batch_helper.next(
batch)
feed_dict_train = {
model.x1: sentence1_batch,
model.x2: sentence2_batch,
model.is_training: True,
model.labels: labels_batch,
}
loss, _ = session.run([model.loss, model.opt],
feed_dict=feed_dict_train)
if batch % main_cfg.eval_every == 0:
feed_dict_train = {
model.x1: train_mini_sen1,
model.x2: train_mini_sen2,
model.is_training: False,
model.labels: train_mini_labels,
}
train_accuracy, train_summary = session.run(
[model.accuracy, model.summary_op],
feed_dict=feed_dict_train,
)
log_saver.log_train(train_summary, global_step)
feed_dict_dev = {
model.x1: dev_sentence1,
model.x2: dev_sentence2,
model.is_training: False,
model.labels: dev_labels
}
dev_accuracy, dev_summary = session.run(
[model.accuracy, model.summary_op],
feed_dict=feed_dict_dev,
)
log_saver.log_dev(dev_summary, global_step)
tqdm_iter.set_postfix(
dev_acc='{:.2f}'.format(float(dev_accuracy)),
train_acc='{:.2f}'.format(float(train_accuracy)),
loss='{:.2f}'.format(float(loss)),
epoch=epoch)
if global_step % main_cfg.save_every == 0:
model_saver.save(session, global_step=global_step)
model_evaluator.evaluate_dev(dev_sentence1, dev_sentence2,
dev_labels)
end_time = time.time()
total_time = timer(start_time, end_time)
time_per_epoch.append(total_time)
model_saver.save(session, global_step=global_step)
model_evaluator.evaluate_test(test_sentence1, test_sentence2,
test_labels)
model_evaluator.save_evaluation(
'{}/{}'.format(main_cfg.model_dir, experiment_name),
time_per_epoch[-1], dataset)
def __init__(self, num_clients, num_servers, iterations, method,
simulation_output_view):
"""
Offline stage of simulation. Initializes clients and servers for iteration as well as gives each client its data.
:param num_clients: number of clients to be use for simulation
:param num_servers: number of servers to be use for simulation. Personalized coding required if greater than 1.
:param iterations: number of iterations to run simulation for
"""
global len_per_iteration
digits = load_digits() # using sklearn's MNIST dataset
X, y = digits.data, digits.target
scaler = MinMaxScaler()
scaler.fit(X)
X = scaler.transform(X)
X_train, X_test = X[:-config.LEN_TEST], X[-config.LEN_TEST:]
y_train, y_test = y[:-config.LEN_TEST], y[-config.LEN_TEST:]
# extract only amount that we require
number_of_samples = 0
for client_name in config.client_names:
len_per_iteration = config.LENS_PER_ITERATION[client_name]
number_of_samples += len_per_iteration * iterations
X_train = X_train[:number_of_samples]
y_train = y_train[:number_of_samples]
client_to_datasets = data_formatting.partition_data(
X_train,
y_train,
config.client_names,
iterations,
config.LENS_PER_ITERATION,
cumulative=config.USING_CUMULATIVE)
# print_config(len_per_iteration=config.LEN_PER_ITERATION)
print('\n \n \nSTARTING SIMULATION \n \n \n')
simulation_output_view.appendPlainText(
'\n \n \n========== FEDERATED LEARNING v2.0 ========')
simulation_output_view.appendPlainText(
'========== STARTING SIMULATION =========== \n \n \n')
print('num_clients:', num_clients)
active_clients = {'client_agent' + str(i) for i in range(num_clients)}
self.clients = {
'client_agent' + str(i): ClientAgent(
agent_number=i,
train_datasets=client_to_datasets['client_agent' + str(i)],
evaluator=ModelEvaluator(X_test, y_test, method),
active_clients=active_clients,
simulation_output_view=simulation_output_view)
for i in range(num_clients)
}
self.server_agents = {
'server_agent' + str(i):
ServerAgent(agent_number=i,
simulation_output_view=simulation_output_view)
for i in range(num_servers)
} # initialize servers
# create directory with mappings from names to instances
self.directory = Directory(clients=self.clients,
server_agents=self.server_agents)
for agent_name, agent in self.clients.items():
agent.set_directory(self.directory)
agent.initializations()
for agent_name, agent in self.server_agents.items():
agent.set_directory(self.directory)
# OFFLINE diffie-helman key exchange
# NOTE: this is sequential in implementation, but simulated as occuring parallel
if config.USE_SECURITY:
key_exchange_start = datetime.datetime.now(
) # measuring how long the python script takes
max_latencies = []
for client_name, client in self.clients.items():
# not including logic of sending/receiving public keys in latency computation since it is nearly zero
client.send_pubkeys()
max_latency = max(config.LATENCY_DICT[client_name].values())
max_latencies.append(max_latency)
simulated_time = max(max_latencies)
key_exchange_end = datetime.datetime.now() # measuring runtime
key_exchange_duration = key_exchange_end - key_exchange_start
simulated_time += key_exchange_duration
if config.SIMULATE_LATENCIES:
print(
'Diffie-helman key exchange simulated duration: {}\nDiffie-helman key exchange real run-time: {}\n'
.format(simulated_time, key_exchange_duration))
simulation_output_view. \
appendPlainText('Diffie-helman key exchange simulated duration: {}\nDiffie-helman key '
'exchange real run-time: {}\n'.format(simulated_time, key_exchange_duration))
for client_name, client in self.clients.items():
client.initialize_common_keys()
def __init__(self, num_clients, iterations):
"""
Offline stage of simulation. Initializes clients and servers for iteration as well as gives each client its data.
:param num_clients: number of clients to be use for simulation
:param num_servers: number of servers to be use for simulation. Personalized coding required if greater than 1.
:param iterations: number of iterations to run simulation for
"""
global len_per_iteration
if config.USING_PYSPARK:
spark = SparkSession.builder.appName(
'SecureFederatedLearning').getOrCreate(
) # initialize spark session
spark.sparkContext.setLogLevel("ERROR") # supress sparks messages
digits = load_digits() # using sklearn's MNIST dataset
X, y = digits.data, digits.target
X_train, X_test = X[:-config.LEN_TEST], X[-config.LEN_TEST:]
y_train, y_test = y[:-config.LEN_TEST], y[-config.LEN_TEST:]
# extract only amount that we require
number_of_samples = 0
for client_name in config.client_names:
len_per_iteration = config.LENS_PER_ITERATION[client_name]
number_of_samples += len_per_iteration * iterations
X_train = X_train[:number_of_samples]
y_train = y_train[:number_of_samples]
client_to_datasets = data_formatting.partition_data(
X_train,
y_train,
config.client_names,
iterations,
config.LENS_PER_ITERATION,
cumulative=config.USING_CUMULATIVE,
pyspark=config.USING_PYSPARK)
#print_config(len_per_iteration=config.LEN_PER_ITERATION)
print('\n \n \nSTARTING SIMULATION \n \n \n')
active_clients = {'client_agent' + str(i) for i in range(num_clients)}
self.clients = {
'client_agent' + str(i): ClientAgentServerless(
agent_number=i,
train_datasets=client_to_datasets['client_agent' + str(i)],
evaluator=ModelEvaluator(X_test, y_test),
active_clients=active_clients)
for i in range(num_clients)
} # initialize the agents
# create directory with mappings from names to instances
self.directory = Directory(clients=self.clients)
for agent_name, agent in self.clients.items():
agent.set_directory(self.directory)
agent.initializations()
# OFFLINE diffie-helman key exchange
# NOTE: this is sequential in implementation, but simulated as occuring parallel
if config.USE_SECURITY:
key_exchange_start = datetime.datetime.now(
) # measuring how long the python script takes
max_latencies = []
for client_name, client in self.clients.items():
# not including logic of sending/receiving public keys in latency computation since it is nearly zero
client.send_pubkeys()
max_latency = max(config.LATENCY_DICT[client_name].values())
max_latencies.append(max_latency)
simulated_time = max(max_latencies)
key_exchange_end = datetime.datetime.now() # measuring runtime
key_exchange_duration = key_exchange_end - key_exchange_start
simulated_time += key_exchange_duration
if config.SIMULATE_LATENCIES:
print(
'Diffie-helman key exchange simulated duration: {}\nDiffie-helman key exchange real run-time: {}\n'
.format(simulated_time, key_exchange_duration))
for client_name, client in self.clients.items():
client.initialize_common_keys()
def train(main_config, model_config, model_name, dataset_name):
main_cfg = MainConfig(main_config)
model = MODELS[model_name]
dataset = DATASETS[dataset_name]()
model_name = '{}_{}'.format(model_name,
main_config['PARAMS']['embedding_size'])
train_data = dataset.train_set_pairs()
vectorizer = DatasetVectorizer(train_data, main_cfg.model_dir)
dataset_helper = Dataset(vectorizer, dataset, main_cfg.batch_size)
max_sentence_len = vectorizer.max_sentence_len
vocabulary_size = vectorizer.vocabulary_size
train_mini_sen1, train_mini_sen2, train_mini_labels = dataset_helper.pick_train_mini_batch(
)
train_mini_labels = train_mini_labels.reshape(-1, 1)
test_sentence1, test_sentence2 = dataset_helper.test_instances()
test_labels = dataset_helper.test_labels()
test_labels = test_labels.reshape(-1, 1)
num_batches = dataset_helper.num_batches
model = model(max_sentence_len, vocabulary_size, main_config, model_config)
model_saver = ModelSaver(main_cfg.model_dir, model_name,
main_cfg.checkpoints_to_keep)
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=main_cfg.log_device_placement)
with tf.Session(config=config) as session:
global_step = 0
init = tf.global_variables_initializer()
session.run(init)
log_saver = LogSaver(main_cfg.logs_path, model_name, dataset_name,
session.graph)
model_evaluator = ModelEvaluator(model, session)
metrics = {'acc': 0.0}
time_per_epoch = []
for epoch in tqdm(range(main_cfg.num_epochs), desc='Epochs'):
start_time = time.time()
train_sentence1, train_sentence2 = dataset_helper.train_instances(
shuffle=True)
train_labels = dataset_helper.train_labels()
train_batch_helper = BatchHelper(train_sentence1, train_sentence2,
train_labels, main_cfg.batch_size)
# small eval set for measuring dev accuracy
dev_sentence1, dev_sentence2, dev_labels = dataset_helper.dev_instances(
)
dev_labels = dev_labels.reshape(-1, 1)
tqdm_iter = tqdm(range(num_batches),
total=num_batches,
desc="Batches",
leave=False,
postfix=metrics)
for batch in tqdm_iter:
global_step += 1
sentence1_batch, sentence2_batch, labels_batch = train_batch_helper.next(
batch)
feed_dict_train = {
model.x1: sentence1_batch,
model.x2: sentence2_batch,
model.is_training: True,
model.labels: labels_batch
}
loss, _ = session.run([model.loss, model.opt],
feed_dict=feed_dict_train)
if batch % main_cfg.eval_every == 0:
feed_dict_train = {
model.x1: train_mini_sen1,
model.x2: train_mini_sen2,
model.is_training: False,
model.labels: train_mini_labels
}
train_accuracy, train_summary = session.run(
[model.accuracy, model.summary_op],
feed_dict=feed_dict_train)
log_saver.log_train(train_summary, global_step)
feed_dict_dev = {
model.x1: dev_sentence1,
model.x2: dev_sentence2,
model.is_training: False,
model.labels: dev_labels
}
dev_accuracy, dev_summary = session.run(
[model.accuracy, model.summary_op],
feed_dict=feed_dict_dev)
log_saver.log_dev(dev_summary, global_step)
tqdm_iter.set_postfix(
dev_acc='{:.2f}'.format(float(dev_accuracy)),
train_acc='{:.2f}'.format(float(train_accuracy)),
loss='{:.2f}'.format(float(loss)),
epoch=epoch)
if global_step % main_cfg.save_every == 0:
model_saver.save(session, global_step=global_step)
model_evaluator.evaluate_dev(dev_sentence1, dev_sentence2,
dev_labels)
end_time = time.time()
total_time = timer(start_time, end_time)
time_per_epoch.append(total_time)
model_saver.save(session, global_step=global_step)
feed_dict_train = {
model.x1: test_sentence1,
model.x2: test_sentence2,
model.is_training: False,
model.labels: test_labels
}
#train_accuracy, train_summary, train_e = session.run([model.accuracy, model.summary_op, model.e],
# feed_dict=feed_dict_train)
train_e = session.run([model.e], feed_dict=feed_dict_train)
plt.clf()
f = plt.figure(figsize=(8, 8.5))
ax = f.add_subplot(1, 1, 1)
i = ax.imshow(train_e[0][0], interpolation='nearest', cmap='gray')
cbaxes = f.add_axes([0.2, 0, 0.6, 0.03])
cbar = f.colorbar(i, cax=cbaxes, orientation='horizontal')
cbar.ax.set_xlabel('Probability', labelpad=2)
f.savefig('attention_maps.pdf', bbox_inches='tight')
f.show()
plt.show()
feed_dict_test = {
model.x1: test_sentence1,
model.x2: test_sentence2,
model.is_training: False,
model.labels: test_labels
}
test_accuracy, test_summary = session.run(
[model.accuracy, model.summary_op], feed_dict=feed_dict_test)
print('tst_acc:%.2f loss:%.2f', test_accuracy, loss)
model_evaluator.evaluate_test(test_sentence1, test_sentence2,
test_labels)
model_evaluator.save_evaluation(
'{}/{}'.format(main_cfg.model_dir, model_name), time_per_epoch[-1],
dataset)