def reduce_silence(path, audio_file, name_audio):
cut_times = 0
start_time = 0
audios_with_silence = []
new_audios = []
end_time = 0
silence_time = 0
score_final = 0
detect_speech(audio_file)
text_file = open("result.txt", "r")
for text_line in text_file.readlines():
text_line = text_line.rstrip()
items = text_line.split(" ")
time_mil_1_final = get_time_file(items[1])
time_mil_2_final = get_time_file(items[2])
if score_final != 0:
silence_time = time_mil_1_final - score_final
if silence_time > 1.0:
end_time = score_final
new_audio_file = split_audios_with_times(path, name_audio,
start_time, end_time,
audio_file, cut_times)
new_audios.append(new_audio_file)
audios_with_silence.append(
mixed_silence(path, name_audio, new_audio_file, cut_times))
start_time = time_mil_1_final
cut_times = cut_times + 1
score_final = time_mil_2_final
audios_with_silence.append(
split_audios_with_times(path, name_audio, start_time, 30, audio_file,
cut_times + 1))
mixed_audio(path, name_audio, audios_with_silence, audio_file)
utils.delete_files(new_audios)
utils.delete_files(audios_with_silence)
utils.delete_file("result.txt")
def create_note(text, title=None):
note = Types.Note()
note.resources = []
note.title = "Test note from EDAMTest.py" if title == None else title
files = list_folder()
# The content of an Evernote note is represented using Evernote Markup Language
# (ENML). The full ENML specification can be found in the Evernote API Overview
# at http://dev.evernote.com/documentation/cloud/chapters/ENML.php
note.content = '<?xml version="1.0" encoding="UTF-8"?>'
note.content += '<!DOCTYPE en-note SYSTEM ' \
'"http://xml.evernote.com/pub/enml2.dtd">'
note.content += '<en-note>'
note.content += f'{text}<br/>'
if len(files) > 0:
for file in files:
print(file)
note.content += add_img(note, file)
print(note.content)
note.content += '</en-note>'
# Finally, send the new note to Evernote using the createNote method
# The new Note object that is returned will contain server-generated
# attributes such as the new note's unique GUID.
created_note = note_store.createNote(note)
delete_files()
print("Successfully created a new note with GUID: ", created_note.guid)
def reset_camera(video_camera, streamer):
delete_files()
if video_camera:
video_camera.release()
video_camera = VideoCamera()
video_camera.initialize()
streamer = CameraStream(camera_source=video_camera)
return (video_camera, streamer,)
def main():
utils.delete_files()
data = Data()
if data.algorithm == Algorithm.GENETIC:
algorithm = GeneticAlgorithm(data)
else:
algorithm = CSPAlgorithm(data)
algorithm.run()
print("Finished!")
def run(seed, save_path, network, em, dataset, _rnd, debug):
ex.commands['print_config']()
# create storage directories if they don't exist yet
utils.create_directory('networks')
utils.create_directory('results')
if debug:
utils.create_directory('debug_output')
utils.delete_files('debug_output')
# seed numpy
np.random.seed(seed)
# load data
data = get_training_data()
# if debug set to true the trainer will report on the dae performance
save_img_ind = np.random.randint(data['valid_data'].shape[1])
if debug:
utils.save_image('debug_output/valid_image.jpg', data['valid_data'][0, save_img_ind])
# produce saving directory filenames
folder_name = dataset['name'] + "_" + dataset['train_set'] + "_" + time.strftime(
"%d_%m_%Y_%H:%M:%S") + "_" + str(seed)
net_folder_path = os.path.join(save_path, folder_name)
utils.create_directory(net_folder_path)
model_config_path = os.path.join(net_folder_path, "model_config.pickle")
em_dump_path = None
if em['suffix'] is not None:
em_dump_path = 'results/{}_{}_{}{}.pickle'.format(dataset['name'], em['nr_iters'], em['k'], em['suffix'])
# save net structure
with open(model_config_path, 'wb') as f:
pickle.dump({'network': network, 'em': em, 'dataset': dataset}, f)
# train dea
train_dae(data=data, net_folder_path=net_folder_path, **{'save_img_ind': save_img_ind})
# load test data
data = get_test_data()
test_data = data['test_data']
test_groups = data['test_groups'] if 'test_groups' in data.keys() else None
# perform rc on best model
return perform_reconstruction_clustering_from_file(net_folder_path, test_data, test_groups, _rnd, em_dump_path=em_dump_path)
def upload_and_delete_files_wrapper(self, service, from_dir_path, files_upload, files_delete):
'''
Args:
service: drive api service instance
from_dir_path: directory path to upload files from
files_upload: list of files to upload
files_to_delete: list of files under from_dir_path, used to purge directory after upload completes
'''
utils.log('Upload ' + str(len(files_upload)) + ' file(s)')
for f in files_upload:
self.upload_file(service, ''.join([from_dir_path, f]), 'application/octet-stream', body={'title': f, 'description': 'zip file', 'mimeType': 'application/octet-stream'})
files_delete.append(f)
print '*',
utils.log('Uploaded')
utils.delete_files(from_dir_path, files_delete)
def reduce_silence(audio_file):
cut_times = 0
start_time = 0
audios_with_silence = []
new_audios = []
end_time = 0
silence_time = 0
score_final = 0
detect_speech(audio_file)
text_file = open("result.txt", "r")
for text_line in text_file.readlines():
text_line = text_line.rstrip()
items = text_line.split(" ")
time1 = items[1].split(":")
time2 = items[2].split(":")
time_mil_1 = time1[2].split(".")
time_mil_2 = time2[2].split(".")
time_mil_1_final = 0
time_mil_2_final = 0
time_mil_1_final = float(float(time_mil_1[1])/1000)+float(time_mil_1[0])
time_mil_2_final = float(float(time_mil_2[1])/1000)+float(time_mil_2[0])
if score_final != 0:
silence_time = time_mil_1_final-score_final
if silence_time > 1.0:
end_time = score_final
new_audio_file = split_audios(start_time,end_time,audio_file,cut_times)
new_audios.append(new_audio_file)
audios_with_silence.append(mixed_silence(new_audio_file,cut_times))
start_time = time_mil_1_final
cut_times = cut_times + 1
score_final = time_mil_2_final
audios_with_silence.append(split_audios(start_time,30,audio_file,cut_times+1))
mixed_audio(audios_with_silence,audio_file)
utils.delete_files(new_audios)
utils.delete_files(audios_with_silence)
utils.delete_file("result.txt")
def train(self, scratch, game, display):
p = PLE(game,
fps=30,
frame_skip=1,
num_steps=1,
force_fps=True,
display_screen=display)
t1 = time.time()
fname = None
if not scratch:
fname = self.load()
else:
delete_files(self.DATA_DIREC)
f0, step, nb_save, nb_games = init_train(fname, self.DATA_DIREC)
eps_tau = (self.NB_FRAMES - f0) // 8
scores = []
while step < self.NB_FRAMES:
if len(scores) == self.SCORE_FREQ:
print('States visited:', len(self.Q))
print_scores(scores, self.SCORE_FREQ)
scores = []
p.reset_game()
state = game.getGameState()
state_tp = self.discretize(state)
if state_tp not in self.Q:
self.Q[state_tp] = [0, 0]
act = 1
episode = deque([], self.SIZE_FIFO)
elig = {}
gscore = 0
nb_games += 1
while not p.game_over():
step += 1
if step != 0 and (step % self.SAVE_FREQ) == 0:
self.save('Q_' + chr(97 + nb_save) + '_' + str(step) +
'_' + str(nb_games) + '.p')
nb_save += 1
if step != 0 and (step % self.EPS_UPDATE_FREQ) == 0:
self.epsilon = update_epsilon(step, f0, self.EPS0, eps_tau,
self.NB_FRAMES)
# 1) Observe r, s′
bare_reward = p.act(ACTIONS[act])
reward = self.reward_engineering(bare_reward)
new_state = game.getGameState()
new_state_tp = self.discretize(new_state)
# 2) Choose a′ (GLIE actor) using Q
if new_state_tp not in self.Q:
self.Q[new_state_tp] = [0, 0]
qvals = self.get_qvals(new_state)
new_act = self.greedy_action(qvals, self.epsilon)
# 3) Temporal difference: δ=r+γQ(s′,a′)−Q(s,a)
delta = reward + self.GAMMA * self.Q[new_state_tp][
new_act] - self.Q[state_tp][act]
# 4) Update Q
episode.append((state_tp, act))
elig[(state_tp, act)] = 1
for (state_tp_ep, act_ep) in episode:
self.Q[state_tp_ep][act_ep] += (
self.ALPHA * delta * elig[(state_tp_ep, act_ep)])
elig[(state_tp_ep, act_ep)] *= self.LAMBDA
# 5) s<-s', a<-a'
state = new_state
state_tp = new_state_tp
act = new_act
if bare_reward > 0:
gscore += 1
scores.append(gscore)
t2 = time.time()
# Unicode code point of a: 97
self.save('Q_' + chr(97 + nb_save) + '_' + str(step) + '_' +
str(nb_games) + '.p')
print()
print('Number of played games:', nb_games)
print('Training completed in', (t2 - t1) / 60, 'minutes')
print()
def train(self, scratch, game, display):
p = PLE(game,
fps=30,
frame_skip=1,
num_steps=1,
force_fps=True,
display_screen=display)
fname = None
if not scratch:
fname = self.load()
else:
delete_files(self.DATA_DIREC)
f0, step, nb_save, nb_games = init_train(fname, self.DATA_DIREC)
eps_tau = (self.NB_FRAMES - f0) // self.EPS_RATE
scores = []
while step < self.NB_FRAMES:
if len(scores) == self.SCORE_FREQ:
print_scores(scores, self.SCORE_FREQ)
scores = []
p.reset_game()
state = game.getGameState()
state_arr = self.state_to_arr(state)
# state_arr = self.scaler.transform(state_arr.reshape(1, -1))
gscore = 0
nb_games += 1
while not p.game_over():
step += 1
if step != 0 and (step % self.SAVE_FREQ) == 0:
self.save(
chr(97 + nb_save) + '_' + str(step) + '_' +
str(nb_games))
nb_save += 1
if step != 0 and (step % self.EPS_UPDATE_FREQ) == 0:
self.epsilon = update_epsilon(step, f0, self.EPS0, eps_tau,
self.NB_FRAMES)
print('WEIGHTS ABS MEAN')
print(abs(np.mean(self.model.get_weights()[0], axis=1)))
# 1) In s, choose a (GLIE actor)
qvals = self.get_qvals(state)
act = self.greedy_action(qvals, self.epsilon)
# 2) Observe r, s′
bare_reward = p.act(ACTIONS[act])
reward = self.reward_engineering(bare_reward)
new_state = game.getGameState()
new_state_arr = self.state_to_arr(state)
self.replay_memory.append(
(state_arr, act, reward, new_state_arr))
if (len(self.replay_memory) == self.BUFFER_SIZE
and step % self.TRAIN_FREQ == 0):
X_train = []
y_train = []
# TEST: TRAIN ONLY WITH A SMALL BUFFER BATCH
replay_memory_copy = list(self.replay_memory)[:]
random.shuffle(replay_memory_copy)
for frame in replay_memory_copy[:self.BATCH_SIZE]:
s_arr_1, act_x, bare_reward_x, s_arr_2 = frame
reward_x = self.reward_engineering(bare_reward_x)
old_qval = self.model.predict(s_arr_1, batch_size=1)
qval_new = self.model.predict(s_arr_2, batch_size=1)
max_qval = np.max(qval_new)
# terminal state
if bare_reward < 0:
delta = reward_x
else:
delta = reward_x + self.GAMMA * max_qval
y = np.zeros((1, len(ACTIONS)))
y[0][:] = old_qval[0][:]
y[0][act_x] = old_qval[0][act_x] + self.ALPHA * delta
X_train.append(s_arr_1.reshape(len(STATES), ))
y_train.append(y.reshape(len(ACTIONS), ))
X_train = np.array(X_train)
y_train = np.array(y_train)
self.model.fit(X_train,
y_train,
batch_size=self.BATCH_SIZE,
epochs=2,
verbose=False)
# 5) s <- s'
state = new_state
state_arr = new_state_arr
if bare_reward > 0:
gscore += 1
scores.append(gscore)
self.save(chr(97 + nb_save) + '_' + str(step) + '_' + str(nb_games))
def train(self, scratch, game, display):
p = PLE(game,
fps=30,
frame_skip=1,
num_steps=1,
force_fps=True,
display_screen=display)
fname = None
if not scratch:
fname = self.load()
else:
delete_files(self.DATA_DIREC)
f0, step, nb_save, nb_games = init_train(fname, self.DATA_DIREC)
eps_tau = (self.NB_FRAMES - f0) // self.EPS_RATE
scores = []
while step < self.NB_FRAMES:
if len(scores) == self.SCORE_FREQ:
print_scores(scores, self.SCORE_FREQ)
scores = []
p.reset_game()
self.game.getGameState()
screen = self.process_screen(p.getScreenRGB())
last_screens_buff = deque([screen] * 4, maxlen=NB_LAST_SCREENS)
last_screens = np.stack(last_screens_buff, axis=-1)
# gscore = 0
nb_games += 1
score = 0
while not p.game_over():
step += 1
if step != 0 and (step % self.SAVE_FREQ) == 0:
self.save(
chr(97 + nb_save) + '_' + str(step) + '_' +
str(nb_games))
nb_save += 1
if step != 0 and (step % self.EPS_UPDATE_FREQ) == 0:
self.epsilon = update_epsilon(step, f0, self.EPS0, eps_tau,
self.NB_FRAMES)
# print('WEIGHTS ABS MEAN')
# print(abs(np.mean(self.model.get_weights()[0], axis=1)))
# 1) In s, choose a (GLIE actor)
qvals = self.get_qvals(last_screens)
act = self.greedy_action(qvals, self.epsilon)
# 2) Observe r, s′
bare_reward = p.act(ACTIONS[act])
if bare_reward > 0:
score += 1
reward = self.reward_engineering(bare_reward)
screen_new = self.process_screen(p.getScreenRGB())
# update replay_memory
self.replay_memory.append(screen, act, screen_new, reward)
if len(self.replay_memory.buff) > self.MIN_REPLAY_MEMORY_SIZE:
# build minibatch
ls, actions, ls_new, r, terms = self.replay_memory.minibatch(
)
qvals_new = self.model_target.predict(ls_new)
qvals_new_max = qvals_new.max(1).reshape(
(self.BATCH_SIZE, 1))
delta = r + (1 - terms) * self.GAMMA * qvals_new_max
qvals = self.model.predict(ls)
qvals[np.arange(self.BATCH_SIZE),
actions.ravel()] = delta.ravel()
self.model.train_on_batch(x=ls, y=qvals)
if step % self.TARGET_FREQ == 0:
self.model.save(filepath=self.DATA_DIREC + 'target.h5')
self.model_target = load_model(
filepath=self.DATA_DIREC + 'target.h5')
last_screens_buff.append(screen_new)
last_screens = np.stack(last_screens_buff, axis=-1)
screen = screen_new
scores.append(score)
saved_id = self.id # delete() unsets the ID attribute, which we need in order to clear out the directory
super(Work, self).delete()
fname = "saved-work-%d.zip" % saved_id
self.id = saved_id
if hasattr(settings, "ATTIC"):
try:
target = os.path.join(settings.ATTIC, fname)
log.info("Saving work to %s" % target)
archive = utils.recreate_ingest_package(self)
utils.movefile(archive,target)
except OSError, ose:
log.error("Error encountered saving deleted work (id:%s) to %s: %r" % ( saved_id, target, ose))
else:
log.info("Cleaning up %s" % (self.get_content_directory()))
try:
utils.delete_files(content_dir)
except Exception, e:
log.exception("unable to delete %s: %r" % ( content_dir, e ) )
# this was an example of an image upload field that would automatically save a thumbnail of
# the uploaded image. It works, but it's not needed here.
##class ThumbnailImageField(models.ImageField):
## """
## Subclass of standard ImageField that automatically thumbnails the incoming image.
## """
## def __init__(self,*args,**kwargs):
## md = kwargs.get('max_dimension', 64)
## if not isinstance(md, int):
## md = 64
def run(net_path, training, validation, nem, seed, log_dir,
_run): # #944371721 is fixed seed?
# seed = 944371721 # or 1 (-0 -0 0.1 0.4)
# tf.set_random_seed(seed)
print('seed: ', seed)
# clear debug dir
if log_dir and net_path is None:
utils.create_directory(log_dir)
utils.delete_files(log_dir, recursive=True)
# Set up data pipelines
nr_iters = nem['nr_steps'] + 1
train_inputs = InputPipeLine('training',
shuffle=True,
sequence_length=nr_iters,
batch_size=training['batch_size'])
valid_inputs = InputPipeLine('validation',
shuffle=False,
sequence_length=nr_iters,
batch_size=validation['batch_size'])
# Set Weigth
with tf.variable_scope("context_and_klloss_weight"):
context_weight = tf.get_variable(name='context_weight',
dtype=tf.float32,
trainable=False,
initializer=0.0)
klloss_weight = tf.get_variable(name='klloss_weight',
dtype=tf.float32,
trainable=False,
initializer=0.0)
# Build Graph
train_op, train_op_v, train_graph, valid_graph, debug_graph = build_graphs(
train_inputs.output, valid_inputs.output)
init = tf.global_variables_initializer()
# print vars
utils.print_vars(tf.trainable_variables())
utils.print_vars(tf.global_variables())
with tf.Session() as session:
# continue training from net_path if specified.
saver = tf.train.Saver()
if net_path is not None:
saver.restore(session, net_path)
else:
session.run(init)
# start training pipelines
coord = tf.train.Coordinator()
train_enqueue_thread = threading.Thread(target=train_inputs.enqueue,
args=[session, coord])
coord.register_thread(train_enqueue_thread)
train_enqueue_thread.start()
valid_enqueue_thread = threading.Thread(target=valid_inputs.enqueue,
args=[session, coord])
coord.register_thread(valid_enqueue_thread)
valid_enqueue_thread.start()
best_valid_loss = np.inf
best_valid_epoch = 0
for epoch in range(1, training['max_epoch'] + 1):
t = time.time()
if epoch < 20: # 5 for em , 20 for rnn-em
update_context_weight = tf.assign(context_weight, 0.)
update_klloss_weight = tf.assign(klloss_weight, 0.)
train_op_v = None
else:
update_context_weight = tf.assign(context_weight, 1.)
update_klloss_weight = tf.assign(klloss_weight, 1.)
session.run(update_context_weight)
session.run(update_klloss_weight)
# ---------------
print('klloss_weight before: ', session.run(klloss_weight))
weight = tf.get_variable(
name=
'NEM/multi_rnn_cell/cell_0/EMCell_discri/input_discri/fully_connected_2/weights'
)
weight = tf.Print(weight, ['-----------------', weight])
# session.run(weight)
print(weight.eval()[:, 0])
# ---------------------
train_loss, others, train_scores, train_ub_loss_last, train_variational_loss = run_epoch(
session,
train_inputs,
train_graph,
debug_graph,
training['debug_samples'],
"train_e{}".format(epoch),
train_op=train_op,
train_op_v=train_op_v)
# ---------------------
print('klloss_weight after: ', session.run(klloss_weight))
weight = tf.get_variable(
name=
'NEM/multi_rnn_cell/cell_0/EMCell_discri/input_discri/fully_connected_2/weights'
)
weight = tf.Print(weight, ['-----------------', weight])
session.run(weight)
print(weight.eval()[:, 0])
# -------------------
# ---------------------
# weight = tf.get_variable(name='train/NEM/step_14/multi_rnn_cell/cell_0/EMCell_discri/input_discri/weighted_context')
# weight = tf.Print(weight, ['-----------------', weight])
# session.run(weight)
# print(weight.eval()[:,0])
# -------------------
add_log('training.loss', train_loss)
add_log('training.others', others)
add_log('training.score', train_scores[0])
add_log('training.score_last', train_scores[1])
add_log('training.ub_loss_last', train_ub_loss_last)
create_curve_plots('train_loss', get_logs('training.loss'),
[0, 2000])
print(
"Epoch: %d Train Loss: %.3f, ARI: %.3f (conf: %0.3f), Last ARI: %.3f (conf: %.3f) took %.3fs"
% (epoch, train_loss, train_scores[0], train_scores[2],
train_scores[1], train_scores[3], time.time() - t))
print(" Other Train Losses: ({})".format(", ".join(
["%.2f" % o for o in others.mean(0)])))
print(" Train Loss UB last: %.2f" % train_ub_loss_last)
# print(" Train Loss including variational loss--final loss: %.2f" % train_loss+train_variational_loss)
print(' Train Variatioanl Los:', train_variational_loss)
t = time.time()
valid_loss, others, valid_scores, valid_ub_loss_last, valid_variational_loss = run_epoch(
session, valid_inputs, valid_graph, debug_graph,
validation['debug_samples'], "valid_e{}".format(epoch))
# valid_scores = seq_ARI, last_ARI, seq_conf, last_conf
add_log('validation.loss', valid_loss)
add_log('validation.others', others)
add_log('validation.score', valid_scores[0])
add_log('validation.score_last', valid_scores[1])
add_log('validation.ub_loss_last', valid_ub_loss_last)
create_curve_plots('valid_loss', get_logs('validation.loss'),
[0, 2000])
create_curve_plots('valid_score', get_logs('validation.score'),
[0, 1])
create_curve_plots('valid_score_last',
get_logs('validation.score_last'), [0, 1])
print(
" Validation Loss: %.3f, ARI: %.3f (conf: %0.3f), Last ARI: %.3f (conf: %.3f) took %.3fs"
% (valid_loss, valid_scores[0], valid_scores[2],
valid_scores[1], valid_scores[3], time.time() - t))
print(" Other Validation Losses: ({})".format(", ".join(
["%.2f" % o for o in others.mean(0)])))
print(" Valid Loss UB last: %.2f" % valid_ub_loss_last)
print(" Valid Variational Loss:", valid_variational_loss)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
best_valid_epoch = epoch
_run.result = float(valid_scores[0]), float(
valid_scores[1]), float(valid_loss)
print(" Best validation loss improved to %.03f" %
best_valid_loss)
save_destination = saver.save(
session, os.path.abspath(os.path.join(log_dir, 'best')))
print(" Saved to:", save_destination)
if epoch in training['save_epochs']:
save_destination = saver.save(
session,
os.path.abspath(
os.path.join(log_dir, 'epoch_{}'.format(epoch))))
print(" Saved to:", save_destination)
best_valid_loss = min(best_valid_loss, valid_loss)
if best_valid_loss < np.min(
get_logs('validation.loss')[-training['max_patience']:]):
print(
'Early Stopping because validation loss did not improve for {} epochs'
.format(training['max_patience']))
break
if np.isnan(valid_loss):
print('Early Stopping because validation loss is nan')
pass
# break
# shutdown everything to avoid zombies
coord.request_stop()
session.run(train_inputs.queue.close(cancel_pending_enqueues=True))
session.run(valid_inputs.queue.close(cancel_pending_enqueues=True))
coord.join()
return float(get_logs('validation.score')[best_valid_epoch - 1]), float(
get_logs('validation.score_last')[best_valid_epoch - 1]), float(
get_logs('validation.loss')[best_valid_epoch - 1])
def clean_up_intermediate_files():
delete_files(TMP_DIR + '*.sh')
delete_files(TMP_DIR + '*.o')
delete_files(TMP_DIR + '*.c')
def main():
utils.delete_files()
data = Data()
algorithm = GeneticAlgorithm(data)
algorithm.run()
print("Finished!")
def run(record_grouping_score, record_relational_loss, feed_actions, net_path,
training, validation, nem, dt, seed, log_dir, _run):
save_epochs = training['save_epochs']
# clear debug dir
if log_dir and net_path is None:
utils.create_directory(log_dir)
utils.delete_files(log_dir, recursive=True)
# prep weights for print out
loss_step_weights = get_loss_step_weights()
s_loss_weights = np.sum(loss_step_weights)
dt_s_loss_weights = np.sum(loss_step_weights[-dt:])
# Set up data pipelines
nr_iters = nem['nr_steps'] + 1
out_list = ['features']
out_list.append('groups') if record_grouping_score else None
out_list.append(record_relational_loss) if record_relational_loss else None
out_list.append('actions') if feed_actions else None
train_inputs = InputPipeLine('training',
shuffle=True,
out_list=out_list,
sequence_length=nr_iters,
batch_size=training['batch_size'])
valid_inputs = InputPipeLine('validation',
shuffle=False,
out_list=out_list,
sequence_length=nr_iters,
batch_size=validation['batch_size'])
# Build Graph
train_op, train_graph, valid_graph, debug_graph = build_graphs(
train_inputs.output, valid_inputs.output)
init = tf.global_variables_initializer()
# print vars
utils.print_vars(tf.trainable_variables())
with tf.Session() as session:
tf.set_random_seed(seed)
# continue training from net_path if specified
saver = tf.train.Saver()
if net_path is not None:
saver.restore(session, net_path)
else:
session.run(init)
# start training pipelines
writer = tf.summary.FileWriter(
log_dir,
graph=session.graph,
)
coord = tf.train.Coordinator()
train_enqueue_thread = threading.Thread(target=train_inputs.enqueue,
args=[session, coord])
coord.register_thread(train_enqueue_thread)
train_enqueue_thread.start()
valid_enqueue_thread = threading.Thread(target=valid_inputs.enqueue,
args=[session, coord])
coord.register_thread(valid_enqueue_thread)
valid_enqueue_thread.start()
best_valid_loss = np.inf
best_valid_epoch = 0
for epoch in range(1, training['max_epoch'] + 1):
# run train epoch
t = time.time()
log_dict = run_epoch(session,
train_inputs,
train_graph,
debug_graph,
training['debug_samples'],
"train_e{}".format(epoch),
train_op=train_op)
# log all items in dict
log_log_dict('training', log_dict)
# produce print-out
print("\n" + 80 * "%" + " EPOCH {} ".format(epoch) + 80 * "%")
print_log_dict(log_dict, 'Train', t, dt, s_loss_weights,
dt_s_loss_weights)
# run valid epoch
t = time.time()
log_dict = run_epoch(session, valid_inputs, valid_graph,
debug_graph, validation['debug_samples'],
"valid_e{}".format(epoch))
# add logs
log_log_dict('validation', log_dict)
# produce plots
create_curve_plots(
'loss', {
'training': get_logs('training.loss'),
'validation': get_logs('validation.loss')
}, [0, 1000], [0, 200])
create_curve_plots(
'r_loss', {
'training': get_logs('training.r_loss'),
'validation': get_logs('validation.r_loss')
}, [0, 100], [0, 20])
create_curve_plots(
'score', {
'score': get_logs('validation.score'),
'score_last': get_logs('validation.score_last')
}, [0, 1], None)
# produce print-out
print("\n")
print_log_dict(log_dict, 'Validation', t, dt, s_loss_weights,
dt_s_loss_weights)
if log_dict['loss'] < best_valid_loss:
best_valid_loss = log_dict['loss']
best_valid_epoch = epoch
_run.result = float(log_dict['score']), float(log_dict['score_last']), \
float(log_dict['loss']), float(log_dict['ub_loss']), \
float(np.sum(log_dict['others'][-dt:, 1])/dt_s_loss_weights), \
float(np.sum(log_dict['others_ub'][-dt:, 1]) / dt_s_loss_weights), \
float(np.sum(log_dict['others'][-dt:, 2]) / dt_s_loss_weights), \
float(np.sum(log_dict['others_ub'][-dt:, 2]) / dt_s_loss_weights), \
float(log_dict['r_loss']), float(log_dict['r_ub_loss']), \
float(np.sum(log_dict['r_others'][-dt:, 1]) / dt_s_loss_weights), \
float(np.sum(log_dict['r_others_ub'][-dt:, 1]) / dt_s_loss_weights), \
float(np.sum(log_dict['r_others'][-dt:, 2]) / dt_s_loss_weights), \
float(np.sum(log_dict['r_others_ub'][-dt:, 2]) / dt_s_loss_weights)
print(" Best validation loss improved to %.03f" %
best_valid_loss)
save_destination = saver.save(
session, os.path.abspath(os.path.join(log_dir, 'best')))
print(" Saved to:", save_destination)
if epoch in save_epochs:
save_destination = saver.save(
session,
os.path.abspath(
os.path.join(log_dir, 'epoch_{}'.format(epoch))))
print(" Saved to:", save_destination)
best_valid_loss = min(best_valid_loss, log_dict['loss'])
if best_valid_loss < np.min(
get_logs('validation.loss')[-training['max_patience']:]):
print(
'Early Stopping because validation loss did not improve for {} epochs'
.format(training['max_patience']))
break
if np.isnan(log_dict['loss']):
print('Early Stopping because validation loss is nan')
break
# shutdown everything to avoid zombies
coord.request_stop()
session.run(train_inputs.queue.close(cancel_pending_enqueues=True))
session.run(valid_inputs.queue.close(cancel_pending_enqueues=True))
coord.join()
# reset the graph
tf.reset_default_graph()
# gather best results
best_valid_score = float(
get_logs('validation.score')[best_valid_epoch - 1])
best_valid_score_last = float(
get_logs('validation.score_last')[best_valid_epoch - 1])
best_valid_loss = float(get_logs('validation.loss')[best_valid_epoch - 1])
best_valid_ub_loss = float(
get_logs('validation.ub_loss')[best_valid_epoch - 1])
best_valid_intra_loss = float(
np.sum(get_logs('validation.others')[best_valid_epoch - 1][-dt:, 1]) /
dt_s_loss_weights)
best_valid_intra_ub_loss = float(
np.sum(
get_logs('validation.others_ub')[best_valid_epoch - 1][-dt:, 1]) /
dt_s_loss_weights)
best_valid_inter_loss = float(
np.sum(get_logs('validation.others')[best_valid_epoch - 1][-dt:, 2]) /
dt_s_loss_weights)
best_valid_inter_ub_loss = float(
np.sum(
get_logs('validation.others_ub')[best_valid_epoch - 1][-dt:, 2]) /
dt_s_loss_weights)
best_valid_r_loss = float(
get_logs('validation.r_loss')[best_valid_epoch - 1])
best_valid_r_ub_loss = float(
get_logs('validation.r_ub_loss')[best_valid_epoch - 1])
best_valid_r_intra_loss = float(
np.sum(get_logs('validation.r_others')[best_valid_epoch - 1][-dt:, 1])
/ dt_s_loss_weights)
best_valid_r_intra_ub_loss = float(
np.sum(
get_logs('validation.r_others_ub')[best_valid_epoch - 1][-dt:, 1])
/ dt_s_loss_weights)
best_valid_r_inter_loss = float(
np.sum(get_logs('validation.r_others')[best_valid_epoch - 1][-dt:, 2])
/ dt_s_loss_weights)
best_valid_r_inter_ub_loss = float(
np.sum(
get_logs('validation.r_others_ub')[best_valid_epoch - 1][-dt:, 2])
/ dt_s_loss_weights)
return best_valid_score, best_valid_score_last, best_valid_loss, best_valid_ub_loss, best_valid_intra_loss, \
best_valid_intra_ub_loss, best_valid_inter_loss, best_valid_inter_ub_loss, best_valid_r_loss, \
best_valid_r_ub_loss, best_valid_r_intra_loss, best_valid_r_intra_ub_loss, best_valid_r_inter_loss, \
best_valid_r_inter_ub_loss
# Local
import constants
from debug import DEBUG_COMMAND_LIST
import dirs
import envvars
import gamedata
import utils
from localization import LOCALIZATION_DATA
# Minimum Python version check
if sys.version_info < (3, 6):
sys.exit("The White Rabbit does not support Python versions below 3.6. Please install a newer version")
# Clear .pkl files on startup to avoid export bug
utils.delete_files(dirs.FONT_DIR, "pkl")
# Enable Server Members gateway intent to find all users
intents = discord.Intents.all()
bot = commands.Bot(command_prefix=commands.when_mentioned_or(constants.COMMAND_PREFIX), intents=intents)
bot.games = {}
# Localization
BOT_CHANNEL = LOCALIZATION_DATA["channels"]["bot-channel"]
SPECTATOR_ROLE = LOCALIZATION_DATA["spectator-role"]
@bot.event
async def on_ready():
# Set custom status
def run(net_path, training, validation, nem, seed, log_dir, _run):
# clear debug dir
if log_dir and net_path is None:
utils.create_directory(log_dir)
utils.delete_files(log_dir, recursive=True)
# Set up data pipelines
nr_iters = nem['nr_steps'] + 1
train_inputs = InputPipeLine('training',
shuffle=True,
sequence_length=nr_iters,
batch_size=training['batch_size'])
valid_inputs = InputPipeLine('validation',
shuffle=False,
sequence_length=nr_iters,
batch_size=validation['batch_size'])
# Build Graph
train_op, train_graph, valid_graph, debug_graph = build_graphs(
train_inputs.output, valid_inputs.output)
init = tf.global_variables_initializer()
# print vars
utils.print_vars(tf.trainable_variables())
with tf.Session() as session:
tf.set_random_seed(seed)
# continue training from net_path if specified.
saver = tf.train.Saver()
if net_path is not None:
saver.restore(session, net_path)
else:
session.run(init)
# start training pipelines
coord = tf.train.Coordinator()
train_enqueue_thread = threading.Thread(target=train_inputs.enqueue,
args=[session, coord])
coord.register_thread(train_enqueue_thread)
train_enqueue_thread.start()
valid_enqueue_thread = threading.Thread(target=valid_inputs.enqueue,
args=[session, coord])
coord.register_thread(valid_enqueue_thread)
valid_enqueue_thread.start()
best_valid_loss = np.inf
best_valid_epoch = 0
for epoch in range(1, training['max_epoch'] + 1):
t = time.time()
train_loss, others, train_scores, train_ub_loss_last = run_epoch(
session,
train_inputs,
train_graph,
debug_graph,
training['debug_samples'],
"train_e{}".format(epoch),
train_op=train_op)
add_log('training.loss', train_loss)
add_log('training.others', others)
add_log('training.score', train_scores[0])
add_log('training.score_last', train_scores[1])
add_log('training.ub_loss_last', train_ub_loss_last)
create_curve_plots('train_loss', get_logs('training.loss'),
[0, 2000])
print(
"Epoch: %d Train Loss: %.3f, ARI: %.3f (conf: %0.3f), Last ARI: %.3f (conf: %.3f) took %.3fs"
% (epoch, train_loss, train_scores[0], train_scores[2],
train_scores[1], train_scores[3], time.time() - t))
print(" Other Train Losses: ({})".format(", ".join(
["%.2f" % o for o in others.mean(0)])))
print(" Train Loss UB last: %.2f" % train_ub_loss_last)
t = time.time()
valid_loss, others, valid_scores, valid_ub_loss_last = run_epoch(
session, valid_inputs, valid_graph, debug_graph,
validation['debug_samples'], "valid_e{}".format(epoch))
# valid_scores = seq_ARI, last_ARI, seq_conf, last_conf
add_log('validation.loss', valid_loss)
add_log('validation.others', others)
add_log('validation.score', valid_scores[0])
add_log('validation.score_last', valid_scores[1])
add_log('validation.ub_loss_last', valid_ub_loss_last)
create_curve_plots('valid_loss', get_logs('validation.loss'),
[0, 2000])
create_curve_plots('valid_score', get_logs('validation.score'),
[0, 1])
create_curve_plots('valid_score_last',
get_logs('validation.score_last'), [0, 1])
print(
" Validation Loss: %.3f, ARI: %.3f (conf: %0.3f), Last ARI: %.3f (conf: %.3f) took %.3fs"
% (valid_loss, valid_scores[0], valid_scores[2],
valid_scores[1], valid_scores[3], time.time() - t))
print(" Other Validation Losses: ({})".format(", ".join(
["%.2f" % o for o in others.mean(0)])))
print(" Valid Loss UB last: %.2f" % valid_ub_loss_last)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
best_valid_epoch = epoch
_run.result = float(valid_scores[0]), float(
valid_scores[1]), float(valid_loss)
print(" Best validation loss improved to %.03f" %
best_valid_loss)
save_destination = saver.save(
session, os.path.abspath(os.path.join(log_dir, 'best')))
print(" Saved to:", save_destination)
if epoch in training['save_epochs']:
save_destination = saver.save(
session,
os.path.abspath(
os.path.join(log_dir, 'epoch_{}'.format(epoch))))
print(" Saved to:", save_destination)
best_valid_loss = min(best_valid_loss, valid_loss)
if best_valid_loss < np.min(
get_logs('validation.loss')[-training['max_patience']:]):
print(
'Early Stopping because validation loss did not improve for {} epochs'
.format(training['max_patience']))
break
if np.isnan(valid_loss):
print('Early Stopping because validation loss is nan')
break
# shutdown everything to avoid zombies
coord.request_stop()
session.run(train_inputs.queue.close(cancel_pending_enqueues=True))
session.run(valid_inputs.queue.close(cancel_pending_enqueues=True))
coord.join()
return float(get_logs('validation.score')[best_valid_epoch - 1]), float(
get_logs('validation.score_last')[best_valid_epoch - 1]), float(
get_logs('validation.loss')[best_valid_epoch - 1])
def run(record_grouping_score, record_relational_loss, feed_actions, net_path,
training, validation, nem, dt, seed, log_dir, _run):
if torch.cuda.is_available():
torch.set_default_tensor_type('torch.cuda.FloatTensor')
save_epochs = training['save_epochs']
# clear debug dir
if log_dir and net_path is None:
utils.create_directory(log_dir)
utils.delete_files(log_dir, recursive=True)
# prep weights for print out
loss_step_weights = get_loss_step_weights()
s_loss_weights = np.sum(loss_step_weights)
dt_s_loss_weights = np.sum(loss_step_weights[-dt:])
# Set up data pipelines
nr_iters = nem['nr_steps'] + 1
out_list = ['features']
out_list.append('groups') if record_grouping_score else None
out_list.append(record_relational_loss) if record_relational_loss else None
out_list.append('actions') if feed_actions else None
train_dataset = InputDataset("training",
training['batch_size'],
out_list,
sequence_length=nem['nr_steps'] + 1)
valid_dataset = InputDataset("validation",
validation['batch_size'],
out_list,
sequence_length=nem['nr_steps'] + 1)
train_data_loader = DataLoader(dataset=train_dataset,
batch_size=1,
shuffle=False,
num_workers=training['num_workers'],
collate_fn=collate)
valid_data_loader = DataLoader(dataset=valid_dataset,
batch_size=1,
shuffle=False,
num_workers=training['num_workers'],
collate_fn=collate)
# Get dimensions
input_shape = train_dataset._data_in_file['features'].shape
W, H, C = list(input_shape)[-3:]
inner_cell = R_NEM(nem['k'])
nem_cell = NEMCell(inner_cell,
input_shape=(W, H, C),
distribution=nem['pixel_dist'])
optimizer = set_up_optimizer(
list(nem_cell.parameters()) + list(inner_cell.parameters()))
best_valid_loss = np.inf
best_valid_epoch = 0
for epoch in range(1, training['max_epoch'] + 1):
# run train epoch
t = time.time()
log_dict = run_epoch(nem_cell,
optimizer,
train_data_loader,
train=True)
# log all items in dict
log_log_dict('training', log_dict)
# produce print-out
print("\n" + 80 * "%" + " EPOCH {} ".format(epoch) + 80 * "%")
print_log_dict(log_dict, 'Train', t, dt, s_loss_weights,
dt_s_loss_weights)
# run valid epoch
t = time.time()
log_dict = run_val_epoch(nem_cell, optimizer, valid_data_loader)
# add logs
log_log_dict('validation', log_dict)
# produce plots
create_curve_plots(
'loss', {
'training': get_logs('training.loss'),
'validation': get_logs('validation.loss')
}, [0, 1000], [0, 200])
create_curve_plots(
'r_loss', {
'training': get_logs('training.r_loss'),
'validation': get_logs('validation.r_loss')
}, [0, 100], [0, 20])
create_curve_plots(
'score', {
'training': get_logs('training.score'),
'validation': get_logs('validation.score')
}, [0, 1], None)
# produce print-out
print("\n")
print_log_dict(log_dict, 'Validation', t, dt, s_loss_weights,
dt_s_loss_weights)
if log_dict['loss'] < best_valid_loss:
best_valid_loss = log_dict['loss']
best_valid_epoch = epoch
_run.result = float(log_dict['score']), \
float(log_dict['loss']), float(log_dict['ub_loss']), float(log_dict['r_loss']), float(log_dict['r_ub_loss'])
#float(np.sum(log_dict['others'][-dt:, 1])/dt_s_loss_weights), \
#float(np.sum(log_dict['others_ub'][-dt:, 1]) / dt_s_loss_weights), \
#float(np.sum(log_dict['others'][-dt:, 2]) / dt_s_loss_weights), \
#float(np.sum(log_dict['others_ub'][-dt:, 2]) / dt_s_loss_weights), \
#float(np.sum(log_dict['r_others'][-dt:, 1]) / dt_s_loss_weights), \
#float(np.sum(log_dict['r_others_ub'][-dt:, 1]) / dt_s_loss_weights), \
#float(np.sum(log_dict['r_others'][-dt:, 2]) / dt_s_loss_weights), \
#float(np.sum(log_dict['r_others_ub'][-dt:, 2]) / dt_s_loss_weights)
print(" Best validation loss improved to %.03f" %
best_valid_loss)
torch.save(nem_cell.state_dict(),
os.path.abspath(os.path.join(log_dir, 'best')))
print(" Saved to:",
os.path.abspath(os.path.join(log_dir, 'best')))
if epoch in save_epochs:
torch.save(
nem_cell.state_dict(),
os.path.abspath(os.path.join(log_dir,
'epoch_{}'.format(epoch))))
print(
" Saved to:",
os.path.abspath(os.path.join(log_dir,
'epoch_{}'.format(epoch))))
best_valid_loss = min(best_valid_loss, log_dict['loss'])
if best_valid_loss < np.min(
get_logs('validation.loss')[-training['max_patience']:]):
print(
'Early Stopping because validation loss did not improve for {} epochs'
.format(training['max_patience']))
break
if np.isnan(log_dict['loss']):
print('Early Stopping because validation loss is nan')
break
# gather best results
best_valid_score = float(
get_logs('validation.score')[best_valid_epoch - 1])
#best_valid_score_last = float(get_logs('validation.score_last')[best_valid_epoch - 1])
best_valid_loss = float(get_logs('validation.loss')[best_valid_epoch - 1])
best_valid_ub_loss = float(
get_logs('validation.ub_loss')[best_valid_epoch - 1])
#best_valid_intra_loss = float(np.sum(get_logs('validation.others')[best_valid_epoch - 1][-dt:, 1])/dt_s_loss_weights)
#best_valid_intra_ub_loss = float(np.sum(get_logs('validation.others_ub')[best_valid_epoch - 1][-dt:, 1])/dt_s_loss_weights)
#best_valid_inter_loss = float(np.sum(get_logs('validation.others')[best_valid_epoch - 1][-dt:, 2])/dt_s_loss_weights)
#best_valid_inter_ub_loss = float(np.sum(get_logs('validation.others_ub')[best_valid_epoch - 1][-dt:, 2])/dt_s_loss_weights)
best_valid_r_loss = float(
get_logs('validation.r_loss')[best_valid_epoch - 1])
best_valid_r_ub_loss = float(
get_logs('validation.r_ub_loss')[best_valid_epoch - 1])
#best_valid_r_intra_loss = float(np.sum(get_logs('validation.r_others')[best_valid_epoch - 1][-dt:, 1])/dt_s_loss_weights)
#best_valid_r_intra_ub_loss = float(np.sum(get_logs('validation.r_others_ub')[best_valid_epoch - 1][-dt:, 1])/dt_s_loss_weights)
#best_valid_r_inter_loss = float(np.sum(get_logs('validation.r_others')[best_valid_epoch - 1][-dt:, 2])/dt_s_loss_weights)
#best_valid_r_inter_ub_loss = float(np.sum(get_logs('validation.r_others_ub')[best_valid_epoch - 1][-dt:, 2])/dt_s_loss_weights)
return best_valid_score, best_valid_loss, best_valid_ub_loss, \
best_valid_r_loss, best_valid_r_ub_loss
