def print_progress(self, batch_i, n_batches, b_loss, mean_loss, train=True):
init_c = '\033[0m' if train else '\033[38;5;238m'
whites = ' '.join([''] * 12)
percent = 20 * (batch_i + 1) / n_batches
progress_s = ''.join(['-'] * percent)
remainder_s = ''.join([' '] * (20 - percent))
loss_name = 'train_loss' if train else 'val_loss'
if train:
t_out = time.time() - self.t_train
else:
t_out = time.time() - self.t_val
time_s = time_to_string(t_out)
t_eta = (t_out / (batch_i + 1)) * (n_batches - (batch_i + 1))
eta_s = time_to_string(t_eta)
batch_s = '%s%sEpoch %03d (%03d/%03d) [%s>%s] %s %f (%f) %s / ETA %s%s' % (
init_c, whites, self.epoch, batch_i + 1, n_batches,
progress_s, remainder_s,
loss_name, b_loss, mean_loss, time_s, eta_s, '\033[0m'
)
print('\033[K', end='')
print(batch_s, end='\r')
sys.stdout.flush()
def predict(
self,
data,
features,
bb,
verbose=True
):
# Init
self.eval()
self.drop = False
self.dropout = 0
whites = ' '.join([''] * 12)
results = []
with torch.no_grad():
cases = len(data)
t_in = time.time()
for i, (data_i, feat_i) in enumerate(zip(data, features)):
# We test the model with the current batch
inputd_i = to_torch_var(
[data_i[tuple([slice(None)] + bb)]], self.device
)
inputf_i = to_torch_var([[feat_i]], self.device)
torch.cuda.synchronize()
pred = F.relu(self(inputd_i, inputf_i)).squeeze().tolist()
torch.cuda.synchronize()
torch.cuda.empty_cache()
results.append(pred)
t_out = time.time() - t_in
t_s = time_to_string(t_out)
# Print stuff
if verbose:
percent = 20 * (i + 1) / cases
progress_s = ''.join(['-'] * percent)
remainder_s = ''.join([' '] * (20 - percent))
t_eta = (t_out / (i + 1)) * (cases - (i + 1))
eta_s = time_to_string(t_eta)
print(
'\033[K%sTested case (%02d/%02d) [%s>%s]'
' %s / ETA: %s' % (
whites, i, cases, progress_s, remainder_s,
t_s, eta_s
),
end='\r'
)
sys.stdout.flush()
if verbose:
print('\033[K%sTesting finished succesfully' % whites)
return results
def on_bt_time_add_clicked(self, widget):
if self.time_mark == "A":
astr = self.label_A.get_text()
a = utils.string_to_time(astr)
a = a+1
self.label_A.set_text(utils.time_to_string(a))
elif self.time_mark == "B":
astr = self.label_B.get_text()
b = utils.string_to_time(astr)
b = b+1
self.label_B.set_text(utils.time_to_string(b))
def current_info(self, verbosity=1, expanded=True):
total_num = sum([
self.population_dict[species]['statistics']['total']
for species in self.species_names
])
pstring = 't = %s' % (self.current_time)
if verbosity >= 1:
if expanded:
pstring = ('... t = %s\n' %
str(self.current_time).zfill(self.pad_zeros) +
' Number of organisms: %s\n' % total_num)
else:
rt_pstring = 't = %s: %s organisms' % (self.current_time,
total_num)
if verbosity >= 4:
if expanded:
for species_name in self.species_names:
pstring += (
' %s: %d organisms\n' %
(species_name, self.population_dict[species_name]
['statistics']['total']))
else:
rt_pstring = rt_pstring + ' ('
for i, species_name in enumerate(self.species_names):
rt_pstring += str(self.population_dict[species_name]
['statistics']['total'])
if i != len(self.species_names) - 1:
rt_pstring += ':'
rt_pstring += ')'
if verbosity >= 2:
now = time.time()
last_time_diff = now - self.last_timestamp
self.last_timestamp = now
if expanded:
pstring += (' Time elapsed since last time step: %s\n' %
utils.time_to_string(last_time_diff))
else:
pstring = rt_pstring + (' (%s)' %
(utils.time_to_string(last_time_diff)))
if verbosity >= 3:
start_time_diff = now - self.start_timestamp
if expanded:
pstring += (' Time elapsed since start: %s\n' %
utils.time_to_string(start_time_diff))
else:
pstring = rt_pstring + (' (%s; %s)' % (utils.time_to_string(
last_time_diff), utils.time_to_string(start_time_diff)))
return pstring
def segment(
self,
data,
verbose=True
):
# Init
self.drop = False
self.dropout = 0
self.eval()
whites = ' '.join([''] * 12)
results = []
with torch.no_grad():
cases = len(data)
t_in = time.time()
for i, data_i in enumerate(data):
# We test the model with the current batch
input_i = torch.unsqueeze(
to_torch_var(data_i, self.device), 0
)
torch.cuda.synchronize()
pred = np.array(self(input_i).squeeze().tolist())
torch.cuda.synchronize()
torch.cuda.empty_cache()
results.append(pred)
t_out = time.time() - t_in
t_s = time_to_string(t_out)
# Print stuff
if verbose:
percent = 20 * (i + 1) / cases
progress_s = ''.join(['-'] * percent)
remainder_s = ''.join([' '] * (20 - percent))
t_eta = (t_out / (i + 1)) * (cases - (i + 1))
eta_s = time_to_string(t_eta)
print(
'\033[K%sTested case (%02d/%02d) [%s>%s]'
' %s / ETA: %s' % (
whites, i, cases, progress_s, remainder_s,
t_s, eta_s
),
end='\r'
)
sys.stdout.flush()
if verbose:
print('\033[K%sTesting finished succesfully' % whites)
return results
def on_bt_time_sub_clicked(self, widget):
if self.time_mark == "A":
astr = self.label_A.get_text()
a = utils.string_to_time(astr)
a = a-1
if a< 0 :
a = 0
self.label_A.set_text(utils.time_to_string(a))
elif self.time_mark == "B":
astr = self.label_B.get_text()
b = utils.string_to_time(astr)
b = b-1
if b < 0:
b=0
self.label_B.set_text(utils.time_to_string(b))
def time_sum_chart(names, values, title="", xlabel="", ylabel=""):
if not names or not values:
return _empty_chart(title, xlabel, ylabel)
figure = mpl_Figure()
canvas = mpl_FigureCanvas(figure)
figure.set_canvas(canvas)
ax = figure.add_subplot(111, projection=BasicChart.name)
size = len(values)
x = xrange(size)
n, bins, patches = ax.hist(x, size,
range=(0, size), weights=values, facecolor='green', alpha=0.75)
xticks = map(lambda x: x+0.5, x)
ax.set_xticks(xticks)
ax.set_xticklabels(names)
for label in ax.xaxis.get_ticklabels():
label.set_fontsize(7)
label.set_rotation(-45)
label.set_horizontalalignment('left')
ax.yaxis.set_major_formatter(mpl_FuncFormatter(
lambda time, pos: utils.time_to_string(time)[:10]))
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
return ChartWidget(figure, with_legend=False, xlock=True)
def histogram(names, values, title="", xlabel="", ylabel=""):
if not names or not values:
return _empty_chart(title, xlabel, ylabel)
figure = mpl_Figure()
canvas = mpl_FigureCanvas(figure)
figure.set_canvas(canvas)
ax = figure.add_subplot(111, projection=BasicChart.name)
colors = [cm.hsv(float(i)/len(values)) for i in xrange(len(values))]
n, bins, patches = ax.hist(
values, 10, normed=0, histtype="bar", label=names, color=colors)
for label in ax.xaxis.get_ticklabels():
label.set_rotation(-35)
label.set_horizontalalignment('left')
ax.plegend = ax.legend(loc="upper right", fancybox=True, shadow=True)
ax.xaxis.set_major_formatter(mpl_FuncFormatter(
lambda time, pos: utils.time_to_string(time)[:-7]))
ax.set_xlim(xmin=0)
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
return ChartWidget(figure, xlock=True)
def histogram(names, values, title="", xlabel="", ylabel=""):
if not names or not values:
return _empty_chart(title, xlabel, ylabel)
figure = mpl_Figure()
canvas = mpl_FigureCanvas(figure)
figure.set_canvas(canvas)
ax = figure.add_subplot(111, projection=BasicChart.name)
colors = [cm.hsv(float(i) / len(values)) for i in xrange(len(values))]
n, bins, patches = ax.hist(values,
10,
normed=0,
histtype="bar",
label=names,
color=colors)
for label in ax.xaxis.get_ticklabels():
label.set_rotation(-35)
label.set_horizontalalignment('left')
ax.plegend = ax.legend(loc="upper right", fancybox=True, shadow=True)
ax.xaxis.set_major_formatter(
mpl_FuncFormatter(lambda time, pos: utils.time_to_string(time)[:-7]))
ax.set_xlim(xmin=0)
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
return ChartWidget(figure, xlock=True)
def rnd_time():
return utils.time_to_string(
year = randint(2015,2018),
month = randint(0,11),
day=randint(0,28),
hour=randint(0,23),
minute=randint(0,59),
sec=randint(0,59)
)
def on_bt_play_clicked(self, widget):
model = self.iconview_src.get_model()
selected = self.iconview_src.get_selected_items()
if len(selected) == 0:
return
item = selected[0][0]
filename = model[item][COL_PATH]
a_time = self.label_A.get_text()
b_time = utils.time_to_string(self.timeline.getB() - self.timeline.getA())
self.player.preview(filename,a_time,b_time)
def update_labels(self):
def format(num, max):
if num is not None:
return "{0:0>{1}}".format(num, len(str(max)))
else:
return "-" * len(str(max))
index = self.get_event_index()
last_index = self.tracelog.get_runinstances_count() - 1
m = str(last_index)
maxtime = utils.time_to_string(self.tracelog.get_event_time(last_index))
self.counter_label.set_text("{0:0>{2}}/{1}".format(index, m, len(m)))
time = "{0:0>{1}}".format(utils.time_to_string(self.tracelog.get_event_time(index)),
len(maxtime))
text = "<span font_family='monospace'>{0} {2} {1}</span>".format(
self.tracelog.get_event_process(index),
self.tracelog.get_event_name(index),
time)
self.info_label.set_markup(text)
def show(self, time, driver_name, timer=FC.FASTEST_LAP_DISPLAY_TIME):
ac.setText(self.nameLabel, driver_name.split()[0])
ac.setText(self.lastNameLabel, driver_name.split()[-1].upper())
ac.setText(self.timeLabel, time_to_string(time))
ac.setVisible(self.fastestLapBackground, 1)
ac.setVisible(self.fastestLapBanner, 1)
ac.setVisible(self.nameLabel, 1)
ac.setVisible(self.lastNameLabel, 1)
ac.setVisible(self.timeLabel, 1)
self.timer = timer
def histogram(names, values, title="", xlabel="", ylabel=""):
if not names or not values:
return _empty_chart(title, xlabel, ylabel)
figure = mpl_Figure()
canvas = mpl_FigureCanvas(figure)
figure.set_canvas(canvas)
ax = figure.add_subplot(111, projection=BasicChart.name)
lines_config = []
cpalete = cm.get_cmap()
vals_size = len(values)
for i, vals in enumerate(values):
times = [time for time, val in vals.items()]
times.sort()
new_x, new_y = [], []
values_len = 0
for time in times:
time_range = time // 5000
if time_range < values_len:
new_y[time_range] += vals[time]
else:
new_x.append(time)
new_y.append(vals[time])
values_len += 1
if len(new_y) > 0:
# TODO: how to add correct version of x-axis values??
xvals = range(0, values_len)
color = cm.Paired(float(i)/vals_size)
line, = ax.plot(xvals, new_y, color=color,
lw=1, label=names[i])
lines_config.append(LineConfig(line, xvals, new_y, color))
for label in ax.xaxis.get_ticklabels():
label.set_fontsize(9)
label.set_rotation(-35)
label.set_horizontalalignment('left')
ax.plegend = ax.legend(loc="upper right", fancybox=True, shadow=True)
_register_histogram_pick_legend(ax, ax.plegend, lines_config)
ax.xaxis.set_major_formatter(mpl_FuncFormatter(
lambda time, pos: utils.time_to_string(time)[:-7]))
ax.set_xlim(xmin=0)
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
return ChartWidget(figure)
def update_labels(self):
def format(num, max):
if num is not None:
return "{0:0>{1}}".format(num, len(str(max)))
else:
return "-" * len(str(max))
index = self.get_event_index()
last_index = self.tracelog.get_runinstances_count() - 1
m = str(last_index)
maxtime = utils.time_to_string(
self.tracelog.get_event_time(last_index))
self.counter_label.set_text("{0:0>{2}}/{1}".format(index, m, len(m)))
time = "{0:0>{1}}".format(
utils.time_to_string(self.tracelog.get_event_time(index)),
len(maxtime))
text = "<span font_family='monospace'>{0} {2} {1}</span>".format(
self.tracelog.get_event_process(index),
self.tracelog.get_event_name(index), time)
self.info_label.set_markup(text)
def show(self, id1, pos1, id2, pos2, time_gap):
ac.setBackgroundTexture(self.positionLabel1,
FC.LEADERBOARD_POSITION_LABEL[pos1 + 1])
ac.setBackgroundTexture(self.positionLabel2,
FC.LEADERBOARD_POSITION_LABEL[pos2 + 1])
ac.setText(self.gapLabel, time_to_string(time_gap))
ac.setVisible(self.rolexLabel, 1)
ac.setVisible(self.speedometerIconLabel, 1)
ac.setVisible(self.backgroundTexture, 1)
ac.setVisible(self.extendedBackgroundTexture, 1)
ac.setVisible(self.positionLabel1, 1)
ac.setVisible(self.positionLabel2, 1)
ac.setVisible(self.nameLabel1, 1)
ac.setVisible(self.nameLabel2, 1)
ac.setVisible(self.carLabel1, 1)
ac.setVisible(self.carLabel2, 1)
ac.setVisible(self.gapsIconLabelL, 1)
ac.setVisible(self.gapsIconLabelR, 1)
ac.setVisible(self.gapLabel, 1)
ac.setVisible(self.secondsLabel, 1)
if self.id1 != id1 and self.id2 != id2:
name1 = ac.getDriverName(id1)
ac.setText(self.nameLabel1, name1.split()[-1].upper())
if FC.TEAM_COLORS:
try:
ac.setBackgroundTexture(self.teamLabel1,
FC.TEAM_COLORS[name1])
ac.setBackgroundTexture(self.carLabel1, FC.CARS[name1])
ac.setVisible(self.teamLabel1, 1)
ac.setVisible(self.carLabel1, 1)
except KeyError:
ac.log("%s:Name Missing in teams.txt" % (FC.APP_NAME))
if self.id2 != id2:
name2 = ac.getDriverName(id2)
ac.setText(self.nameLabel2, name2.split()[-1].upper())
if FC.TEAM_COLORS:
try:
ac.setBackgroundTexture(self.teamLabel2,
FC.TEAM_COLORS[name2])
ac.setBackgroundTexture(self.carLabel2, FC.CARS[name2])
ac.setVisible(self.teamLabel2, 1)
ac.setVisible(self.carLabel2, 1)
except KeyError:
ac.log("%s:Name Missing in teams.txt" % (FC.APP_NAME))
self.id1 = id1
self.id2 = id2
self.visible = True
def run(timesheet):
# times = utils.create_time_index(TIME_DIV, TOTAL_TIME, start_time= START_TIME)
# data = np.full(TOTAL_TIME//TIME_DIV, 0)
times = timesheet.timesheet.index
schedule.every().day.at("00:00").do(new_day)
for time in times:
schedule.every().day.at(time).do(actions,
timesheet=timesheet.timesheet,
time=time)
schedule.every().day.at(utils.time_to_string(
timesheet.end_time)).do(data_management)
while True:
schedule.run_pending()
def print_progress(self, batch_i, n_batches, b_loss, mean_loss):
"""
Function to print the progress of a batch. It takes into account
whether we are training or validating and uses different colors to
show that. It's based on Keras arrow progress bar, but it only shows
the current (and current mean) training loss, elapsed time and ETA.
:param batch_i: Current batch number.
:param n_batches: Total number of batches.
:param b_loss: Current loss.
:param mean_loss: Current mean loss.
:return: None.
"""
init_c = '\033[0m' if self.training else '\033[38;5;238m'
percent = 25 * (batch_i + 1) // n_batches
progress_s = ''.join(['█'] * percent)
remainder_s = ''.join([' '] * (25 - percent))
loss_name = 'train_loss' if self.training else 'val_loss'
if self.training:
t_out = time.time() - self.t_train
else:
t_out = time.time() - self.t_val
time_s = time_to_string(t_out)
t_eta = (t_out / (batch_i + 1)) * (n_batches - (batch_i + 1))
eta_s = time_to_string(t_eta)
epoch_hdr = '{:}Epoch {:03} ({:03d}/{:03d} - {:05.2f}%) [{:}] '
loss_s = '{:} {:f} ({:f}) {:} / ETA {:}'
batch_s = (epoch_hdr + loss_s).format(init_c, self.epoch, batch_i + 1,
n_batches,
100 * (batch_i + 1) / n_batches,
progress_s + remainder_s,
loss_name, b_loss, mean_loss,
time_s, eta_s + '\033[0m')
print('\033[K', end='', flush=True)
print(batch_s, end='\r', flush=True)
def on_audio_item_selection_changed(self, widget):
model = widget.get_model()
selected = widget.get_selected_items()
if len(selected) == 0:
return
item = selected[0][0]
name = model[item][COL_NAME]
info = model[item][COL_INFO]
infos = info.strip('[]').split(',')
length = infos[0].strip('\'')
a = infos[1].strip(' \'')
b = infos[2].strip(' \'')
str_len = utils.time_to_string(float(length))
context = name +":"+str_len
self.statusbar.push(0,context)
print "audio file a-b",a, b
def get_new_tokens(self, place):
tokens = []
for net_instance in self.net_instances.values():
t = net_instance.new_tokens.get(place.id)
if t is not None:
for token_pointer, token_value, token_time in t:
if token_value is None:
token_value = Perspective.ARROW_LEFT
if token_time:
tokens.append("{0}@{1} ({2})".format(
token_value,
net_instance.process_id,
utils.time_to_string(token_time, seconds=True)))
else:
tokens.append("{0}@{1}".format(token_value, net_instance.process_id))
return tokens
def get_new_tokens(self, place):
tokens = []
for net_instance in self.net_instances.values():
t = net_instance.new_tokens.get(place.id)
if t is not None:
for token_pointer, token_value, token_time in t:
if token_value is None:
token_value = Perspective.ARROW_LEFT
if token_time:
tokens.append("{0}@{1} ({2})".format(
token_value,
net_instance.process_id,
utils.time_to_string(token_time, seconds=True)))
else:
tokens.append("{0}@{1}".format(token_value, net_instance.process_id))
return tokens
def place_chart(names, values, title="", xlabel="", ylabel=""):
if not names or not values:
return _empty_chart(title, xlabel, ylabel)
figure = mpl_Figure()
canvas = mpl_FigureCanvas(figure)
figure.set_canvas(canvas)
ax = figure.add_subplot(111, projection=BasicChart.name)
llines = []
for line, name in enumerate(names):
xvalues, yvalues = zip(*values[line])
llines.append((name, xvalues, yvalues))
# fill data
lines_config = []
for ldata in llines:
name, xvalues, yvalues = ldata
line, = ax.plot(
xvalues, yvalues, 'o-', drawstyle='steps-post', label=name)
lines_config.append(
LineConfig(line, xvalues, yvalues, line.get_color()))
for label in ax.xaxis.get_ticklabels():
label.set_rotation(-35)
label.set_horizontalalignment('left')
# set legend
ax.plegend = ax.legend(loc="upper left", fancybox=True, shadow=True)
ax.register_pick_legend(ax.plegend, lines_config)
ax.xaxis.set_major_formatter(mpl_FuncFormatter(
lambda time, pos: utils.time_to_string(time)[:-7]))
# set basic properties
ax.set_xlim(xmin = 0)
ax.get_figure().tight_layout()
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
return ChartWidget(figure)
def on_src_item_selection_changed(self, widget):
self.iconview_dst.unselect_all()
model = widget.get_model()
selected = widget.get_selected_items()
if len(selected) == 0:
return
item = selected[0][0]
icon = model[item][COL_PIXBUF_BIG]
self.preview_image.set_from_pixbuf(icon)
info = model[item][COL_INFO]
infos = info.strip('[]').split(',')
length = infos[0].strip('\'')
w = infos[1].strip(' \'')
h = infos[2].strip(' \'')
self.timeline.setNbFrames(float(length))
str_len = utils.time_to_string(float(length))
name = model[item][COL_NAME]
context = name +":"+str_len
self.statusbar.push(0,context)
def export_sequence(self, index):
time = utils.time_to_string(self.get_event_time(index))
name = "Tracelog upto {0}".format(time)
sequence = controlseq.ControlSequence(name)
ri = self.first_runinstance.copy()
for i in xrange(index):
event_pointer = self.timeline[i]
trace = self.traces[event_pointer["process"]]
trace.pointer = event_pointer["pointer"]
trace.process_event(ri)
if ri.last_event == "fire":
sequence.add_transition_start(ri.last_event_process,
ri.last_event_activity.transition.get_name())
elif ri.last_event == "finish":
sequence.add_transition_finish(ri.last_event_process)
elif ri.last_event == "receive":
sequence.add_receive(ri.last_event_process,
ri.last_event_activity.origin_id)
return sequence
def write_result(task_id, result):
path = os.path.join('results', '{}.csv'.format(task_id))
trimmed_list = []
with open(path, "ab+") as csv_file:
writer = csv.writer(csv_file, delimiter=';')
writer.writerow([utils.time_to_string(), result])
# check amount of lines does not exceeds maximum
# and trim csv below maximum lines
csv_file.seek(0)
reader = csv.DictReader(csv_file, delimiter=';')
reader_list = list(reader)
if len(reader_list) > MAXIMUM_LINES:
trimmed_list = reader_list[-MAXIMUM_LINES:]
# write trimmed csv if needed
if trimmed_list:
with open(path, 'wb') as output_file:
dict_writer = csv.writer(output_file, delimiter=';')
dict_writer.writerow(['time', 'result'])
for row in trimmed_list:
dict_writer.writerow([row['time'], row['result']])
def place_chart(names, values, title="", xlabel="", ylabel=""):
if not names or not values:
return _empty_chart(title, xlabel, ylabel)
figure = mpl_Figure()
canvas = mpl_FigureCanvas(figure)
figure.set_canvas(canvas)
ax = figure.add_subplot(111, projection=BasicChart.name)
# fill data
lines_config = []
for i, (xvalues, yvalues) in enumerate(values):
line, = ax.plot(xvalues,
yvalues,
'o-',
drawstyle="steps-post",
label=names[i])
lines_config.append(
LineConfig(line, xvalues, yvalues, line.get_color()))
for label in ax.xaxis.get_ticklabels():
label.set_rotation(-35)
label.set_horizontalalignment('left')
# set legend
ax.plegend = ax.legend(loc="upper left", fancybox=True, shadow=True)
ax.register_pick_legend(ax.plegend, lines_config)
ax.xaxis.set_major_formatter(
mpl_FuncFormatter(lambda time, pos: utils.time_to_string(time)[:-7]))
# set basic properties
ax.set_xlim(xmin=0)
ax.get_figure().tight_layout()
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
return ChartWidget(figure)
def __init__(self, player):
super().__init__(player)
self.selected = 0
# self.arrow = utils.get_part_i(MENU_IMAGE, (0, 64, 22, 91), (12, 14))
self.arrow = utils.ARROW
self.text = [
game.get_game_instance().get_message(t)
for t in ["save_game", "back"]
]
self.text_2 = [
game.FONT_16.render(game.get_game_instance().get_message(t) + " :",
True, (0, 0, 0)) for t in [
"date_hour",
"actual_position",
"time_play",
"pokedex",
]
]
self.last_save_f = game.FONT_16.render(
game.get_game_instance().get_message("last_save"), True,
(255, 255, 255))
self.last_save_size = game.FONT_SIZE_16[0] * len(
game.get_game_instance().get_message("last_save"))
self.time_play = game.FONT_16.render(
utils.time_to_string(game.get_game_instance().get_save_value(
"time_played", 0)), True, (0, 0, 0))
# todo: pokedex n
self.pokedex = game.FONT_16.render(
str(
sum(
map(
game.POKEDEX_CATCH.__eq__,
game.get_game_instance().
get_pokedex_catch_status_values()))), True, (0, 0, 0))
self.last_save = game.FONT_16.render(
str(
datetime.fromtimestamp(game.get_game_instance().get_save_value(
"last_save", 0)).strftime('%d/%m/%y %H:%M')), True,
(255, 255, 255))
self.open_time = time.time()
def write_metadata(metadata, out_dir):
df_columns = [
"original_audio_path", "processed_audio_path", "mel_path",
"linear_path", "timesteps", "mel_frames", "audio_time", "text",
"text_length"
]
metadata_df = pd.DataFrame(metadata, columns=df_columns)
metadata_df.to_csv(os.path.join(out_dir, 'train.csv'))
mel_frames = np.sum(metadata_df['mel_frames'].values)
timesteps = np.sum(metadata_df['timesteps'].values)
sr = hparams.sample_rate
hours = time_to_string(np.sum(metadata_df['audio_time'].values))
print(
'Write {} utterances, {} mel frames, {} audio timesteps, ({})'.format(
len(metadata_df), mel_frames, timesteps, hours))
print('Max input length (text chars): {}'.format(
np.max(metadata_df['text_length'].values)))
print('Max mel frames length: {}'.format(
np.max(metadata_df['mel_frames'].values)))
print('Max audio time : {}'.format(np.max(
metadata_df['audio_time'].values)))
def fit(
self,
train_loader,
val_loader,
epochs=100,
patience=10,
initial_dropout=0.99,
ann_rate=1e-2,
initial_lr=1,
verbose=True
):
# Init
l_names = [
'train', ' val ', ' BCK ', ' NET ', ' ED ', ' ET ',
' WT ', ' TC ', 'p_drop'
]
self.dropout = initial_dropout
model_params = filter(lambda p: p.requires_grad, self.parameters())
# If we are refining, the best train and validation losses are the ones
# we already have. That is the point of refining.
self.optimizer_alg = torch.optim.SGD(
model_params, lr=initial_lr
)
no_improv_e = 0
best_state = deepcopy(self.state_dict())
best_opt = deepcopy(self.optimizer_alg.state_dict())
best_loss_tr = np.inf
best_loss_val = np.inf
best_losses = [-np.inf] * (len(l_names))
best_e = 0
t_start = time.time()
for self.epoch in range(epochs):
# Main epoch loop
self.t_train = time.time()
self.train()
loss_tr = self.mini_batch_loop(
train_loader
)
improvement_tr = loss_tr < best_loss_tr
if improvement_tr:
best_loss_tr = loss_tr
tr_loss_s = '\033[32m%0.5f\033[0m' % loss_tr
else:
# Learning rate update
tr_loss_s = '%0.5f' % loss_tr
with torch.no_grad():
self.eval()
self.t_val = time.time()
loss_val, mid_losses = self.mini_batch_loop(
val_loader, train=False
)
losses_color = map(
lambda (bl, l): '\033[36m%s\033[0m' if l > bl else '%s',
zip(best_losses, mid_losses)
)
losses_s = map(
lambda (c, l): c % '{:8.4f}'.format(l),
zip(losses_color, mid_losses)
)
best_losses = map(
lambda (bl, l): l if l > bl else bl,
zip(best_losses, mid_losses)
)
# Patience check
improvement = loss_val < best_loss_val
loss_s = '{:7.5f}'.format(loss_val)
if improvement:
best_loss_val = loss_val
epoch_s = '\033[32mEpoch %03d\033[0m' % self.epoch
loss_s = '\033[32m%s\033[0m' % loss_s
best_e = self.epoch
best_state = deepcopy(self.state_dict())
best_opt = deepcopy(self.optimizer_alg.state_dict())
no_improv_e = 0
else:
epoch_s = 'Epoch %03d' % self.epoch
no_improv_e += 1
if not (improvement_tr or improvement):
self.optimizer_alg.load_state_dict(best_opt)
if self.dropout <= 0.5:
self.load_state_dict(best_state)
for param_group in self.optimizer_alg.param_groups:
param_group['lr'] = param_group['lr'] * 0.9
t_out = time.time() - self.t_train
t_s = time_to_string(t_out)
drop_s = '{:8.5f}'.format(self.dropout)
if self.final_dropout <= self.dropout:
self.dropout = max(
self.final_dropout, self.dropout - ann_rate
)
if verbose:
print('\033[K', end='')
whites = ' '.join([''] * 12)
if self.epoch == 0:
l_bars = '--|--'.join(
['-' * 5] * 2 + ['-' * 6] * len(l_names[2:])
)
l_hdr = ' | '.join(l_names)
print('%sEpoch num | %s |' % (whites, l_hdr))
print('%s----------|--%s--|' % (whites, l_bars))
final_s = whites + ' | '.join(
[epoch_s, tr_loss_s, loss_s] + losses_s + [drop_s, t_s]
)
print(final_s)
if no_improv_e == int(patience / (1 - self.dropout)):
break
self.epoch = best_e
self.load_state_dict(best_state)
t_end = time.time() - t_start
t_end_s = time_to_string(t_end)
if verbose:
print(
'Training finished in %d epochs (%s) '
'with minimum loss = %f (epoch %d)' % (
self.epoch + 1, t_end_s, best_loss_val, best_e)
)
def uncertainty(
self,
data,
dropout=0.5,
steps=100,
verbose=True
):
# Init
self.drop = True
self.dropout = dropout
self.eval()
whites = ' '.join([''] * 12)
seg_results = []
with torch.no_grad():
cases = len(data)
t_in = time.time()
for i, data_i in enumerate(data):
outputs = np.zeros((4,) + data_i.shape[1:])
for e in range(steps):
# We test the model with the current batch
input_i = torch.unsqueeze(
to_torch_var(data_i, self.device), 0
)
# Testing itself
torch.cuda.synchronize()
pred = np.array(self(input_i).squeeze().tolist())
torch.cuda.synchronize()
torch.cuda.empty_cache()
outputs += pred
# Print stuff
if verbose:
percent_i = 20 * (i + 1) / cases
percent_e = 20 * (e + 1) / steps
progress_is = ''.join(['-'] * percent_i)
progress_es = ''.join(['-'] * percent_e)
remainder_is = ''.join([' '] * (20 - percent_i))
remainder_es = ''.join([' '] * (20 - percent_e))
remaining_e = steps - (e + 1)
remaining_i = steps * (cases - (i + 1))
completed = steps * cases - (remaining_i + remaining_e)
t_out = time.time() - t_in
t_out_e = t_out / completed
t_s = time_to_string(t_out)
t_eta = (remaining_e + remaining_i) * t_out_e
eta_s = time_to_string(t_eta)
print(
'\033[K%sTested case (%02d/%02d - %02d/%02d) '
'[%s>%s][%s>%s] %s / ETA: %s' % (
whites, e, steps, i, cases,
progress_es, remainder_es,
progress_is, remainder_is,
t_s, eta_s
),
end='\r'
)
sys.stdout.flush()
mean_output = outputs / steps
seg_results.append(mean_output)
if verbose:
print('\033[K%sTesting finished succesfully' % whites)
return seg_results
def fit(
self,
train_loader,
val_loader,
epochs=50,
patience=5,
initial_lr=5e-2,
verbose=True
):
# Init
self.train()
self.base_model.eval()
# Now we actually train the prediction network.
best_loss_tr = np.inf
best_loss_val = np.inf
best_loss_abs = np.inf
best_loss_cat = np.inf
no_improv_e = 0
best_state = deepcopy(self.state_dict())
for p in self.base_model.parameters():
p.requires_grad = False
model_params = filter(lambda p: p.requires_grad, self.parameters())
t_start = time.time()
l_names = ['train', ' val ', ' cat ', ' abs ', 'pdrop']
best_e = 0
# SGD for L1
# self.optimizer_alg = torch.optim.SGD(
# model_params, lr=initial_lr, #weight_decay=1e-1
# )
self.optimizer_alg = torch.optim.Adam(
model_params, lr=initial_lr, # weight_decay=1e-1
)
for self.epoch in range(epochs):
# Main epoch loop
self.t_train = time.time()
loss_tr, _, _ = self.mini_batch_loop(train_loader)
if loss_tr < best_loss_tr:
best_loss_tr = loss_tr
tr_loss_s = '\033[32m{:7.4f}\033[0m'.format(loss_tr)
else:
tr_loss_s = '{:7.4f}'.format(loss_tr)
with torch.no_grad():
self.t_val = time.time()
loss_val, loss_cat, loss_abs = self.mini_batch_loop(
val_loader, False
)
# Mid losses check
if best_loss_cat > loss_cat:
best_loss_cat = loss_cat
cat_s = '\033[36m{:7.3f}\033[0m'.format(loss_cat)
else:
cat_s = '{:7.3f}'.format(loss_cat)
if best_loss_abs > loss_abs:
best_loss_abs = loss_abs
abs_s = '\033[36m{:7.3f}\033[0m'.format(loss_abs)
else:
abs_s = '{:7.3f}'.format(loss_abs)
# Patience check
if loss_val < best_loss_val:
best_loss_val = loss_val
epoch_s = '\033[32mEpoch %03d\033[0m' % self.epoch
loss_s = '\033[32m{:7.4f}\033[0m'.format(loss_val)
best_e = self.epoch
best_state = deepcopy(self.state_dict())
no_improv_e = 0
else:
epoch_s = 'Epoch %03d' % self.epoch
loss_s = '{:7.4f}'.format(loss_val)
no_improv_e += 1
t_out = time.time() - self.t_train
t_s = time_to_string(t_out)
drop_s = '{:7.5f}'.format(self.dropout)
self.dropout = max(
self.final_dropout, self.dropout - self.ann_rate
)
if verbose:
print('\033[K', end='')
whites = ' '.join([''] * 12)
if self.epoch == 0:
l_bars = '--|--'.join(['-' * 5] * len(l_names))
l_hdr = ' | '.join(l_names)
print('%sEpoch num | %s |' % (whites, l_hdr))
print('%s----------|--%s--|' % (whites, l_bars))
final_s = whites + ' | '.join(
[epoch_s, tr_loss_s, loss_s, cat_s, abs_s, drop_s] + [t_s]
)
print(final_s)
if no_improv_e == int(patience / (1 - self.dropout)):
break
self.epoch = best_e
self.load_state_dict(best_state)
t_end = time.time() - t_start
t_end_s = time_to_string(t_end)
if verbose:
print(
'Training finished in %d epochs (%s) '
'with minimum loss = %f (epoch %d)' % (
self.epoch + 1, t_end_s, best_loss_val, best_e)
)
def test(self, data, patch_size=256, verbose=True):
# Init
self.eval()
seg = list()
unc = list()
# Init
t_in = time.time()
for i, im in enumerate(data):
# Case init
t_case_in = time.time()
# This branch is only used when images are too big. In this case
# they are split in patches and each patch is trained separately.
# Currently, the image is partitioned in blocks with no overlap,
# however, it might be a good idea to sample all possible patches,
# test them, and average the results. I know both approaches
# produce unwanted artifacts, so I don't know.
if patch_size is not None:
# Initial results. Filled to 0.
seg_i = np.zeros(im.shape[1:])
unc_i = np.zeros(im.shape[1:])
limits = tuple(
list(range(0, lim, patch_size))[:-1] + [lim - patch_size]
for lim in data.shape[1:])
limits_product = list(itertools.product(*limits))
n_patches = len(limits_product)
# The following code is just a normal test loop with all the
# previously computed patches.
for pi, (xi, xj) in enumerate(limits_product):
# Here we just take the current patch defined by its slice
# in the x and y axes. Then we convert it into a torch
# tensor for testing.
xslice = slice(xi, xi + patch_size)
yslice = slice(xj, xj + patch_size)
data_tensor = to_torch_var(
np.expand_dims(im[slice(None), xslice, yslice],
axis=0))
# Testing itself.
with torch.no_grad():
torch.cuda.synchronize()
seg_pi, unc_pi, _ = self(data_tensor)
torch.cuda.synchronize()
torch.cuda.empty_cache()
# Then we just fill the results image.
seg_i[xslice, yslice] = np.squeeze(seg_pi.cpu().numpy())
unc_i[xslice, yslice] = np.squeeze(unc_pi.cpu().numpy())
# Printing
init_c = '\033[0m' if self.training else '\033[38;5;238m'
whites = ' '.join([''] * 12)
percent = 20 * (pi + 1) // n_patches
progress_s = ''.join(['-'] * percent)
remainder_s = ''.join([' '] * (20 - percent))
t_out = time.time() - t_in
t_case_out = time.time() - t_case_in
time_s = time_to_string(t_out)
t_eta = (t_case_out / (pi + 1)) * (n_patches - (pi + 1))
eta_s = time_to_string(t_eta)
batch_s = '{:}Case {:03}/{:03} ({:03d}/{:03d})' \
' [{:}>{:}] {:} ETA: {:}'.format(
init_c + whites, i + 1, len(data), pi + 1, n_patches,
progress_s, remainder_s, time_s, eta_s + '\033[0m'
)
print('\033[K', end='', flush=True)
print(batch_s, end='\r', flush=True)
else:
# If we use the whole image the process is way simpler.
# We only need to convert the data into a torch tensor,
# test it and return the results.
data_tensor = to_torch_var(np.expand_dims(im, axis=0))
# Testing
with torch.no_grad():
torch.cuda.synchronize(self.device)
seg_pi, unc_pi, _ = self(data_tensor)
torch.cuda.synchronize(self.device)
torch.cuda.empty_cache()
# Image squeezing.
# The images have a batch number at the beginning. Since each
# batch is just an image, that batch number is useless.
seg_i = np.squeeze(seg_pi.cpu().numpy())
unc_i = np.squeeze(unc_pi.cpu().numpy())
# Printing
init_c = '\033[0m' if self.training else '\033[38;5;238m'
whites = ' '.join([''] * 12)
percent = 20 * (i + 1)
progress_s = ''.join(['-'] * percent)
remainder_s = ''.join([' '] * (20 - percent))
t_out = time.time() - t_in
t_case_out = time.time() - t_case_in
time_s = time_to_string(t_out)
t_eta = (t_case_out / (i + 1)) * (len(data) - i + 1)
eta_s = time_to_string(t_eta)
batch_s = '{:}Case {:03}/{:03} [{:}>{:}] {:} ETA: {:}'.format(
init_c + whites, i + 1, len(data), progress_s, remainder_s,
time_s, eta_s + '\033[0m')
print('\033[K', end='', flush=True)
print(batch_s, end='\r', flush=True)
if verbose:
print('\033[K%sSegmentation finished' % ' '.join([''] * 12))
seg.append(seg_i)
unc.append(unc_i)
return seg, unc
def utilization_chart(names,
values,
title="",
xlabel="",
ylabel="",
idles=None):
if not names or not values:
return _empty_chart(title, xlabel, ylabel)
figure = mpl_Figure()
canvas = mpl_FigureCanvas(figure)
figure.set_canvas(canvas)
# TODO: Change it to TimeChart
ax = figure.add_subplot(111, projection=BasicChart.name)
ywidth = 2
yticks = []
if idles is not None:
for i, lidle in enumerate(idles):
y = ((i+1) * ywidth) + (i+1)
ax.broken_barh(
lidle, (y, ywidth),
edgecolor='face', facecolor='#EAA769')
for i, ldata in enumerate(values):
y = (ywidth+1) * (i+ 1)
yticks.append(y + ywidth/2)
ax.broken_barh(
ldata, (y, ywidth),
edgecolor='face', facecolor='green')
ax.set_yticks(yticks)
ax.set_yticklabels(names)
for label in ax.xaxis.get_ticklabels():
label.set_rotation(-35)
label.set_horizontalalignment('left')
for i, label in enumerate(ax.yaxis.get_ticklabels()):
# add 3 white space on the begining of name
names[i] = " %s" % names[i]
label.set_horizontalalignment("left")
label.set_verticalalignment('center')
p = mpl_Rectangle((0, 0), 1, 1, edgecolor='green', fc='green', alpha=0.75)
if idles is not None:
idle_leg = mpl_Rectangle((0,0), 1, 1, edgecolor='#eaa769', fc='#eaa769', alpha=0.75)
ax.plegend = ax.legend(
[p,idle_leg], ["Running", "Idle"], loc="upper left", fancybox=True, shadow=True)
else:
ax.plegend = ax.legend(
[p], ["Running"], loc="upper left", fancybox=True, shadow=True)
ax.xaxis.grid(True, linestyle="-", which='major', color='black', alpha=0.7)
ax.xaxis.set_major_formatter(mpl_FuncFormatter(
lambda time, pos: utils.time_to_string(time)[:-7]))
ax.set_xlim(xmin=0)
ax.get_figure().tight_layout()
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
# resize figure
w, h = figure.get_size_inches()
figure.set_size_inches(w, len(values) * 0.4)
return ChartWidget(figure, ylock=True)
def on_bt_audio_merge_clicked(self, widget):
a_model = self.iconview_audio.get_model()
v_model = self.iconview_src.get_model()
selected = self.iconview_src.get_selected_items()
if len(selected) == 0:
return
item = selected[0][0]
video_filename = v_model[item][COL_PATH]
v_info = v_model[item][COL_INFO]
v_infos = v_info.strip('[]').split(',')
v_length = v_infos[0].strip('\'')
v_end = utils.time_to_string(float(v_length))
audio_filename = ""
iter = a_model.get_iter_first()
time_p = "00:00:00"
cmd = ""
num = 0
subffix = utils.get_file_subffix(video_filename)
cmd_list =[]
while ( iter != None ):
row = a_model.get_path(iter)
audio_filename = a_model[row][COL_PATH]
info = a_model[row][COL_INFO]
infos = info.strip('[]').split(',')
a_time = infos[1].strip(' \'')
b = infos[2].strip(' \'')
if b == "0":
iter = a_model.iter_next(iter)
continue
b_time = utils.string_time_sub(b, a_time)
if a_time == time_p:
cmd = "mencoder -ss "+ a_time + " -endpos "+ b_time +" -ovc copy -oac mp3lame -audiofile "+audio_filename + " " + video_filename + " -o /tmp/dumpvideo" + str(num) +subffix
cmd_list.append(cmd)
num +=1
time_p = b_time
else:
cmd = "mencoder -ss "+ time_p + " -endpos "+ a_time +" -ovc copy -oac copy " + video_filename + " -o /tmp/dumpvideo" + str(num) +subffix
cmd_list.append(cmd)
num +=1
cmd = "mencoder -ss "+ a_time + " -endpos "+ b_time +" -ovc copy -oac mp3lame -audiofile "+audio_filename + " " + video_filename + " -o /tmp/dumpvideo" + str(num) +subffix
cmd_list.append(cmd)
num +=1
time_p = b_time
iter = a_model.iter_next(iter)
if time_p != v_end:
cmd = "mencoder -ss "+ time_p + " -endpos "+ v_end +" -ovc copy -oac copy " + video_filename + " -o /tmp/dumpvideo" + str(num) +subffix
cmd_list.append(cmd)
num += 1
filelist=""
for i in range(0,num):
filelist += " /tmp/dumpvideo"+str(i)+subffix
cmd =" mencoder -ovc lavc -oac mp3lame -o /tmp/dumpvideo.avi "+filelist
cmd_list.append(cmd)
num += 1
for i in range(0,num):
self.statusbar.push(0,cmd)
print cmd_list[i]
self.wait_run(cmd_list[i])
self.statusbar.push(0,"done")
self.load_src_file("/tmp/dumpvideo.avi")
def on_bt_a_clicked(self, widget):
val = self.timeline.get_value()
self.timeline.setA(float('%.2f'%val))
a = self.timeline.getA()
self.label_A.set_text(utils.time_to_string(a))
self.time_mark = "A"
def test_seg_validation(net_name):
# Init
c = color_codes()
options = parse_inputs()
depth = options['blocks']
filters = options['filters']
d_path = options['loo_dir']
v_path = options['val_dir']
seg_path = os.path.join(v_path, 'segmentation')
if not os.path.isdir(seg_path):
os.mkdir(seg_path)
unc_path = os.path.join(v_path, 'uncertainty')
if not os.path.isdir(unc_path):
os.mkdir(unc_path)
p = get_dirs(d_path)[0]
test_patients = filter(lambda p: 'BraTS19' in p, get_dirs(v_path))
_, test_x = get_images(test_patients, True)
print(
'Testing patients = %d' % (
len(test_patients)
)
)
# The sub-regions considered for evaluation are:
# 1) the "enhancing tumor" (ET)
# 2) the "tumor core" (TC)
# 3) the "whole tumor" (WT)
#
# The provided segmentation labels have values of 1 for NCR & NET,
# 2 for ED, 4 for ET, and 0 for everything else.
# The participants are called to upload their segmentation labels
# as a single multi-label file in nifti (.nii.gz) format.
#
# The participants are called to upload 4 nifti (.nii.gz) volumes
# (3 uncertainty maps and 1 multi-class segmentation volume from
# Task 1) onto CBICA's Image Processing Portal format. For example,
# for each ID in the dataset, participants are expected to upload
# following 4 volumes:
# 1. {ID}.nii.gz (multi-class label map)
# 2. {ID}_unc_whole.nii.gz (Uncertainty map associated with whole tumor)
# 3. {ID}_unc_core.nii.gz (Uncertainty map associated with tumor core)
# 4. {ID}_unc_enhance.nii.gz (Uncertainty map associated with enhancing tumor)
for i, (p_i, test_i) in enumerate(zip(test_patients, test_x)):
t_in = time.time()
# unc_i = np.zeros((4,) + test_i.shape[1:])
pred_i = np.zeros((4,) + test_i.shape[1:])
for f in range(5):
model_name = '%s-f%d.mdl' % (net_name, f)
net = BratsSegmentationNet(depth=depth, filters=filters)
net.load_model(os.path.join(d_path, model_name))
#
# unc_i += net.uncertainty([test_i], steps=10)[0] * 0.2
pred_i += net.segment([test_i])[0]
seg_i = np.argmax(pred_i, axis=0)
seg_i[seg_i == 3] = 4
# seg_unc_i = np.argmax(unc_i, axis=0)
# seg_unc_i[seg_unc_i == 3] = 4
tumor_mask = remove_small_regions(
seg_i.astype(np.bool), min_size=30
)
seg_i[log_not(tumor_mask)] = 0
# seg_unc_i[log_not(tumor_mask)] = 0
#
# whole_i = np.sum(unc_i[1:]) * tumor_mask.astype(np.float32)
# core_i = unc_i[1] + unc_i[-1] * tumor_mask.astype(np.float32)
# enhance_i = unc_i[-1] * tumor_mask.astype(np.float32)
#
# seg_unc_i = np.argmax(unc_i, axis=0)
# seg_unc_i[seg_unc_i == 3] = 4
niiname = os.path.join(d_path, p, p + '_seg.nii.gz')
nii = load_nii(niiname)
nii.get_data()[:] = seg_i
save_nii(nii, os.path.join(seg_path, p_i + '.nii.gz'))
# nii.get_data()[:] = seg_unc_i
# save_nii(nii, os.path.join(unc_path, p_i + '.nii.gz'))
# niiname = os.path.join(v_path, p_i, p_i + '_flair.nii.gz')
# nii = load_nii(niiname)
# nii.get_data()[:] = whole_i
# save_nii(
# nii, os.path.join(unc_path, p_i + '_unc_whole.nii.gz')
# )
# nii.get_data()[:] = core_i
# save_nii(
# nii, os.path.join(unc_path, p_i + '_unc_core.nii.gz')
# )
# nii.get_data()[:] = enhance_i
# save_nii(
# nii, os.path.join(unc_path, p_i + '_unc_enhance.nii.gz')
# )
t_s = time_to_string(time.time() - t_in)
print(
'Finished patient %s (%d/%d) %s' % (p_i, i + 1, len(test_x), t_s)
)
def fit(self,
train_loader,
val_loader,
test_loader=None,
epochs=100,
patience=20,
log_file=None,
verbose=True):
# Init
self.train_loader = train_loader
self.val_loader = val_loader
self.test_loader = test_loader
self.log_file = log_file
best_e = 0
l_names = ['train', ' val '] + [
'{:^6s}'.format(l_f['name'][:6]) for l_f in self.val_functions
]
l_bars = '--|--'.join(['-' * 5] * 2 + ['-' * 6] * len(l_names[2:]))
l_hdr = ' | '.join(l_names)
# Since we haven't trained the network yet, we'll assume that the
# initial values are the best ones.
self.best_state = deepcopy(self.state_dict())
t_start = time.time()
# We'll just take the maximum losses and accuracies (inf, -inf)
# and print the headers.
print('\033[K', end='')
print('Epoch num | {:} |'.format(l_hdr))
print('----------|--{:}--|'.format(l_bars))
best_loss_tr = [np.inf] * len(self.val_functions)
best_loss_val = [np.inf] * len(self.val_functions)
if log_file is not None:
log_file.writerow(
['Epoch', 'train', 'val'] +
['train_' + l_f['name'] for l_f in self.val_functions] +
['val_' + l_f['name']
for l_f in self.val_functions] + ['time'])
# We are looking for the output, without training, so no need to
# use grad.
with torch.no_grad():
loss_tr, best_loss_tr, _, mid_tr = self.validate(
self.train_loader, best_loss_tr)
loss_val, best_loss_val, losses_val_s, mid_val = self.validate(
self.val_loader, best_loss_val)
# Doing this also helps setting an initial best loss for all
# the necessary losses.
if verbose:
# This is just the print for each epoch, but including the
# header.
# Mid losses check
t_out = time.time() - self.t_val
t_s = time_to_string(t_out)
epoch_s = '\033[K\033[32mInit \033[0m'
tr_loss_s = '\033[32m{:7.4f}\033[0m'.format(loss_tr)
loss_s = '\033[32m{:7.4f}\033[0m'.format(loss_val)
final_s = ' | '.join([epoch_s, tr_loss_s, loss_s] +
losses_val_s + [t_s])
print(final_s)
if log_file is not None:
log_file.writerow([
'Init', '{:7.4f}'.format(loss_tr), '{:7.4f}'.format(
loss_val)
] + mid_tr.tolist() + mid_val.tolist() + [t_s])
no_improvement = 0
for self.epoch in range(epochs):
# Main epoch loop
self.t_train = time.time()
self.train()
# First we train and check if there has been an improvement.
# with torch.autograd.detect_anomaly():
# loss_tr = self.mini_batch_loop(self.train_loader)
loss_tr = self.mini_batch_loop(self.train_loader)
improvement_tr = self.best_loss_tr > loss_tr
if improvement_tr:
self.best_loss_tr = loss_tr
tr_loss_s = '\033[32m{:7.4f}\033[0m'.format(loss_tr)
else:
tr_loss_s = '{:7.4f}'.format(loss_tr)
# Then we validate and check all the losses
_, best_loss_tr, _, mid_tr = self.validate(self.train_loader,
best_loss_tr)
loss_val, best_loss_val, losses_val_s, mid_val = self.validate(
self.val_loader, best_loss_val)
# Patience check
# We check the patience to stop early if the network is not
# improving. Otherwise we are wasting resources and time.
improvement_val = self.best_loss_val > loss_val
loss_s = '{:7.4f}'.format(loss_val)
if improvement_val:
self.best_loss_val = loss_val
epoch_s = '\033[32mEpoch {:03d}\033[0m'.format(self.epoch)
loss_s = '\033[32m{:}\033[0m'.format(loss_s)
best_e = self.epoch
self.best_state = deepcopy(self.state_dict())
no_improvement = 0
else:
epoch_s = 'Epoch {:03d}'.format(self.epoch)
no_improvement += 1
t_out = time.time() - self.t_train
t_s = time_to_string(t_out)
if verbose:
print('\033[K', end='', flush=True)
final_s = ' | '.join([epoch_s, tr_loss_s, loss_s] +
losses_val_s + [t_s])
print(final_s)
if self.log_file is not None:
self.log_file.writerow([
'Epoch {:03d}'.format(self.epoch), '{:7.4f}'.format(
loss_tr), '{:7.4f}'.format(loss_val)
] + mid_tr.tolist() + mid_val.tolist() + [t_s])
self.epoch_update(epochs)
if no_improvement == patience:
break
self.epoch = best_e
self.load_state_dict(self.best_state)
t_end = time.time() - t_start
t_end_s = time_to_string(t_end)
if verbose:
print('Training finished in {:} epochs ({:}) '
'with minimum validation loss = {:f} '
'(epoch {:03d})'.format(self.epoch + 1, t_end_s,
self.best_loss_val, best_e))
def on_bt_b_clicked(self, widget):
val = self.timeline.get_value()
self.timeline.setB(float('%.2f'%val))
b = self.timeline.getB()
self.label_B.set_text(utils.time_to_string(b))
self.time_mark = "B"
def log_config(self, f):
desc = str(self.cfg)
desc += "Start Time: " + time_to_string(self.start_time) + "\n"
desc += "End Time: " + time_to_string(self.end_time) + "\n"
f.write(desc)
def acUpdate(deltaT):
# TIMERS
global timer0, timer1, timer2
# VARIABLES
global totalDrivers
global drivers
global fastest_lap
global race_started, replay_started, quali_started
global qualify_session_time
global replay_file
global replay_data
# Widgets
global leaderboardWindow, driverWidget, driverComparisonWidget
# LABELS
global leaderboard
global lapCountTimerLabel, leaderboardBaseLabel, leaderboardInfoBackgroundLabel, leaderboardBackgroundLabel
global flagLabel
# ============================
# UPDATE TIMERS
timer0 += deltaT
timer1 += deltaT
timer2 += deltaT
# ============================
# ================================================================
# RACES
# ================================================================
if info.graphics.session == 2:
# 10 times per second
if timer2 > 0.1:
timer2 = 0
# =============================
# SAVE SPLIT TIMES
for d in drivers:
if ac.isConnected(d.id) == 0: continue
current_split = d.get_split_id(ac.getCarState(d.id, acsys.CS.NormalizedSplinePosition))
if d.current_split != current_split: # save split time at each split of the track
d.split_times[current_split-1] = time.time()
d.current_split = current_split
# 3 times per second
if timer1 > 0.3:
# CHECK RACE RESTART
if race_started and info.graphics.completedLaps == 0 and info.graphics.iCurrentTime == 0:
race_started = False
# CHECK RACE START
if not race_started and info.graphics.iCurrentTime > 0:
# RESET THINGS
fastest_lap = MAX_LAP_TIME
LeaderboardRow.FASTEST_LAP_ID = -1
for row in leaderboard: # clean the fastest lap marker
row.mark_fastest_lap()
# in case we are coming from a qualify
ac.setFontColor(lapCountTimerLabel, 0.86, 0.86, 0.86, 1)
ac.setBackgroundTexture(leaderboardBaseLabel, FC.LEADERBOARD_BASE_RACE)
ac.setVisible(lapCountTimerLabel, 1)
race_started = True
quali_started = False
for d in drivers:
d.starting_position = ac.getCarLeaderboardPosition(d.id)
d.tyre = ac.getCarTyreCompound(d.id)
d.pits = 0
replay_file = setup_replay_file(drivers, info.graphics.numberOfLaps) # save starting info on the drivers
# ============================
# POSITION UPDATE
for i in range(totalDrivers):
pos = ac.getCarRealTimeLeaderboardPosition(i)
connected = ac.isConnected(i)
if connected == 0: # mark unconnected drivers
leaderboard[pos].mark_out()
drivers[i].out = True
else:
leaderboard[pos].mark_in()
drivers[i].out = False
leaderboard[pos].update_name(i)
# OVERTAKE
if pos != drivers[i].position: # there was an overtake
drivers[i].timer = FC.OVERTAKE_POSITION_LABEL_TIMER # set timer
if pos < drivers[i].position:
leaderboard[pos].mark_green_position()
elif pos > drivers[i].position:
leaderboard[pos].mark_red_position()
elif drivers[i].timer <= 0:
leaderboard[pos].mark_white_position()
else:
drivers[i].timer -= timer1
drivers[i].position = pos
# END OVERTAKE
# ============================
# FASTEST LAP BANNER TIMER
if fastest_lap_banner.timer > 0:
fastest_lap_banner.timer -= timer1
fastest_lap_banner.hide()
# ============================
# FLAGS
if info.graphics.flag == 1:
ac.setBackgroundTexture(flagLabel, FC.BLUE_FLAG)
ac.setVisible(flagLabel, 1)
elif info.graphics.flag == 2:
ac.setBackgroundTexture(flagLabel, FC.YELLOW_FLAG)
ac.setVisible(flagLabel, 1)
elif info.graphics.flag == 5:
ac.setBackgroundTexture(flagLabel, FC.CHECKERED_FLAG)
ac.setVisible(flagLabel, 1)
elif info.graphics.flag == 0:
ac.setVisible(flagLabel, 0)
timer1 = 0
# Once per second
if timer0 > 1:
timer0 = 0
ac.setBackgroundOpacity(leaderboardWindow, 0)
ac.setBackgroundOpacity(driverWidget.window, 0)
ac.setBackgroundOpacity(driverComparisonWidget.window, 0)
ac.setBackgroundOpacity(fastest_lap_banner.window, 0)
# ============================
# SERVER LAP
for i in range(totalDrivers):
drivers[i].current_lap = ac.getCarState(i, acsys.CS.LapCount)
lc = max((drivers[i].current_lap for i in range(totalDrivers))) + 1
if lc >= info.graphics.numberOfLaps:
ac.setVisible(lapCountTimerLabel, 0)
ac.setBackgroundTexture(leaderboardBaseLabel, FC.LEADERBOARD_FINAL_LAP)
else:
ac.setText(lapCountTimerLabel, "%d / %d" % (lc, info.graphics.numberOfLaps))
# ===========================
# CALCULATE TIME DIFERENCES
dPosition = sorted(drivers, key=lambda x: x.position)
if LeaderboardRow.update_type == INFO_TYPE.GAPS:
for i in range(1, len(dPosition)):
driver_ahead, driver = dPosition[i-1], dPosition[i]
timeDiff = driver.split_times[driver.current_split - 1] - driver_ahead.split_times[driver.current_split - 1]
if timeDiff < 0: continue # ignore these times, happens on overtakes
if driver.position > totalDrivers: continue # might try to update before it is possible
driver.timeDiff = timeDiff
if timeDiff > 60:
leaderboard[driver.position].update_time("+1 MIN")
else:
leaderboard[driver.position].update_time("+" + time_to_string(timeDiff*1000))
leaderboard[0].update_time("Interval") # Force it
elif LeaderboardRow.update_type == INFO_TYPE.POSITIONS:
for d in dPosition:
posDiff = d.starting_position - d.position - 1
leaderboard[d.position].update_positions(posDiff)
# ============================
# MARK FASTEST LAP
if lc > FC.FASTEST_LAP_STARTING_LAP:
for d in drivers:
lap = ac.getCarState(d.id, acsys.CS.BestLap)
if lap != 0 and lap < fastest_lap:
fastest_lap = lap
fastest_lap_banner.show(lap, ac.getDriverName(d.id))
LeaderboardRow.FASTEST_LAP_ID = d.id
if replay_file:
write_fastest_lap(replay_file, info.graphics.completedLaps, info.graphics.iCurrentTime, d, fastest_lap)
for row in leaderboard:
row.mark_fastest_lap()
# ============================
# PITS MARKER
for row in leaderboard:
if ac.isCarInPitline(row.driverId) == 1:
row.mark_enter_pits()
drivers[row.driverId].tyre = ac.getCarTyreCompound(row.driverId) # maybe will change tyre
else:
row.mark_left_pits()
if time.time() - drivers[row.driverId].pit_time > 20 and ac.isCarInPit(row.driverId):
drivers[row.driverId].pits += 1
drivers[row.driverId].pit_time = time.time()
# ============================
# CHANGE CAR FOCUS AND DRIVER WIDGET
if race_started:
id = ac.getFocusedCar()
if drivers[id].position <= totalDrivers: # in case it wasnt updated yet
driverWidget.show(id, drivers[id].position, drivers[id].starting_position, drivers[id].tyre, drivers[id].pits)
if drivers[id].position == 0:
driverComparisonWidget.hide()
else:
for d in drivers: # find driver ahead
if d.position == drivers[id].position - 1:
driverComparisonWidget.show(d.id, d.position, id, drivers[id].position, drivers[id].timeDiff*1000)
break
else:
driverWidget.hide()
driverComparisonWidget.hide()
# ========================================================
# SAVE DRIVER STATUS IN A FILE TO LOAD ON REPLAY
if replay_file:
write_driver_info(replay_file, info.graphics.completedLaps, info.graphics.iCurrentTime, drivers)
# ========================================================
# ================================================================
# QUALIFY / PRACTICE
# ================================================================
elif info.graphics.session == 1 or (info.graphics.session == 0 and info.graphics.status != 1):
# 3 times per second
if timer1 > 0.3:
# =============================================
# QUALIFY RESTART
if quali_started and qualify_session_time - info.graphics.sessionTimeLeft < 1:
quali_started = False
# =============================================
# QUALIFY START
if not quali_started:
ac.setBackgroundTexture(leaderboardBaseLabel, FC.LEADERBOARD_BASE_QUALI)
if (info.graphics.session == 0 and info.graphics.status != 0):
ac.setBackgroundTexture(leaderboardBaseLabel, FC.LEADERBOARD_BASE_PRACTICE)
ac.setFontColor(lapCountTimerLabel, 0.86, 0.86, 0.86, 1)
qualify_session_time = info.graphics.sessionTimeLeft
fastest_lap = MAX_LAP_TIME
LeaderboardRow.FASTEST_LAP_ID = -1
quali_started = True
race_started = False
# =============================================
# SAVE BEST LAPS FOR EACH DRIVER
for i in range(totalDrivers):
lap = ac.getCarState(i, acsys.CS.BestLap)
if lap != 0:
drivers[i].best_lap = lap
if lap != 0 and lap < fastest_lap:
fastest_lap = lap
fastest_lap_banner.show(lap, ac.getDriverName(i))
# MARK IN/OUT DRIVERS
connected = ac.isConnected(i)
if connected == 0: # mark unconnected drivers
drivers[i].out = True
else:
drivers[i].out = False
# =============================================
# MANAGE LEADERBOARD
# Sorting: sort drivers by this order 1. in or out drivers, 2. best lap, 3. driver id
dPosition = sorted(drivers, key=lambda d: (d.out, d.best_lap, d.id))
for i in range(totalDrivers):
if dPosition[i].out:
leaderboard[i].mark_out()
else:
leaderboard[i].mark_in()
leaderboard[i].update_name(dPosition[i].id)
if dPosition[i].best_lap == MAX_LAP_TIME:
leaderboard[i].update_time("NO TIME")
elif i == 0:
leaderboard[i].update_time(time_to_string(dPosition[i].best_lap))
else:
timeDiff = dPosition[i].best_lap - dPosition[0].best_lap
if timeDiff > 60000:
leaderboard[i].update_time("+1 MIN")
else:
leaderboard[i].update_time("+" + time_to_string(timeDiff))
# OVERTAKES
if i != dPosition[i].position: # there was an overtake on this driver
dPosition[i].timer = FC.OVERTAKE_POSITION_LABEL_TIMER
if i < dPosition[i].position:
leaderboard[i].mark_green_position()
elif i > dPosition[i].position:
leaderboard[i].mark_red_position()
elif dPosition[i].timer <= 0:
leaderboard[i].mark_white_position()
else:
dPosition[i].timer -= timer1
dPosition[i].position = i
# END OVERTAKE
# ============================
# FASTEST LAP BANNER TIMER
if fastest_lap_banner.timer > 0:
fastest_lap_banner.timer -= timer1
fastest_lap_banner.hide()
timer1 = 0
# Once per second
if timer0 > 1:
timer0 = 0
ac.setBackgroundOpacity(leaderboardWindow, 0)
ac.setBackgroundOpacity(driverWidget.window, 0)
ac.setBackgroundOpacity(driverComparisonWidget.window, 0)
ac.setBackgroundOpacity(fastest_lap_banner.window, 0)
if quali_started:
if info.graphics.sessionTimeLeft < 0:
ac.setText(lapCountTimerLabel, "0:00")
else:
timeText = time_to_string(info.graphics.sessionTimeLeft)[:-4]
ac.setText(lapCountTimerLabel, "0:00"[:-len(timeText)] + timeText)
if info.graphics.sessionTimeLeft < qualify_session_time / 5:
ac.setFontColor(lapCountTimerLabel, 1,0,0,1)
driverWidget.hide()
driverComparisonWidget.hide()
# ================================================================
# REPLAYS
# ================================================================
elif info.graphics.status == 1:
# three times per second
if timer1 > 0.3:
if not replay_started:
if info.graphics.iCurrentTime > 0:
replay_data = load_replay_file(drivers)
replay_started = True
# ============================
# FASTEST LAP BANNER TIMER
if fastest_lap_banner.timer > 0:
fastest_lap_banner.timer -= timer1
fastest_lap_banner.hide()
# ============================
# GET DATA FOR THIS UPDATE
if replay_data:
new_positions = lookup_data(info.graphics.completedLaps, info.graphics.iCurrentTime, replay_data, drivers)
# ============================
# POSITION UPDATE
for i in range(totalDrivers):
pos = new_positions[i]
if drivers[i].out: # mark unconnected drivers
leaderboard[pos].mark_out()
else:
leaderboard[pos].mark_in()
leaderboard[pos].update_name(i)
# OVERTAKE
if pos != drivers[i].position: # there was an overtake
drivers[i].timer = FC.OVERTAKE_POSITION_LABEL_TIMER # set timer
if pos < drivers[i].position:
leaderboard[pos].mark_green_position()
elif pos > drivers[i].position:
leaderboard[pos].mark_red_position()
elif drivers[i].timer <= 0:
leaderboard[pos].mark_white_position()
else:
drivers[i].timer -= timer1
drivers[i].position = pos
# END OVERTAKE
timer1 = 0
# Once per second
if timer0 > 1:
timer0 = 0
ac.setBackgroundOpacity(leaderboardWindow, 0)
ac.setBackgroundOpacity(driverWidget.window, 0)
ac.setBackgroundOpacity(driverComparisonWidget.window, 0)
ac.setBackgroundOpacity(fastest_lap_banner.window, 0)
# ============================
# GET FASTEST LAP UPDATE
if replay_data:
fl_data = lookup_fastest_lap(info.graphics.completedLaps, info.graphics.iCurrentTime, replay_data)
if fl_data:
display_time = FC.FASTEST_LAP_DISPLAY_TIME - (info.graphics.iCurrentTime - fl_data[0]) / 1000
fastest_lap_banner.show(fl_data[2], ac.getDriverName(fl_data[1]), timer=display_time) # display only for the time left
# ============================
# SERVER LAP
if replay_data:
lc = max((drivers[i].current_lap for i in range(totalDrivers))) + 1
if lc >= replay_data['nLaps']:
ac.setVisible(lapCountTimerLabel, 0)
ac.setBackgroundTexture(leaderboardBaseLabel, FC.LEADERBOARD_FINAL_LAP)
else:
ac.setText(lapCountTimerLabel, "%d / %d" % (lc, replay_data['nLaps']))
ac.setVisible(lapCountTimerLabel, 1)
ac.setBackgroundTexture(leaderboardBaseLabel, FC.LEADERBOARD_BASE_RACE)
# ============================
# PITS MARKER
for row in leaderboard:
if ac.isCarInPitline(row.driverId) == 1:
row.mark_enter_pits()
else:
row.mark_left_pits()
# ============================
# DRIVER WIDGET UPDATE
if replay_started:
id = ac.getFocusedCar()
if drivers[id].position <= totalDrivers: # in case it wasnt updated yet
driverWidget.show(id, drivers[id].position, drivers[id].starting_position, drivers[id].tyre, drivers[id].pits)
if drivers[id].position == 0:
driverComparisonWidget.hide()
else:
for d in drivers: # find driver ahead
if d.position == drivers[id].position - 1:
driverComparisonWidget.show(d.id, d.position, id, drivers[id].position, drivers[id].timeDiff*1000)
break
else:
driverWidget.hide()
driverComparisonWidget.hide()
# ============================
# UPDATE TIMES
if replay_data:
for row in leaderboard:
if LeaderboardRow.update_type == INFO_TYPE.GAPS:
row.update_time("+" + time_to_string(drivers[row.driverId].timeDiff*1000))
if row.row == 0:
row.update_time("Interval") # Force it
elif LeaderboardRow.update_type == INFO_TYPE.POSITIONS:
posDiff = drivers[row.driverId].starting_position - drivers[row.driverId].position - 1
row.update_positions(posDiff)
def utilization_chart(names,
values,
title="",
xlabel="",
ylabel="",
idles=None):
if not names or not values:
return _empty_chart(title, xlabel, ylabel)
figure = mpl_Figure()
canvas = mpl_FigureCanvas(figure)
figure.set_canvas(canvas)
# TODO: Change it to TimeChart
ax = figure.add_subplot(111, projection=BasicChart.name)
ywidth = 2
yticks = []
if idles is not None:
for i, lidle in enumerate(idles):
y = ((i + 1) * ywidth) + (i + 1)
ax.broken_barh(lidle, (y, ywidth),
edgecolor='face',
facecolor='#EAA769')
for i, ldata in enumerate(values):
y = (ywidth + 1) * (i + 1)
yticks.append(y + ywidth / 2)
ax.broken_barh(ldata, (y, ywidth), edgecolor='face', facecolor='green')
ax.set_yticks(yticks)
ax.set_yticklabels(names)
for label in ax.xaxis.get_ticklabels():
label.set_rotation(-35)
label.set_horizontalalignment('left')
for i, label in enumerate(ax.yaxis.get_ticklabels()):
# add 3 white space on the begining of name
names[i] = " %s" % names[i]
label.set_horizontalalignment("left")
label.set_verticalalignment('center')
p = mpl_Rectangle((0, 0), 1, 1, edgecolor='green', fc='green', alpha=0.75)
if idles is not None:
idle_leg = mpl_Rectangle((0, 0),
1,
1,
edgecolor='#eaa769',
fc='#eaa769',
alpha=0.75)
ax.plegend = ax.legend([p, idle_leg], ["Running", "Idle"],
loc="upper left",
fancybox=True,
shadow=True)
else:
ax.plegend = ax.legend([p], ["Running"],
loc="upper left",
fancybox=True,
shadow=True)
ax.xaxis.grid(True, linestyle="-", which='major', color='black', alpha=0.7)
ax.xaxis.set_major_formatter(
mpl_FuncFormatter(lambda time, pos: utils.time_to_string(time)[:-7]))
ax.set_xlim(xmin=0)
ax.get_figure().tight_layout()
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
# resize figure
w, h = figure.get_size_inches()
figure.set_size_inches(w, len(values) * 0.4)
return ChartWidget(figure, ylock=True)