def test_with_scope_extern():
# Unfortunetly the notebooks are really the best way to test if
# things are working.
display_objects = []
plot = PlotLearningCurve(display_fn=display_replacer(display_objects))
with plot:
for i in range(1, 10):
plot.append(i, {'loss': i * 10 + 1, 'val_loss': i * 10})
plot.draw()
assert_equal(len(display_objects), 6)
def test_crude_sanity_check():
# Unfortunetly the notebooks are really the best way to test if
# things are working.
display_objects = []
plot = PlotLearningCurve(display_fn=display_replacer(display_objects))
plot.append(0, {'loss': {'train': 1, 'validation': 0}})
plot.draw()
for i in range(1, 10):
plot.append(i, {'loss': {'train': i * 10 + 1, 'validation': i * 10}})
plot.draw()
plot.finalize()
assert_equal(len(display_objects), 6)
def monitor(self, timeout=MAX_DELAY_TIME):
if 'epochs' in self.script_params:
self.epochs = self.script_params['epochs']
# for end condition
epochs = int(self.epochs)
if self.validation:
plot = PlotLearningCurve(mappings={
'loss': {
'line': 'train',
'facet': 'loss'
},
'val_loss': {
'line': 'validation',
'facet': 'loss'
},
'acc': {
'line': 'train',
'facet': 'acc'
},
'val_acc': {
'line': 'validation',
'facet': 'acc'
}
},
facet_config={
'loss': {
'name': 'Loss',
'limit': [None, None],
'scale': 'linear'
},
'acc': {
'name': 'Accuracy',
'limit': [None, 1],
'scale': 'linear'
}
},
xaxis_config={
'name': 'Epoch',
'limit': [0, epochs]
})
else:
plot = PlotLearningCurve(mappings={
'loss': {
'line': 'train',
'facet': 'loss'
},
'acc': {
'line': 'train',
'facet': 'acc'
}
},
facet_config={
'loss': {
'name': 'Loss',
'limit': [None, None],
'scale': 'linear'
},
'acc': {
'name': 'Accuracy',
'limit': [0, 1],
'scale': 'linear'
}
},
xaxis_config={
'name': 'Epoch',
'limit': [0, epochs]
})
# log monitoring loop
delay_time = 0.
try:
with open(self.this_job_path + "/epoch.log", "r") as f:
while True:
where = f.tell()
line = f.readline().strip()
if not line:
sleep(0.1)
delay_time += 0.1
f.seek(where)
if timeout is not None:
if delay_time > timeout:
print("Delay has been exceeded.")
break
else:
if delay_time > MAX_DELAY_TIME:
print("Delay has been exceeded.")
break
else:
delay_time = 0. # reset delay time
# print(line) # already has newline
strlist = line.split(',')
phase = strlist[0].strip()
epoch = strlist[1].strip()
loss = strlist[2].strip()
acc = strlist[3].strip()
print("Phase: {}, Epoch: {}, Loss: {}, Acc: {}".format(
phase, epoch, loss, acc))
# append and update
if phase == "Train":
plot.append(epoch, {'loss': loss, 'acc': acc})
plot.draw()
else:
plot.append(epoch, {
'val_loss': loss,
'val_acc': acc
})
plot.draw()
# End Condition (If the Last Epoch Finished, Terminate the Loop)
if self.validation:
if phase == "Validation":
if int(epoch) == epochs:
break
else:
if int(epoch) == epochs:
break
except:
print(
"Even one epoch has not been completed.\nPlease execute the command again after a while."
)
