def extract(self, log_file):
lines = read_lines(log_file)
epoch_lines = []
train_lines = []
valid_lines = []
for l in lines:
if l.startswith("Epoch"):
epoch_lines.append(l)
elif ' train ' in l:
train_lines.append(l)
elif ' valid ' in l:
valid_lines.append(l)
train_ends, train_losses, train_accs = zip(
*map(parse_metric_line(p=self.patterns['train']), train_lines))
valid_ends, valid_losses, valid_accs = zip(
*map(parse_metric_line(p=self.patterns['valid']), valid_lines))
train_losses, train_accs, valid_losses, valid_accs = [
np.array(x)
for x in [train_losses, train_accs, valid_losses, valid_accs]
]
train_results = pd.DataFrame({
"end": train_ends,
"loss": train_losses,
"acc": train_accs,
})
valid_results = pd.DataFrame({
"end": valid_ends,
"loss": valid_losses,
"acc": valid_accs,
})
return train_results, valid_results
def parse_log(fp):
lines = read_lines(fp)
train_start = lines[0][:8]
train_lines = []
valid_lines = []
cur_epoch_lines = []
for l in lines:
if l.startswith("Epoch"):
if len(cur_epoch_lines) == 1:
train_lines.append(cur_epoch_lines[0])
valid_lines.append(None)
elif len(cur_epoch_lines) == 2:
train_lines.append(cur_epoch_lines[0])
valid_lines.append(cur_epoch_lines[1])
cur_epoch_lines = []
elif ' train ' in l or 'valid' in l:
cur_epoch_lines.append(l)
if len(cur_epoch_lines) == 1:
train_lines.append(cur_epoch_lines[0])
valid_lines.append(None)
elif len(cur_epoch_lines) == 2:
train_lines.append(cur_epoch_lines[0])
valid_lines.append(cur_epoch_lines[1])
train_metrics = parse_metric_lines(train_lines)
valid_metrics = parse_metric_lines(valid_lines)
train_ends, valid_ends = train_metrics['time'], valid_metrics['time']
for i in range(len(train_ends)):
if valid_ends[i] is None:
valid_ends[i] = train_ends[i]
return train_start, train_metrics, valid_metrics
def parse_log(fp):
lines = read_lines(fp)
train_start = lines[0][:8]
epoch_lines = []
train_lines = []
valid_lines = []
for l in lines:
if l.startswith("Epoch"):
epoch_lines.append(l)
elif ' train ' in l:
train_lines.append(l)
elif ' valid ' in l:
valid_lines.append(l)
total_epochs = int(epoch_lines[0].split('/')[1])
train_p = re.compile(f"""({dt_p}) train - loss: ({m_p}), acc: ({m_p})""")
valid_p = re.compile(
f"""({dt_p}) valid - loss: ({m_p}), acc: ({m_p}), acc5: ({m_p})""")
train_ends, train_losses, train_accs = zip(
*map(parse_metric_line(p=train_p), train_lines))
valid_ends, valid_losses, valid_accs, valid_acc5s = zip(
*map(parse_metric_line(p=valid_p), valid_lines))
train_losses, valid_accs, valid_acc5s = [
np.array(x) for x in [train_losses, valid_accs, valid_acc5s]
]
return total_epochs, train_start, train_losses, valid_ends, valid_accs, valid_acc5s
def parse_log(fp):
lines = read_lines(fp)
train_start = lines[0][:8]
train_lines = []
valid_lines = []
for l in lines:
if l.startswith("Epoch"):
continue
elif ' train ' in l:
train_lines.append(l)
elif ' valid ' in l:
valid_lines.append(l)
train_metrics = parse_metric_lines(train_lines)
valid_metrics = parse_metric_lines(valid_lines)
return train_start, train_metrics, valid_metrics
def extract(fp):
lines = read_lines(fp)
epoch_lines = []
train_lines = []
valid_lines = []
for l in lines:
if 'Epoch' in l:
epoch_lines.append(l)
elif 'train' in l:
train_lines.append(l)
elif 'valid' in l:
valid_lines.append(l)
lrs = lmap(get_lr, epoch_lines)
train_losses, train_accs = zip(*map(extract_train, train_lines))
valid_losses, valid_accs = zip(*map(extract_valid, valid_lines))
lrs, train_losses, train_accs, valid_losses, valid_accs = [
np.array(x)
for x in [lrs, train_losses, train_accs, valid_losses, valid_accs]
]
return lrs, train_losses, train_accs, valid_losses, valid_accs