def parse_runs(tag):
template_root = os.path.abspath(os.path.join(app.root_path, '..'))
logroot = os.path.join(template_root, 'logs')
logpath = os.path.join(logroot, tag, 'meta.json')
if not os.path.isfile(logpath):
return []
with open(logpath, 'r') as f:
meta = json.load(f)
runs = []
for line in meta:
run = {
'command':
line['command'],
'starttime':
misc_utils.get_time_str(line['starttime'], fmt='%Y-%m-%d %H:%M:%S')
}
if 'finishtime' in line:
runtime = int(line['finishtime']) - int(line['starttime'])
else:
runtime = int(misc_utils.get_time_stamp()) - int(line['starttime'])
run['runtime'] = misc_utils.format_time(runtime)
runs.append(run)
return runs
def load_meta(new=False):
path = os.path.join(opt.log_dir, opt.tag, 'meta.json')
if os.path.isfile(path):
with open(path, 'r') as f:
meta = json.load(f)
else:
meta = []
if new:
new_meta = {
'command': get_command_run(),
'starttime': utils.get_time_stamp(),
'best_acc': 0.,
'gpu': get_gpu_id(),
'opt': opt.__dict__,
}
meta.append(new_meta)
return meta
def parse_meta_json(metapath):
with open(metapath, 'r') as f:
meta = json.load(f)
state = None
for line in meta:
if 'finishtime' not in line:
if line['gpu'] == '-1':
state = 'cpu'
else:
state = 'cuda:' + line['gpu']
break
else:
state = 'finished'
model = meta[0]['opt']['model']
acc = 0.
runtime = 0.
for line in meta:
acc = max(acc, line['best_acc'])
if 'finishtime' in line:
runtime += int(line['finishtime']) - int(line['starttime'])
else:
runtime += int(misc_utils.get_time_stamp()) - int(
line['starttime'])
runtime = misc_utils.format_time(runtime)
if 'remarks' in meta[0]:
remarks = meta[0]['remarks']
else:
remarks = ''
return {
'state': state,
'model': model,
'acc': acc,
'runtime': runtime,
'remarks': remarks
}
epoch,
writer,
logger,
data_name='val')
model.train()
if scheduler is not None:
scheduler.step()
# 保存结束信息
if opt.tag != 'cache':
with open('run_log.txt', 'a') as f:
f.writelines(' Accuracy:' + eval_result + '\n')
meta = load_meta()
meta[-1]['finishtime'] = utils.get_time_stamp()
save_meta(meta)
except Exception as e:
# if not opt.debug: # debug模式不会发短信 12是短信模板字数限制
# send_notification([opt.tag[:12], str(e)[:12]], template='error')
if opt.tag != 'cache':
with open('run_log.txt', 'a') as f:
f.writelines(' Error: ' + str(e)[:120] + '\n')
meta = load_meta()
meta[-1]['finishtime'] = utils.get_time_stamp()
save_meta(meta)
# print(e)
type=str,
default='cache',
help='folder name to clear')
parser.add_argument('--rm', action='store_true', help='debug mode')
return parser.parse_args()
opt = parse_args()
paths = ['checkpoints', 'logs', 'results']
if opt.rm:
for path in paths:
p = os.path.join(path, opt.tag)
if os.path.isdir(p):
command = 'rm -r ' + p
print(command)
os.system(command)
else:
for path in paths:
tmp = os.path.join('_.trash', utils.get_time_stamp(), path)
utils.try_make_dir(tmp)
p = os.path.join(path, opt.tag)
if os.path.isdir(p):
command = 'mv %s %s' % (p, tmp)
print(command)
os.system(command)
utils.color_print("Directory '%s' cleared." % opt.tag, 1)