def parse_runs(tag): template_root = os.path.abspath(os.path.join(app.root_path, '..')) logroot = os.path.join(template_root, 'logs') logpath = os.path.join(logroot, tag, 'meta.json') if not os.path.isfile(logpath): return [] with open(logpath, 'r') as f: meta = json.load(f) runs = [] for line in meta: run = { 'command': line['command'], 'starttime': misc_utils.get_time_str(line['starttime'], fmt='%Y-%m-%d %H:%M:%S') } if 'finishtime' in line: runtime = int(line['finishtime']) - int(line['starttime']) else: runtime = int(misc_utils.get_time_stamp()) - int(line['starttime']) run['runtime'] = misc_utils.format_time(runtime) runs.append(run) return runs
def load_meta(new=False): path = os.path.join(opt.log_dir, opt.tag, 'meta.json') if os.path.isfile(path): with open(path, 'r') as f: meta = json.load(f) else: meta = [] if new: new_meta = { 'command': get_command_run(), 'starttime': utils.get_time_stamp(), 'best_acc': 0., 'gpu': get_gpu_id(), 'opt': opt.__dict__, } meta.append(new_meta) return meta
def parse_meta_json(metapath): with open(metapath, 'r') as f: meta = json.load(f) state = None for line in meta: if 'finishtime' not in line: if line['gpu'] == '-1': state = 'cpu' else: state = 'cuda:' + line['gpu'] break else: state = 'finished' model = meta[0]['opt']['model'] acc = 0. runtime = 0. for line in meta: acc = max(acc, line['best_acc']) if 'finishtime' in line: runtime += int(line['finishtime']) - int(line['starttime']) else: runtime += int(misc_utils.get_time_stamp()) - int( line['starttime']) runtime = misc_utils.format_time(runtime) if 'remarks' in meta[0]: remarks = meta[0]['remarks'] else: remarks = '' return { 'state': state, 'model': model, 'acc': acc, 'runtime': runtime, 'remarks': remarks }
epoch, writer, logger, data_name='val') model.train() if scheduler is not None: scheduler.step() # 保存结束信息 if opt.tag != 'cache': with open('run_log.txt', 'a') as f: f.writelines(' Accuracy:' + eval_result + '\n') meta = load_meta() meta[-1]['finishtime'] = utils.get_time_stamp() save_meta(meta) except Exception as e: # if not opt.debug: # debug模式不会发短信 12是短信模板字数限制 # send_notification([opt.tag[:12], str(e)[:12]], template='error') if opt.tag != 'cache': with open('run_log.txt', 'a') as f: f.writelines(' Error: ' + str(e)[:120] + '\n') meta = load_meta() meta[-1]['finishtime'] = utils.get_time_stamp() save_meta(meta) # print(e)
type=str, default='cache', help='folder name to clear') parser.add_argument('--rm', action='store_true', help='debug mode') return parser.parse_args() opt = parse_args() paths = ['checkpoints', 'logs', 'results'] if opt.rm: for path in paths: p = os.path.join(path, opt.tag) if os.path.isdir(p): command = 'rm -r ' + p print(command) os.system(command) else: for path in paths: tmp = os.path.join('_.trash', utils.get_time_stamp(), path) utils.try_make_dir(tmp) p = os.path.join(path, opt.tag) if os.path.isdir(p): command = 'mv %s %s' % (p, tmp) print(command) os.system(command) utils.color_print("Directory '%s' cleared." % opt.tag, 1)