def find_checkpoint(checkpoint_str):
# find checkpoint
steps = 0
if checkpoint_str is not None:
if ':' in checkpoint_str:
prefix, steps = checkpoint_str.split(':')
else:
prefix = checkpoint_str
steps = None
log_file, run_id = path.find_log(prefix)
if steps is None:
checkpoint, steps = path.find_checkpoints(run_id)[-1]
else:
checkpoints = path.find_checkpoints(run_id)
try:
checkpoint, steps = next(
filter(lambda t: t[1] == steps, checkpoints))
except StopIteration:
print('The steps not found in checkpoints', steps, checkpoints)
sys.stdout.flush()
raise StopIteration
steps = int(steps)
if args.clear_steps:
steps = 0
else:
_, exp_info = path.read_log(log_file)
exp_info = exp_info[-1]
for k in args.__dict__:
if k in exp_info and k in ('tag', ):
setattr(args, k, eval(exp_info[k]))
print('{}={}, '.format(k, exp_info[k]), end='')
print()
sys.stdout.flush()
return checkpoint, steps
log_file, run_id = path.find_log(prefix)
if steps is None:
checkpoint, steps = path.find_checkpoints(run_id)[-1]
else:
checkpoints = path.find_checkpoints(run_id)
try:
checkpoint, steps = next(filter(lambda t : t[1] == steps, checkpoints))
except StopIteration:
print('The steps not found in checkpoints', steps, checkpoints)
sys.stdout.flush()
raise StopIteration
steps = int(steps)
if args.clear_steps:
steps = 0
else:
_, exp_info = path.read_log(log_file)
exp_info = exp_info[-1]
for k in args.__dict__:
if k in exp_info and k in ('tag',):
setattr(args, k, eval(exp_info[k]))
print('{}={}, '.format(k, exp_info[k]), end='')
print()
sys.stdout.flush()
# generate id
if args.checkpoint is None or args.clear_steps:
uid = (socket.gethostname() + logger.FileLog._localtime().strftime('%b%d-%H%M') + args.gpu_device)
tag = hashlib.sha224(uid.encode()).hexdigest()[:3]
run_id = tag + logger.FileLog._localtime().strftime('%b%d-%H%M')
# initiate
from network import get_pipeline