def process_dent(self, nwo, ext, library_candidates) -> Tuple[List[Dict[str, Any]], List[Tuple[str, str]]]:
# Process dependents (applications) to get function calls
dents = []
edges = []
_, nwo = remap_nwo(nwo)
if nwo is None:
return dents, edges
tmp_dir = download(nwo)
files = walk(tmp_dir, ext)
sha = None
for f in files:
context_and_calls = self.get_context_and_function_calls(f)
if context_and_calls is None:
continue
if sha is None:
sha = get_sha(tmp_dir, nwo)
nwo, path, context, calls = context_and_calls
libraries = []
for cxt in context:
if type(cxt) == dict:
libraries.extend([v.split('.')[0] for v in cxt.values()])
elif type(cxt) == list:
libraries.extend(cxt)
match_scopes = {}
for cxt in set(libraries):
if cxt in library_candidates:
match_scopes[cxt] = library_candidates[cxt]
for call in calls:
for depended_library_name, dependend_library_functions in match_scopes.items():
for depended_library_function in dependend_library_functions:
# Other potential filters: len(call['identifier']) > 6 or len(call['identifier'].split('_')) > 1
if (call['identifier'] not in self.language_parser.STOPWORDS and
((depended_library_function['identifier'].split('.')[-1] == '__init__' and
call['identifier'] == depended_library_function['identifier'].split('.')[0]) or
((len(call['identifier']) > 9 or
(not call['identifier'].startswith('_') and len(call['identifier'].split('_')) > 1)) and
call['identifier'] == depended_library_function['identifier'])
)):
dent = {
'nwo': nwo,
'sha': sha,
'path': path,
'language': self.language,
'identifier': call['identifier'],
'argument_list': call['argument_list'],
'url': 'https://github.com/{}/blob/{}/{}#L{}-L{}'.format(nwo, sha, path,
call['start_point'][0] + 1,
call['end_point'][0] + 1)
}
dents.append(dent)
edges.append((dent['url'], depended_library_function['url']))
return dents, edges
async def game_new():
"""
"""
game_id = get_sha()
code = get_random_code()
logger.debug("New game /game/new: %s || %s" % (game_id, code))
valid_game = await create_game(game_id, code)
if not valid_game:
return make_json_response(code=1, msg='Something went wrong')
else:
return make_json_response(code=0, body={'game_id': game_id})
def process_dee(self, nwo, ext) -> List[Dict[str, Any]]:
# Process dependees (libraries) to get function implementations
indexes = []
_, nwo = remap_nwo(nwo)
if nwo is None:
return indexes
tmp_dir = download(nwo)
files = walk(tmp_dir, ext)
# files = glob.iglob(tmp_dir.name + '/**/*.{}'.format(ext), recursive=True)
sha = None
for f in files:
definitions = self.get_function_definitions(f)
if definitions is None:
continue
if sha is None:
sha = get_sha(tmp_dir, nwo)
nwo, path, functions = definitions
indexes.extend((self.extract_function_data(func, nwo, path, sha) for func in functions if len(func['function_tokens']) > 1))
return indexes
def setup_experiment():
np.random.seed(FLAGS.seed)
if not tf.gfile.Exists(FLAGS.data_dir) or not tf.gfile.IsDirectory(
FLAGS.data_dir):
raise ValueError("Could not find folder %s" % FLAGS.data_dir)
assert FLAGS.batch_size % FLAGS.carlini_batch_size == 0
if not tf.gfile.Exists(FLAGS.load_dir) or not tf.gfile.IsDirectory(
FLAGS.load_dir):
raise ValueError("Could not find folder %s" % FLAGS.load_dir)
FLAGS.working_dir = os.path.join(FLAGS.working_dir, os.path.basename(os.path.normpath(FLAGS.load_dir)))
FLAGS.adv_data_dir = os.path.join(FLAGS.working_dir, FLAGS.adv_data_dir)
FLAGS.samples_dir = os.path.join(FLAGS.working_dir, FLAGS.samples_dir)
FLAGS.git_revision = get_sha()
if tf.gfile.Exists(FLAGS.working_dir):
tf.gfile.DeleteRecursively(FLAGS.working_dir)
tf.gfile.MakeDirs(FLAGS.working_dir)
tf.gfile.MakeDirs(FLAGS.adv_data_dir)
tf.gfile.MakeDirs(FLAGS.samples_dir)
train_params = load_training_params(FLAGS.load_dir)
FLAGS.model = train_params['model']
FLAGS.model_name = train_params['model_name']
FLAGS.activation_fn = train_params['activation_fn']
FLAGS.num_classes = train_params['num_classes']
FLAGS.layer_dims = train_params['layer_dims']
# configure logging
logger = getLogger('tensorflow')
tf.logging.set_verbosity(tf.logging.INFO)
file_hndl = FileHandler(os.path.join(FLAGS.working_dir, 'tensorflow.log'))
file_hndl.setLevel(logging.DEBUG)
logger.addHandler(file_hndl)
# print config
logging.info({k: v.value for k, v in FLAGS._flags().items()})
logging.info("Training params: %s", train_params)
def automated_operators_repository_commit_message(
operator_repository: str,
operator_name: str,
operator_directory: str,
operator_git_tag: str,
debug: bool,
) -> Tuple[int, str, str, str]:
rc, stdout, stderr = get_sha(operator_directory, debug)
if rc != 0:
return (
rc,
"",
"",
f"Failed to get git SHA from '{operator_directory}'" +
f"\nstdout:\n{stdout}\nstderr:\n{stderr}",
)
operator_git_sha = stdout.strip()
operator_repository_url = github_repository_url(operator_repository)
operator_git_tag_url = (
f"{operator_repository_url}/releases/tag/{operator_git_tag}")
operator_git_sha_url = (
f"{operator_repository_url}/commit/{operator_git_sha}")
commit_message_subject = (
f"Release {operator_name} {operator_git_tag} (automated commit).")
commit_message_body = "\n".join([
f"| | |",
f"|-|-|",
f"| Repository | {operator_repository_url} |",
f"| Operator | {operator_name} |",
f"| Git tag | {operator_git_tag_url} |",
f"| Git SHA | {operator_git_sha_url} |",
f"| Date (UTC) | {datetime.utcnow()} |",
])
return 0, commit_message_subject, commit_message_body, ""
def eval_linear(args):
utils.init_distributed_mode(args)
print("git:\n {}\n".format(utils.get_sha()))
print("\n".join("%s: %s" % (k, str(v)) for k, v in sorted(dict(vars(args)).items())))
cudnn.benchmark = True
# ============ preparing data ... ============
train_transform = pth_transforms.Compose([
pth_transforms.RandomResizedCrop(224),
pth_transforms.RandomHorizontalFlip(),
pth_transforms.ToTensor(),
pth_transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
val_transform = pth_transforms.Compose([
pth_transforms.Resize(256, interpolation=3),
pth_transforms.CenterCrop(224),
pth_transforms.ToTensor(),
pth_transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
dataset_train = datasets.ImageFolder(os.path.join(args.data_path, "train"), transform=train_transform)
dataset_val = datasets.ImageFolder(os.path.join(args.data_path, "val"), transform=val_transform)
sampler = torch.utils.data.distributed.DistributedSampler(dataset_train)
train_loader = torch.utils.data.DataLoader(
dataset_train,
sampler=sampler,
batch_size=args.batch_size_per_gpu,
num_workers=args.num_workers,
pin_memory=True,
)
val_loader = torch.utils.data.DataLoader(
dataset_val,
batch_size=args.batch_size_per_gpu,
num_workers=args.num_workers,
pin_memory=True,
)
print(f"Data loaded with {len(dataset_train)} train and {len(dataset_val)} val imgs.")
# ============ building network ... ============
model = vits.__dict__[args.arch](patch_size=args.patch_size, num_classes=0)
model.cuda()
model.eval()
print(f"Model {args.arch} {args.patch_size}x{args.patch_size} built.")
# load weights to evaluate
utils.load_pretrained_weights(model, args.pretrained_weights, args.checkpoint_key, args.arch, args.patch_size)
linear_classifier = LinearClassifier(model.embed_dim * (args.n_last_blocks + int(args.avgpool_patchtokens)), num_labels=args.num_labels)
linear_classifier = linear_classifier.cuda()
linear_classifier = nn.parallel.DistributedDataParallel(linear_classifier, device_ids=[args.gpu])
# set optimizer
optimizer = torch.optim.SGD(
linear_classifier.parameters(),
args.lr * (args.batch_size_per_gpu * utils.get_world_size()) / 256., # linear scaling rule
momentum=0.9,
weight_decay=0, # we do not apply weight decay
)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epochs, eta_min=0)
# Optionally resume from a checkpoint
to_restore = {"epoch": 0, "best_acc": 0.}
utils.restart_from_checkpoint(
os.path.join(args.output_dir, "checkpoint.pth.tar"),
run_variables=to_restore,
state_dict=linear_classifier,
optimizer=optimizer,
scheduler=scheduler,
)
start_epoch = to_restore["epoch"]
best_acc = to_restore["best_acc"]
for epoch in range(start_epoch, args.epochs):
train_loader.sampler.set_epoch(epoch)
train_stats = train(model, linear_classifier, optimizer, train_loader, epoch, args.n_last_blocks, args.avgpool_patchtokens)
scheduler.step()
log_stats = {**{f'train_{k}': v for k, v in train_stats.items()},
'epoch': epoch}
if epoch % args.val_freq == 0 or epoch == args.epochs - 1:
test_stats = validate_network(val_loader, model, linear_classifier, args.n_last_blocks, args.avgpool_patchtokens)
print(f"Accuracy at epoch {epoch} of the network on the {len(dataset_val)} test images: {test_stats['acc1']:.1f}%")
best_acc = max(best_acc, test_stats["acc1"])
print(f'Max accuracy so far: {best_acc:.2f}%')
log_stats = {**{k: v for k, v in log_stats.items()},
**{f'test_{k}': v for k, v in test_stats.items()}}
if utils.is_main_process():
with (Path(args.output_dir) / "log.txt").open("a") as f:
f.write(json.dumps(log_stats) + "\n")
save_dict = {
"epoch": epoch + 1,
"state_dict": linear_classifier.state_dict(),
"optimizer": optimizer.state_dict(),
"scheduler": scheduler.state_dict(),
"best_acc": best_acc,
}
torch.save(save_dict, os.path.join(args.output_dir, "checkpoint.pth.tar"))
print("Training of the supervised linear classifier on frozen features completed.\n"
"Top-1 test accuracy: {acc:.1f}".format(acc=best_acc))
if args.dataset == 'ptb':
train_data, val_data, test_data = PTBSeq2Seq.iters(
batch_size=args.batch_size, device=device)
corpus = train_data.dataset.fields['target'].vocab # includes BOS
ntokens = len(corpus)
model = rvae.RVAE(ntokens, args.emsize, args.nhid, args.z_dim, 1,
args.dropout, args.dropouth, args.dropouti,
args.dropoute, args.wdrop)
if args.cuda and torch.cuda.is_available():
model.cuda()
total_params = sum(x.size()[0] *
x.size()[1] if len(x.size()) > 1 else x.size()[0]
for x in model.parameters())
print("sha: {}".format(get_sha().strip()))
print('args:', args)
print('model total parameters:', total_params)
print('model architecture:')
print(model)
criterion = nn.CrossEntropyLoss()
# Loop over epochs.
args.anneal = 0.0001
lr = args.lr
best_val_loss = []
stored_loss = 100000000
# At any point you can hit Ctrl + C to break out of training early.
try:
help='Number of data loading workers per GPU.')
parser.add_argument("--dist_url",
default="env://",
type=str,
help="""url used to set up
distributed training; see https://pytorch.org/docs/stable/distributed.html"""
)
parser.add_argument("--local_rank",
default=0,
type=int,
help="Please ignore and do not set this argument.")
parser.add_argument('--data_path', default='/path/to/imagenet/', type=str)
args = parser.parse_args()
utils.init_distributed_mode(args)
print("git:\n {}\n".format(utils.get_sha()))
print("\n".join("%s: %s" % (k, str(v))
for k, v in sorted(dict(vars(args)).items())))
cudnn.benchmark = True
if args.load_features:
train_features = torch.load(
os.path.join(args.load_features, "trainfeat.pth"))
test_features = torch.load(
os.path.join(args.load_features, "testfeat.pth"))
train_labels = torch.load(
os.path.join(args.load_features, "trainlabels.pth"))
test_labels = torch.load(
os.path.join(args.load_features, "testlabels.pth"))
else:
# need to extract features !
def train_dino(args):
utils.init_distributed_mode(args)
utils.fix_random_seeds(args.seed)
print("git:\n {}\n".format(utils.get_sha()))
print("\n".join("%s: %s" % (k, str(v))
for k, v in sorted(dict(vars(args)).items())))
cudnn.benchmark = True
# ============ preparing data ... ============
transform = DataAugmentationDINO(
args.global_crops_scale,
args.local_crops_scale,
args.local_crops_number,
)
#dataset = datasets.ImageFolder(args.data_path, transform=transform)
from sen12ms import get_transform
dataset = AllSen12MSDataset(args.data_path,
"train",
transform=transform,
tansform_coord=None,
classes=None,
seasons=None,
split_by_region=True,
download=False)
sampler = torch.utils.data.DistributedSampler(dataset, shuffle=True)
data_loader = torch.utils.data.DataLoader(
dataset,
sampler=sampler,
batch_size=args.batch_size_per_gpu,
num_workers=args.num_workers,
pin_memory=True,
drop_last=True,
)
print(f"Data loaded: there are {len(dataset)} images.")
# ============ building student and teacher networks ... ============
# if the network is a vision transformer (i.e. deit_tiny, deit_small, vit_base)
if args.arch in vits.__dict__.keys():
student = vits.__dict__[args.arch](
patch_size=args.patch_size,
drop_path_rate=0.1, # stochastic depth
)
teacher = vits.__dict__[args.arch](patch_size=args.patch_size)
embed_dim = student.embed_dim
student = utils.replace_input_layer(student, inchannels=13)
teacher = utils.replace_input_layer(teacher, inchannels=13)
# otherwise, we check if the architecture is in torchvision models
elif args.arch in torchvision_models.__dict__.keys():
student = torchvision_models.__dict__[args.arch]()
teacher = torchvision_models.__dict__[args.arch]()
embed_dim = student.fc.weight.shape[1]
else:
print(f"Unknow architecture: {args.arch}")
# multi-crop wrapper handles forward with inputs of different resolutions
student = utils.MultiCropWrapper(
student,
DINOHead(
embed_dim,
args.out_dim,
use_bn=args.use_bn_in_head,
norm_last_layer=args.norm_last_layer,
))
teacher = utils.MultiCropWrapper(
teacher,
DINOHead(embed_dim, args.out_dim, args.use_bn_in_head),
)
# move networks to gpu
student, teacher = student.cuda(), teacher.cuda()
# synchronize batch norms (if any)
if utils.has_batchnorms(student):
student = nn.SyncBatchNorm.convert_sync_batchnorm(student)
teacher = nn.SyncBatchNorm.convert_sync_batchnorm(teacher)
# we need DDP wrapper to have synchro batch norms working...
teacher = nn.parallel.DistributedDataParallel(teacher,
device_ids=[args.gpu])
teacher_without_ddp = teacher.module
else:
# teacher_without_ddp and teacher are the same thing
teacher_without_ddp = teacher
student = nn.parallel.DistributedDataParallel(student,
device_ids=[args.gpu])
# teacher and student start with the same weights
teacher_without_ddp.load_state_dict(student.module.state_dict())
# there is no backpropagation through the teacher, so no need for gradients
for p in teacher.parameters():
p.requires_grad = False
print(f"Student and Teacher are built: they are both {args.arch} network.")
# ============ preparing loss ... ============
dino_loss = DINOLoss(
args.out_dim,
args.local_crops_number +
2, # total number of crops = 2 global crops + local_crops_number
args.warmup_teacher_temp,
args.teacher_temp,
args.warmup_teacher_temp_epochs,
args.epochs,
).cuda()
# ============ preparing optimizer ... ============
params_groups = utils.get_params_groups(student)
if args.optimizer == "adamw":
optimizer = torch.optim.AdamW(params_groups) # to use with ViTs
elif args.optimizer == "sgd":
optimizer = torch.optim.SGD(params_groups, lr=0,
momentum=0.9) # lr is set by scheduler
elif args.optimizer == "lars":
optimizer = utils.LARS(
params_groups) # to use with convnet and large batches
# for mixed precision training
fp16_scaler = None
if args.use_fp16:
fp16_scaler = torch.cuda.amp.GradScaler()
# ============ init schedulers ... ============
lr_schedule = utils.cosine_scheduler(
args.lr * (args.batch_size_per_gpu * utils.get_world_size()) /
256., # linear scaling rule
args.min_lr,
args.epochs,
len(data_loader),
warmup_epochs=args.warmup_epochs,
)
wd_schedule = utils.cosine_scheduler(
args.weight_decay,
args.weight_decay_end,
args.epochs,
len(data_loader),
)
# momentum parameter is increased to 1. during training with a cosine schedule
momentum_schedule = utils.cosine_scheduler(args.momentum_teacher, 1,
args.epochs, len(data_loader))
print(f"Loss, optimizer and schedulers ready.")
# ============ optionally resume training ... ============
to_restore = {"epoch": 0}
utils.restart_from_checkpoint(
os.path.join(args.output_dir, "checkpoint.pth"),
run_variables=to_restore,
student=student,
teacher=teacher,
optimizer=optimizer,
fp16_scaler=fp16_scaler,
dino_loss=dino_loss,
)
start_epoch = to_restore["epoch"]
start_time = time.time()
print("Starting DINO training !")
for epoch in range(start_epoch, args.epochs):
data_loader.sampler.set_epoch(epoch)
# ============ training one epoch of DINO ... ============
train_stats = train_one_epoch(student, teacher, teacher_without_ddp,
dino_loss, data_loader, optimizer,
lr_schedule, wd_schedule,
momentum_schedule, epoch, fp16_scaler,
args)
# ============ writing logs ... ============
save_dict = {
'student': student.state_dict(),
'teacher': teacher.state_dict(),
'optimizer': optimizer.state_dict(),
'epoch': epoch + 1,
'args': args,
'dino_loss': dino_loss.state_dict(),
}
if fp16_scaler is not None:
save_dict['fp16_scaler'] = fp16_scaler.state_dict()
utils.save_on_master(save_dict,
os.path.join(args.output_dir, 'checkpoint.pth'))
if args.saveckp_freq and epoch % args.saveckp_freq == 0:
utils.save_on_master(
save_dict,
os.path.join(args.output_dir, f'checkpoint{epoch:04}.pth'))
log_stats = {
**{f'train_{k}': v
for k, v in train_stats.items()}, 'epoch': epoch
}
if utils.is_main_process():
with (Path(args.output_dir) / "log.txt").open("a") as f:
f.write(json.dumps(log_stats) + "\n")
total_time = time.time() - start_time
total_time_str = str(datetime.timedelta(seconds=int(total_time)))
print('Training time {}'.format(total_time_str))
def main(args):
utils.init_distributed_mode(args)
print("git:\n {}\n".format(utils.get_sha()))
if args.mask_model != "none":
args.masks = True
print(args)
device = torch.device(args.device)
# fix the seed for reproducibility
seed = args.seed + utils.get_rank()
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
model, criterion, postprocessor = build_model(args)
postprocessor.rescale_to_orig_size = True # for evaluation
model.to(device)
model_without_ddp = model
if args.distributed:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
model_without_ddp = model.module
n_parameters = builtins.sum(p.numel() for p in model.parameters()
if p.requires_grad)
print("number of params:", n_parameters)
# optimizer = torch.optim.Adam(model.parameters())
param_dicts = [
{
"params": [
p for n, p in model_without_ddp.named_parameters()
if "backbone" not in n
]
},
{
"params": [
p for n, p in model_without_ddp.named_parameters()
if "backbone" in n
],
"lr":
args.lr_backbone,
},
]
if args.optimizer == "sgd":
optimizer = torch.optim.SGD(param_dicts,
lr=args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
elif args.optimizer in ["adam", "adamw"]:
optimizer = torch.optim.AdamW(param_dicts,
lr=args.lr,
weight_decay=args.weight_decay)
else:
raise RuntimeError(f"Unsupported optimizer {args.optimizer}")
if args.schedule == "step":
lr_scheduler = StepLR(optimizer, args.lr_drop)
elif args.schedule == "multistep":
milestones = list(range(args.lr_drop, args.epochs, 50))
lr_scheduler = MultiStepLR(optimizer, gamma=0.5, milestones=milestones)
dataset_train = build_dataset(image_set="trainval", args=args)
dataset_val = build_dataset(image_set="test", args=args)
if args.distributed:
sampler_train = DistributedSampler(dataset_train)
sampler_val = DistributedSampler(dataset_val, shuffle=False)
else:
sampler_train = torch.utils.data.RandomSampler(dataset_train)
sampler_val = torch.utils.data.SequentialSampler(dataset_val)
batch_sampler_train = torch.utils.data.BatchSampler(sampler_train,
args.batch_size,
drop_last=True)
data_loader_train = DataLoader(
dataset_train,
batch_sampler=batch_sampler_train,
collate_fn=utils.collate_fn,
num_workers=args.num_workers,
)
data_loader_val = DataLoader(
dataset_val,
args.batch_size,
sampler=sampler_val,
drop_last=False,
collate_fn=utils.collate_fn,
num_workers=args.num_workers,
)
if args.dataset_file == "coco_panoptic":
# We also evaluate AP during panoptic training, on original coco DS
coco_val = datasets.coco.build("val", args)
base_ds = to_coco_api.get_coco_api_from_dataset(coco_val)
else:
base_ds = None # to_coco_api.get_coco_api_from_dataset(dataset_val)
output_dir = Path(args.output_dir)
if args.resume:
checkpoint = torch.load(args.resume, map_location="cpu")
model_without_ddp.load_state_dict(checkpoint["model"])
optimizer.load_state_dict(checkpoint["optimizer"])
lr_scheduler.load_state_dict(checkpoint["lr_scheduler"])
args.start_epoch = checkpoint["epoch"] + 1
if args.eval:
test_stats, coco_evaluator = evaluate(
model,
criterion,
postprocessor,
data_loader_val,
base_ds,
device,
eval_bbox=True,
eval_masks=args.masks,
)
if args.output_dir:
utils.save_on_master(coco_evaluator.coco_eval["bbox"].eval,
output_dir / "eval.pth")
return
print("Start training")
start_time = time.time()
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
sampler_train.set_epoch(epoch)
train_stats = train_one_epoch(model, criterion, data_loader_train,
optimizer, device, epoch,
args.clip_max_norm)
lr_scheduler.step()
if args.output_dir:
checkpoint_paths = [output_dir / "checkpoint.pth"]
# extra checkpoint before LR drop and every 100 epochs
if (epoch + 1) % args.lr_drop == 0 or (epoch + 1) % 100 == 0:
checkpoint_paths.append(output_dir /
f"checkpoint{epoch:04}.pth")
for checkpoint_path in checkpoint_paths:
utils.save_on_master(
{
"model": model_without_ddp.state_dict(),
"optimizer": optimizer.state_dict(),
"lr_scheduler": lr_scheduler.state_dict(),
"epoch": epoch,
"args": args,
},
checkpoint_path,
)
# if epoch % args.eval_skip == 0:
# test_stats, coco_evaluator = evaluate(
# model, criterion, postprocessor, data_loader_val, base_ds, device, eval_bbox=True, eval_masks=args.masks
# )
# else:
# test_stats, coco_evaluator = {}, None
test_stats, coco_evaluator = {}, None
log_stats = {
**{f"train_{k}": v
for k, v in train_stats.items()},
**{f"test_{k}": v
for k, v in test_stats.items()},
"n_parameters": n_parameters,
}
if args.output_dir and utils.is_main_process():
with (output_dir / "log.txt").open("a") as f:
f.write(json.dumps(log_stats) + "\n")
# for evaluation logs
if coco_evaluator is not None:
os.makedirs(os.path.join(args.output_dir, "eval"),
exist_ok=True)
if "bbox" in coco_evaluator.coco_eval:
filenames = ["latest.pth"]
if epoch % 50 == 0:
filenames.append(f"{epoch:03}.pth")
for name in filenames:
torch.save(coco_evaluator.coco_eval["bbox"].eval,
output_dir / "eval" / name)
with (output_dir / "log_tb.txt").open("a") as f:
f.write(f"TORCHBOARD_METRICS[epoch] = {epoch}\n")
for k, v in vars(args).items():
f.write(f"TORCHBOARD_METRICS[{k}] = {v}")
for key in log_stats:
v = log_stats[key]
if isinstance(v, list):
for i, vi in enumerate(v):
f.write(f"TORCHBOARD_METRICS[{key}_{i}] = {vi}\n")
else:
f.write(f"TORCHBOARD_METRICS[{key}] = {v}\n")
total_time = time.time() - start_time
total_time_str = str(datetime.timedelta(seconds=int(total_time)))
print("Training time {}".format(total_time_str))
