def process_dent(self, nwo, ext, library_candidates) -> Tuple[List[Dict[str, Any]], List[Tuple[str, str]]]: # Process dependents (applications) to get function calls dents = [] edges = [] _, nwo = remap_nwo(nwo) if nwo is None: return dents, edges tmp_dir = download(nwo) files = walk(tmp_dir, ext) sha = None for f in files: context_and_calls = self.get_context_and_function_calls(f) if context_and_calls is None: continue if sha is None: sha = get_sha(tmp_dir, nwo) nwo, path, context, calls = context_and_calls libraries = [] for cxt in context: if type(cxt) == dict: libraries.extend([v.split('.')[0] for v in cxt.values()]) elif type(cxt) == list: libraries.extend(cxt) match_scopes = {} for cxt in set(libraries): if cxt in library_candidates: match_scopes[cxt] = library_candidates[cxt] for call in calls: for depended_library_name, dependend_library_functions in match_scopes.items(): for depended_library_function in dependend_library_functions: # Other potential filters: len(call['identifier']) > 6 or len(call['identifier'].split('_')) > 1 if (call['identifier'] not in self.language_parser.STOPWORDS and ((depended_library_function['identifier'].split('.')[-1] == '__init__' and call['identifier'] == depended_library_function['identifier'].split('.')[0]) or ((len(call['identifier']) > 9 or (not call['identifier'].startswith('_') and len(call['identifier'].split('_')) > 1)) and call['identifier'] == depended_library_function['identifier']) )): dent = { 'nwo': nwo, 'sha': sha, 'path': path, 'language': self.language, 'identifier': call['identifier'], 'argument_list': call['argument_list'], 'url': 'https://github.com/{}/blob/{}/{}#L{}-L{}'.format(nwo, sha, path, call['start_point'][0] + 1, call['end_point'][0] + 1) } dents.append(dent) edges.append((dent['url'], depended_library_function['url'])) return dents, edges
async def game_new(): """ """ game_id = get_sha() code = get_random_code() logger.debug("New game /game/new: %s || %s" % (game_id, code)) valid_game = await create_game(game_id, code) if not valid_game: return make_json_response(code=1, msg='Something went wrong') else: return make_json_response(code=0, body={'game_id': game_id})
def process_dee(self, nwo, ext) -> List[Dict[str, Any]]: # Process dependees (libraries) to get function implementations indexes = [] _, nwo = remap_nwo(nwo) if nwo is None: return indexes tmp_dir = download(nwo) files = walk(tmp_dir, ext) # files = glob.iglob(tmp_dir.name + '/**/*.{}'.format(ext), recursive=True) sha = None for f in files: definitions = self.get_function_definitions(f) if definitions is None: continue if sha is None: sha = get_sha(tmp_dir, nwo) nwo, path, functions = definitions indexes.extend((self.extract_function_data(func, nwo, path, sha) for func in functions if len(func['function_tokens']) > 1)) return indexes
def setup_experiment(): np.random.seed(FLAGS.seed) if not tf.gfile.Exists(FLAGS.data_dir) or not tf.gfile.IsDirectory( FLAGS.data_dir): raise ValueError("Could not find folder %s" % FLAGS.data_dir) assert FLAGS.batch_size % FLAGS.carlini_batch_size == 0 if not tf.gfile.Exists(FLAGS.load_dir) or not tf.gfile.IsDirectory( FLAGS.load_dir): raise ValueError("Could not find folder %s" % FLAGS.load_dir) FLAGS.working_dir = os.path.join(FLAGS.working_dir, os.path.basename(os.path.normpath(FLAGS.load_dir))) FLAGS.adv_data_dir = os.path.join(FLAGS.working_dir, FLAGS.adv_data_dir) FLAGS.samples_dir = os.path.join(FLAGS.working_dir, FLAGS.samples_dir) FLAGS.git_revision = get_sha() if tf.gfile.Exists(FLAGS.working_dir): tf.gfile.DeleteRecursively(FLAGS.working_dir) tf.gfile.MakeDirs(FLAGS.working_dir) tf.gfile.MakeDirs(FLAGS.adv_data_dir) tf.gfile.MakeDirs(FLAGS.samples_dir) train_params = load_training_params(FLAGS.load_dir) FLAGS.model = train_params['model'] FLAGS.model_name = train_params['model_name'] FLAGS.activation_fn = train_params['activation_fn'] FLAGS.num_classes = train_params['num_classes'] FLAGS.layer_dims = train_params['layer_dims'] # configure logging logger = getLogger('tensorflow') tf.logging.set_verbosity(tf.logging.INFO) file_hndl = FileHandler(os.path.join(FLAGS.working_dir, 'tensorflow.log')) file_hndl.setLevel(logging.DEBUG) logger.addHandler(file_hndl) # print config logging.info({k: v.value for k, v in FLAGS._flags().items()}) logging.info("Training params: %s", train_params)
def automated_operators_repository_commit_message( operator_repository: str, operator_name: str, operator_directory: str, operator_git_tag: str, debug: bool, ) -> Tuple[int, str, str, str]: rc, stdout, stderr = get_sha(operator_directory, debug) if rc != 0: return ( rc, "", "", f"Failed to get git SHA from '{operator_directory}'" + f"\nstdout:\n{stdout}\nstderr:\n{stderr}", ) operator_git_sha = stdout.strip() operator_repository_url = github_repository_url(operator_repository) operator_git_tag_url = ( f"{operator_repository_url}/releases/tag/{operator_git_tag}") operator_git_sha_url = ( f"{operator_repository_url}/commit/{operator_git_sha}") commit_message_subject = ( f"Release {operator_name} {operator_git_tag} (automated commit).") commit_message_body = "\n".join([ f"| | |", f"|-|-|", f"| Repository | {operator_repository_url} |", f"| Operator | {operator_name} |", f"| Git tag | {operator_git_tag_url} |", f"| Git SHA | {operator_git_sha_url} |", f"| Date (UTC) | {datetime.utcnow()} |", ]) return 0, commit_message_subject, commit_message_body, ""
def eval_linear(args): utils.init_distributed_mode(args) print("git:\n {}\n".format(utils.get_sha())) print("\n".join("%s: %s" % (k, str(v)) for k, v in sorted(dict(vars(args)).items()))) cudnn.benchmark = True # ============ preparing data ... ============ train_transform = pth_transforms.Compose([ pth_transforms.RandomResizedCrop(224), pth_transforms.RandomHorizontalFlip(), pth_transforms.ToTensor(), pth_transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) val_transform = pth_transforms.Compose([ pth_transforms.Resize(256, interpolation=3), pth_transforms.CenterCrop(224), pth_transforms.ToTensor(), pth_transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) dataset_train = datasets.ImageFolder(os.path.join(args.data_path, "train"), transform=train_transform) dataset_val = datasets.ImageFolder(os.path.join(args.data_path, "val"), transform=val_transform) sampler = torch.utils.data.distributed.DistributedSampler(dataset_train) train_loader = torch.utils.data.DataLoader( dataset_train, sampler=sampler, batch_size=args.batch_size_per_gpu, num_workers=args.num_workers, pin_memory=True, ) val_loader = torch.utils.data.DataLoader( dataset_val, batch_size=args.batch_size_per_gpu, num_workers=args.num_workers, pin_memory=True, ) print(f"Data loaded with {len(dataset_train)} train and {len(dataset_val)} val imgs.") # ============ building network ... ============ model = vits.__dict__[args.arch](patch_size=args.patch_size, num_classes=0) model.cuda() model.eval() print(f"Model {args.arch} {args.patch_size}x{args.patch_size} built.") # load weights to evaluate utils.load_pretrained_weights(model, args.pretrained_weights, args.checkpoint_key, args.arch, args.patch_size) linear_classifier = LinearClassifier(model.embed_dim * (args.n_last_blocks + int(args.avgpool_patchtokens)), num_labels=args.num_labels) linear_classifier = linear_classifier.cuda() linear_classifier = nn.parallel.DistributedDataParallel(linear_classifier, device_ids=[args.gpu]) # set optimizer optimizer = torch.optim.SGD( linear_classifier.parameters(), args.lr * (args.batch_size_per_gpu * utils.get_world_size()) / 256., # linear scaling rule momentum=0.9, weight_decay=0, # we do not apply weight decay ) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epochs, eta_min=0) # Optionally resume from a checkpoint to_restore = {"epoch": 0, "best_acc": 0.} utils.restart_from_checkpoint( os.path.join(args.output_dir, "checkpoint.pth.tar"), run_variables=to_restore, state_dict=linear_classifier, optimizer=optimizer, scheduler=scheduler, ) start_epoch = to_restore["epoch"] best_acc = to_restore["best_acc"] for epoch in range(start_epoch, args.epochs): train_loader.sampler.set_epoch(epoch) train_stats = train(model, linear_classifier, optimizer, train_loader, epoch, args.n_last_blocks, args.avgpool_patchtokens) scheduler.step() log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, 'epoch': epoch} if epoch % args.val_freq == 0 or epoch == args.epochs - 1: test_stats = validate_network(val_loader, model, linear_classifier, args.n_last_blocks, args.avgpool_patchtokens) print(f"Accuracy at epoch {epoch} of the network on the {len(dataset_val)} test images: {test_stats['acc1']:.1f}%") best_acc = max(best_acc, test_stats["acc1"]) print(f'Max accuracy so far: {best_acc:.2f}%') log_stats = {**{k: v for k, v in log_stats.items()}, **{f'test_{k}': v for k, v in test_stats.items()}} if utils.is_main_process(): with (Path(args.output_dir) / "log.txt").open("a") as f: f.write(json.dumps(log_stats) + "\n") save_dict = { "epoch": epoch + 1, "state_dict": linear_classifier.state_dict(), "optimizer": optimizer.state_dict(), "scheduler": scheduler.state_dict(), "best_acc": best_acc, } torch.save(save_dict, os.path.join(args.output_dir, "checkpoint.pth.tar")) print("Training of the supervised linear classifier on frozen features completed.\n" "Top-1 test accuracy: {acc:.1f}".format(acc=best_acc))
if args.dataset == 'ptb': train_data, val_data, test_data = PTBSeq2Seq.iters( batch_size=args.batch_size, device=device) corpus = train_data.dataset.fields['target'].vocab # includes BOS ntokens = len(corpus) model = rvae.RVAE(ntokens, args.emsize, args.nhid, args.z_dim, 1, args.dropout, args.dropouth, args.dropouti, args.dropoute, args.wdrop) if args.cuda and torch.cuda.is_available(): model.cuda() total_params = sum(x.size()[0] * x.size()[1] if len(x.size()) > 1 else x.size()[0] for x in model.parameters()) print("sha: {}".format(get_sha().strip())) print('args:', args) print('model total parameters:', total_params) print('model architecture:') print(model) criterion = nn.CrossEntropyLoss() # Loop over epochs. args.anneal = 0.0001 lr = args.lr best_val_loss = [] stored_loss = 100000000 # At any point you can hit Ctrl + C to break out of training early. try:
help='Number of data loading workers per GPU.') parser.add_argument("--dist_url", default="env://", type=str, help="""url used to set up distributed training; see https://pytorch.org/docs/stable/distributed.html""" ) parser.add_argument("--local_rank", default=0, type=int, help="Please ignore and do not set this argument.") parser.add_argument('--data_path', default='/path/to/imagenet/', type=str) args = parser.parse_args() utils.init_distributed_mode(args) print("git:\n {}\n".format(utils.get_sha())) print("\n".join("%s: %s" % (k, str(v)) for k, v in sorted(dict(vars(args)).items()))) cudnn.benchmark = True if args.load_features: train_features = torch.load( os.path.join(args.load_features, "trainfeat.pth")) test_features = torch.load( os.path.join(args.load_features, "testfeat.pth")) train_labels = torch.load( os.path.join(args.load_features, "trainlabels.pth")) test_labels = torch.load( os.path.join(args.load_features, "testlabels.pth")) else: # need to extract features !
def train_dino(args): utils.init_distributed_mode(args) utils.fix_random_seeds(args.seed) print("git:\n {}\n".format(utils.get_sha())) print("\n".join("%s: %s" % (k, str(v)) for k, v in sorted(dict(vars(args)).items()))) cudnn.benchmark = True # ============ preparing data ... ============ transform = DataAugmentationDINO( args.global_crops_scale, args.local_crops_scale, args.local_crops_number, ) #dataset = datasets.ImageFolder(args.data_path, transform=transform) from sen12ms import get_transform dataset = AllSen12MSDataset(args.data_path, "train", transform=transform, tansform_coord=None, classes=None, seasons=None, split_by_region=True, download=False) sampler = torch.utils.data.DistributedSampler(dataset, shuffle=True) data_loader = torch.utils.data.DataLoader( dataset, sampler=sampler, batch_size=args.batch_size_per_gpu, num_workers=args.num_workers, pin_memory=True, drop_last=True, ) print(f"Data loaded: there are {len(dataset)} images.") # ============ building student and teacher networks ... ============ # if the network is a vision transformer (i.e. deit_tiny, deit_small, vit_base) if args.arch in vits.__dict__.keys(): student = vits.__dict__[args.arch]( patch_size=args.patch_size, drop_path_rate=0.1, # stochastic depth ) teacher = vits.__dict__[args.arch](patch_size=args.patch_size) embed_dim = student.embed_dim student = utils.replace_input_layer(student, inchannels=13) teacher = utils.replace_input_layer(teacher, inchannels=13) # otherwise, we check if the architecture is in torchvision models elif args.arch in torchvision_models.__dict__.keys(): student = torchvision_models.__dict__[args.arch]() teacher = torchvision_models.__dict__[args.arch]() embed_dim = student.fc.weight.shape[1] else: print(f"Unknow architecture: {args.arch}") # multi-crop wrapper handles forward with inputs of different resolutions student = utils.MultiCropWrapper( student, DINOHead( embed_dim, args.out_dim, use_bn=args.use_bn_in_head, norm_last_layer=args.norm_last_layer, )) teacher = utils.MultiCropWrapper( teacher, DINOHead(embed_dim, args.out_dim, args.use_bn_in_head), ) # move networks to gpu student, teacher = student.cuda(), teacher.cuda() # synchronize batch norms (if any) if utils.has_batchnorms(student): student = nn.SyncBatchNorm.convert_sync_batchnorm(student) teacher = nn.SyncBatchNorm.convert_sync_batchnorm(teacher) # we need DDP wrapper to have synchro batch norms working... teacher = nn.parallel.DistributedDataParallel(teacher, device_ids=[args.gpu]) teacher_without_ddp = teacher.module else: # teacher_without_ddp and teacher are the same thing teacher_without_ddp = teacher student = nn.parallel.DistributedDataParallel(student, device_ids=[args.gpu]) # teacher and student start with the same weights teacher_without_ddp.load_state_dict(student.module.state_dict()) # there is no backpropagation through the teacher, so no need for gradients for p in teacher.parameters(): p.requires_grad = False print(f"Student and Teacher are built: they are both {args.arch} network.") # ============ preparing loss ... ============ dino_loss = DINOLoss( args.out_dim, args.local_crops_number + 2, # total number of crops = 2 global crops + local_crops_number args.warmup_teacher_temp, args.teacher_temp, args.warmup_teacher_temp_epochs, args.epochs, ).cuda() # ============ preparing optimizer ... ============ params_groups = utils.get_params_groups(student) if args.optimizer == "adamw": optimizer = torch.optim.AdamW(params_groups) # to use with ViTs elif args.optimizer == "sgd": optimizer = torch.optim.SGD(params_groups, lr=0, momentum=0.9) # lr is set by scheduler elif args.optimizer == "lars": optimizer = utils.LARS( params_groups) # to use with convnet and large batches # for mixed precision training fp16_scaler = None if args.use_fp16: fp16_scaler = torch.cuda.amp.GradScaler() # ============ init schedulers ... ============ lr_schedule = utils.cosine_scheduler( args.lr * (args.batch_size_per_gpu * utils.get_world_size()) / 256., # linear scaling rule args.min_lr, args.epochs, len(data_loader), warmup_epochs=args.warmup_epochs, ) wd_schedule = utils.cosine_scheduler( args.weight_decay, args.weight_decay_end, args.epochs, len(data_loader), ) # momentum parameter is increased to 1. during training with a cosine schedule momentum_schedule = utils.cosine_scheduler(args.momentum_teacher, 1, args.epochs, len(data_loader)) print(f"Loss, optimizer and schedulers ready.") # ============ optionally resume training ... ============ to_restore = {"epoch": 0} utils.restart_from_checkpoint( os.path.join(args.output_dir, "checkpoint.pth"), run_variables=to_restore, student=student, teacher=teacher, optimizer=optimizer, fp16_scaler=fp16_scaler, dino_loss=dino_loss, ) start_epoch = to_restore["epoch"] start_time = time.time() print("Starting DINO training !") for epoch in range(start_epoch, args.epochs): data_loader.sampler.set_epoch(epoch) # ============ training one epoch of DINO ... ============ train_stats = train_one_epoch(student, teacher, teacher_without_ddp, dino_loss, data_loader, optimizer, lr_schedule, wd_schedule, momentum_schedule, epoch, fp16_scaler, args) # ============ writing logs ... ============ save_dict = { 'student': student.state_dict(), 'teacher': teacher.state_dict(), 'optimizer': optimizer.state_dict(), 'epoch': epoch + 1, 'args': args, 'dino_loss': dino_loss.state_dict(), } if fp16_scaler is not None: save_dict['fp16_scaler'] = fp16_scaler.state_dict() utils.save_on_master(save_dict, os.path.join(args.output_dir, 'checkpoint.pth')) if args.saveckp_freq and epoch % args.saveckp_freq == 0: utils.save_on_master( save_dict, os.path.join(args.output_dir, f'checkpoint{epoch:04}.pth')) log_stats = { **{f'train_{k}': v for k, v in train_stats.items()}, 'epoch': epoch } if utils.is_main_process(): with (Path(args.output_dir) / "log.txt").open("a") as f: f.write(json.dumps(log_stats) + "\n") total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=int(total_time))) print('Training time {}'.format(total_time_str))
def main(args): utils.init_distributed_mode(args) print("git:\n {}\n".format(utils.get_sha())) if args.mask_model != "none": args.masks = True print(args) device = torch.device(args.device) # fix the seed for reproducibility seed = args.seed + utils.get_rank() torch.manual_seed(seed) np.random.seed(seed) random.seed(seed) model, criterion, postprocessor = build_model(args) postprocessor.rescale_to_orig_size = True # for evaluation model.to(device) model_without_ddp = model if args.distributed: model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.gpu]) model_without_ddp = model.module n_parameters = builtins.sum(p.numel() for p in model.parameters() if p.requires_grad) print("number of params:", n_parameters) # optimizer = torch.optim.Adam(model.parameters()) param_dicts = [ { "params": [ p for n, p in model_without_ddp.named_parameters() if "backbone" not in n ] }, { "params": [ p for n, p in model_without_ddp.named_parameters() if "backbone" in n ], "lr": args.lr_backbone, }, ] if args.optimizer == "sgd": optimizer = torch.optim.SGD(param_dicts, lr=args.lr, momentum=0.9, weight_decay=args.weight_decay) elif args.optimizer in ["adam", "adamw"]: optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, weight_decay=args.weight_decay) else: raise RuntimeError(f"Unsupported optimizer {args.optimizer}") if args.schedule == "step": lr_scheduler = StepLR(optimizer, args.lr_drop) elif args.schedule == "multistep": milestones = list(range(args.lr_drop, args.epochs, 50)) lr_scheduler = MultiStepLR(optimizer, gamma=0.5, milestones=milestones) dataset_train = build_dataset(image_set="trainval", args=args) dataset_val = build_dataset(image_set="test", args=args) if args.distributed: sampler_train = DistributedSampler(dataset_train) sampler_val = DistributedSampler(dataset_val, shuffle=False) else: sampler_train = torch.utils.data.RandomSampler(dataset_train) sampler_val = torch.utils.data.SequentialSampler(dataset_val) batch_sampler_train = torch.utils.data.BatchSampler(sampler_train, args.batch_size, drop_last=True) data_loader_train = DataLoader( dataset_train, batch_sampler=batch_sampler_train, collate_fn=utils.collate_fn, num_workers=args.num_workers, ) data_loader_val = DataLoader( dataset_val, args.batch_size, sampler=sampler_val, drop_last=False, collate_fn=utils.collate_fn, num_workers=args.num_workers, ) if args.dataset_file == "coco_panoptic": # We also evaluate AP during panoptic training, on original coco DS coco_val = datasets.coco.build("val", args) base_ds = to_coco_api.get_coco_api_from_dataset(coco_val) else: base_ds = None # to_coco_api.get_coco_api_from_dataset(dataset_val) output_dir = Path(args.output_dir) if args.resume: checkpoint = torch.load(args.resume, map_location="cpu") model_without_ddp.load_state_dict(checkpoint["model"]) optimizer.load_state_dict(checkpoint["optimizer"]) lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) args.start_epoch = checkpoint["epoch"] + 1 if args.eval: test_stats, coco_evaluator = evaluate( model, criterion, postprocessor, data_loader_val, base_ds, device, eval_bbox=True, eval_masks=args.masks, ) if args.output_dir: utils.save_on_master(coco_evaluator.coco_eval["bbox"].eval, output_dir / "eval.pth") return print("Start training") start_time = time.time() for epoch in range(args.start_epoch, args.epochs): if args.distributed: sampler_train.set_epoch(epoch) train_stats = train_one_epoch(model, criterion, data_loader_train, optimizer, device, epoch, args.clip_max_norm) lr_scheduler.step() if args.output_dir: checkpoint_paths = [output_dir / "checkpoint.pth"] # extra checkpoint before LR drop and every 100 epochs if (epoch + 1) % args.lr_drop == 0 or (epoch + 1) % 100 == 0: checkpoint_paths.append(output_dir / f"checkpoint{epoch:04}.pth") for checkpoint_path in checkpoint_paths: utils.save_on_master( { "model": model_without_ddp.state_dict(), "optimizer": optimizer.state_dict(), "lr_scheduler": lr_scheduler.state_dict(), "epoch": epoch, "args": args, }, checkpoint_path, ) # if epoch % args.eval_skip == 0: # test_stats, coco_evaluator = evaluate( # model, criterion, postprocessor, data_loader_val, base_ds, device, eval_bbox=True, eval_masks=args.masks # ) # else: # test_stats, coco_evaluator = {}, None test_stats, coco_evaluator = {}, None log_stats = { **{f"train_{k}": v for k, v in train_stats.items()}, **{f"test_{k}": v for k, v in test_stats.items()}, "n_parameters": n_parameters, } if args.output_dir and utils.is_main_process(): with (output_dir / "log.txt").open("a") as f: f.write(json.dumps(log_stats) + "\n") # for evaluation logs if coco_evaluator is not None: os.makedirs(os.path.join(args.output_dir, "eval"), exist_ok=True) if "bbox" in coco_evaluator.coco_eval: filenames = ["latest.pth"] if epoch % 50 == 0: filenames.append(f"{epoch:03}.pth") for name in filenames: torch.save(coco_evaluator.coco_eval["bbox"].eval, output_dir / "eval" / name) with (output_dir / "log_tb.txt").open("a") as f: f.write(f"TORCHBOARD_METRICS[epoch] = {epoch}\n") for k, v in vars(args).items(): f.write(f"TORCHBOARD_METRICS[{k}] = {v}") for key in log_stats: v = log_stats[key] if isinstance(v, list): for i, vi in enumerate(v): f.write(f"TORCHBOARD_METRICS[{key}_{i}] = {vi}\n") else: f.write(f"TORCHBOARD_METRICS[{key}] = {v}\n") total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=int(total_time))) print("Training time {}".format(total_time_str))