def prepare_and_run(params): loo = LeaveOneOut() plates, classes = get_train_exemplars() cv_monitor = MetricsMonitorCV() fold_num = 0 for train_index, val_index in loo.split(plates): train_paths, val_paths = plates[train_index], plates[val_index] train_classes, val_classes = classes[train_index], classes[val_index] train_dataset = get_dataset(list(zip(train_paths, classes)), train_transforms) train_dataloader = DataLoader(train_dataset, batch_size=params["batch_size"], shuffle=True) val_dataset = get_dataset(list(zip(val_paths, val_classes)), val_test_transforms) # we using leave one out validation, so batch_size should be 1 val_dataloader = DataLoader(val_dataset, batch_size=1) train_monitor, validation_monitor = train(train_dataloader, val_dataloader, fold_num, params) cv_monitor.add_train_monitor(train_monitor).add_val_monitor( validation_monitor) fold_num += 1 return cv_monitor
def test_udd_ofr_nids(self): """test one of haed and tail entity ID is larger than number of entities """ extra_train = ['7\t0\t1\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_train = ['0\t0\t6\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_valid = ['7\t0\t1\n'] files = gen_udd_files(extra_valid=extra_valid) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_test = ['7\t0\t1\n'] files = gen_udd_files(extra_test=extra_test) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files)
def test_udd_error_nids(self): """test one of head and tail entity ID < 0 """ extra_train = ['-1\t0\t1\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_train = ['0\t0\t-1\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_valid = ['-1\t0\t1\n'] files = gen_udd_files(extra_valid=extra_valid) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_test = ['-1\t0\t1\n'] files = gen_udd_files(extra_test=extra_test) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files)
def test_udd_noint_triplets(self): """test one of (h, r, t) is not an integer """ extra_train = ['deadbeaf\t0\t1\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(ValueError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_train = ['0\tdeadbeaf\t1\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(ValueError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_train = ['0\t0\tdeadbeaf\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(ValueError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_valid = ['deadbeaf\t0\t1\n'] files = gen_udd_files(extra_valid=extra_valid) with self.assertRaises(ValueError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_test = ['deadbeaf\t0\t1\n'] files = gen_udd_files(extra_test=extra_test) with self.assertRaises(ValueError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files)
def generate_datasets(DATASET_PATH, batch_size): ''' Load train, val and test dataset. :param DATASET_PATH: Path to dataset :param batch_size: batch size obviously ... :return: ''' # Load training data train_dataset = get_dataset("train", dataset_path=DATASET_PATH) # Load validation data test_dataset = get_dataset("test", dataset_path=DATASET_PATH) # Create the dataloader train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, num_workers=2, shuffle=True, collate_fn=collate_remove_none, worker_init_fn=worker_init_fn) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, num_workers=2, shuffle=False, collate_fn=collate_remove_none, worker_init_fn=worker_init_fn) vis_train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=1, num_workers=2, shuffle=False, collate_fn=collate_remove_none, worker_init_fn=worker_init_fn) return test_loader, train_loader, vis_train_loader
def gen_data(ckpt_path, flag, dataset_config, dataloader_config, njobs): ckpt_dir = os.path.join(ckpt_path, '') ckpt_dir_flag = os.path.join(ckpt_dir, flag) prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path)) train_pkl = os.path.join(ckpt_dir, f'{prefix}_train.pkl') dev_pkl = os.path.join(ckpt_dir, f'{prefix}_dev.pkl') os.makedirs(ckpt_dir, exist_ok=True) with open(os.path.join(ckpt_dir, 'dirtype'), 'w') as f: f.write('ckpt_dir') os.makedirs(ckpt_dir_flag, exist_ok=True) with open(os.path.join(ckpt_dir_flag, 'dirtype'), 'w') as f: f.write('ckpt_dir_flag') file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log')) logging.getLogger().addHandler(file_handler) if os.path.exists(train_pkl): logger.info(f'=> load train, dev data from {ckpt_dir}') train_data = pk.load(open(train_pkl, 'rb')) # train_set is a of 1600 elements which are dict. # with keys:'mel' for mel-spectogram np array (80,xxx) and 'speaker' for speaker string ('p225') # print(f"**********************data is: {train_data[0]}********************") # print(f"**********************type is: {train_data[0]['mel'].shape}********************") dev_data = pk.load(open(dev_pkl, 'rb')) train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs, metadata=train_data) # train_set is a class: dataloader.vctk.Dataset. # Has attribute data such that train_set.data is a list of 1600 features so train_set.data[i] is o size (80,xxx) # print(f"**********************type is: {type(train_set)}********************") # print(f"**********************shape is: {train_set.data[0]['mel'].shape}********************") dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs, metadata=dev_data) else: train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs) dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs) pk.dump(train_set.data, open(train_pkl, 'wb')) pk.dump(dev_set.data, open(dev_pkl, 'wb')) train_loader = get_dataloader(dset='train', dataloader_config=dataloader_config, dataset=train_set) dev_loader = get_dataloader(dset='dev', dataloader_config=dataloader_config, dataset=dev_set) # print(f"**********************type is: {type(train_loader)}********************") return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def load_data(ckpt_path, dataset_config, dataloader_config, njobs): if os.path.isdir(ckpt_path): d = os.path.join(ckpt_path, '') with open(os.path.join(d, 'dirtype'), 'r') as f: dirtype = f.read().strip() if dirtype == 'ckpt_dir': logger.warn( f'The ckpt_path is {ckpt_path}, the flag is not specified.' ) logger.warn(f'Use "default" flag.') ckpt_dir = d flag = 'default' ckpt_dir_flag = os.path.join(ckpt_dir, flag) ckpt_path = ckpt_dir_flag elif dirtype == 'ckpt_dir_flag': ckpt_dir_flag = os.path.dirname(d) ckpt_dir = os.path.dirname(ckpt_dir_flag) flag = os.path.basename(ckpt_dir_flag) else: raise NotImplementedError( f'Wrong dirtype: {dirtype} from {d}.') else: ckpt_dir_flag = os.path.dirname(ckpt_path) ckpt_dir = os.path.dirname(ckpt_dir_flag) flag = os.path.basename(ckpt_dir_flag) file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log')) logging.getLogger().addHandler(file_handler) logger.info(f'=> load train, dev data from {os.path.join(ckpt_dir)}') prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path)) train_data = pk.load( open(os.path.join(ckpt_dir, f'{prefix}_train.pkl'), 'rb')) dev_data = pk.load( open(os.path.join(ckpt_dir, f'{prefix}_dev.pkl'), 'rb')) train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs, metadata=train_data) dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs, metadata=dev_data) train_loader = get_dataloader(dset='train', dataloader_config=dataloader_config, dataset=train_set) dev_loader = get_dataloader(dset='dev', dataloader_config=dataloader_config, dataset=dev_set) return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def get_data_loaders(args, tokenizer): """ Prepare the dataset for training and evaluation """ datasets_raw = {} logger.info("Loading training data") datasets_raw['train'] = get_dataset(tokenizer, args.dataset_cache, args.dataset_path, 'train') logger.info("Loading validation data") datasets_raw['valid'] = get_dataset(tokenizer, args.dataset_cache, args.dataset_path, 'dev') logger.info("Build inputs and labels") datasets = {"train": defaultdict(list), "valid": defaultdict(list)} for dataset_name, dataset in datasets_raw.items(): for data_point in dataset: instance, _ = build_input_from_segments(data_point, tokenizer) for input_name, input_array in instance.items(): datasets[dataset_name][input_name].append(input_array) logger.info("Pad inputs and convert to Tensor") tensor_datasets = {"train": [], "valid": []} for dataset_name, dataset in datasets.items(): dataset = pad_dataset(dataset, padding=tokenizer.convert_tokens_to_ids( SPECIAL_TOKENS[-1])) for input_name in MODEL_INPUTS: tensor = torch.tensor(dataset[input_name]) tensor_datasets[dataset_name].append(tensor) logger.info("Build train and validation dataloaders") train_dataset, valid_dataset = TensorDataset( *tensor_datasets["train"]), TensorDataset(*tensor_datasets["valid"]) train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) if args.distributed else None valid_sampler = torch.utils.data.distributed.DistributedSampler( valid_dataset) if args.distributed else None train_loader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size, shuffle=(not args.distributed)) valid_loader = DataLoader(valid_dataset, sampler=valid_sampler, batch_size=args.valid_batch_size, shuffle=False) logger.info("Train dataset (Batch, Candidates, Seq length): {}".format( train_dataset.tensors[0].shape)) logger.info("Valid dataset (Batch, Candidates, Seq length): {}".format( valid_dataset.tensors[0].shape)) return train_loader, valid_loader, train_sampler, valid_sampler
def load_data(): BATCH_SIZE = 2 train_video_dataset = dataloader.get_dataset(dataloader.TRAIN_JSON_PATH, dataloader.SINGLE_FRAME) train_audio_dataset = dataloader.AudioDataset() train_loader = dataloader.AVDataLoader(train_video_dataset, train_audio_dataset, batch_size=BATCH_SIZE, shuffle=False, single_frame=True) return train_loader
def test_udd_error_rids(self): """test one of relation ID < 0 """ extra_train = ['0\t-1\t1\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files)
def gen_data(ckpt_path, flag, dataset_config, dataloader_config, njobs): ckpt_dir = os.path.join(ckpt_path, '') ckpt_dir_flag = os.path.join(ckpt_dir, flag) prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path)) train_pkl = os.path.join(ckpt_dir, f'{prefix}_train.pkl') dev_pkl = os.path.join(ckpt_dir, f'{prefix}_dev.pkl') os.makedirs(ckpt_dir, exist_ok=True) with open(os.path.join(ckpt_dir, 'dirtype'), 'w') as f: f.write('ckpt_dir') os.makedirs(ckpt_dir_flag, exist_ok=True) with open(os.path.join(ckpt_dir_flag, 'dirtype'), 'w') as f: f.write('ckpt_dir_flag') file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log')) logging.getLogger().addHandler(file_handler) if os.path.exists(train_pkl): logger.info(f'=> load train, dev data from {ckpt_dir}') train_data = pk.load(open(train_pkl, 'rb')) dev_data = pk.load(open(dev_pkl, 'rb')) train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs, metadata=train_data) dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs, metadata=dev_data) else: train_set = get_dataset(dset='train', dataset_config=dataset_config, njobs=njobs) dev_set = get_dataset(dset='dev', dataset_config=dataset_config, njobs=njobs) pk.dump(train_set.data, open(train_pkl, 'wb')) pk.dump(dev_set.data, open(dev_pkl, 'wb')) train_loader = get_dataloader(dset='train', dataloader_config=dataloader_config, dataset=train_set) dev_loader = get_dataloader(dset='dev', dataloader_config=dataloader_config, dataset=dev_set) return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def main(config): trainset = get_dataset(config.train_data_dir, config) blockSampler = RandomBatchSampler(trainset, config.batch_size, drop_last=True) train_loader = DataLoader(dataset=trainset, batch_sampler=blockSampler, collate_fn=CropCollate(0), num_workers=config.num_workers) solver = Solver(train_loader, config) solver.train()
def test_udd_ofr_rids(self): """test one of relation ID is larger than number of relations """ extra_train = ['0\t2\t1\n'] files = gen_udd_files(extra_train=extra_train) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_valid = ['0\t2\t1\n'] files = gen_udd_files(extra_valid=extra_valid) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files) extra_test = ['0\t2\t1\n'] files = gen_udd_files(extra_test=extra_test) with self.assertRaises(AssertionError): dataset = get_dataset('./', 'udd_test', 'udd_hrt', '\t', files) cleanup(files)
def main(): """ Main predict function for the wikikg90m """ args = ArgParser().parse_args() config = load_model_config( os.path.join(args.model_path, 'model_config.json')) args = use_config_replace_args(args, config) dataset = get_dataset(args, args.data_path, args.dataset, args.format, args.delimiter, args.data_files, args.has_edge_importance) print("Load the dataset done.") eval_dataset = EvalDataset(dataset, args) model = BaseKEModel( args=args, n_entities=dataset.n_entities, n_relations=dataset.n_relations, model_name=args.model_name, hidden_size=args.hidden_dim, entity_feat_dim=dataset.entity_feat.shape[1], relation_feat_dim=dataset.relation_feat.shape[1], gamma=args.gamma, double_entity_emb=args.double_ent, cpu_emb=args.cpu_emb, relation_times=args.ote_size, scale_type=args.scale_type) print("Create the model done.") model.entity_feat = dataset.entity_feat model.relation_feat = dataset.relation_feat load_model_from_checkpoint(model, args.model_path) print("The model load the checkpoint done.") if args.infer_valid: valid_sampler_tail = eval_dataset.create_sampler( 'valid', args.batch_size_eval, mode='tail', num_workers=args.num_workers, rank=0, ranks=1) infer(args, model, config, 0, [valid_sampler_tail], "valid") if args.infer_test: test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size_eval, mode='tail', num_workers=args.num_workers, rank=i, ranks=args.num_proc) infer(args, model, config, 0, [test_sampler_tail], "test")
def main(): print('===> Loading datasets') train_set = get_dataset(args.patch_size, args.upscale_factor, phase='train') train_loader = DataLoader(dataset=train_set, batch_size=args.bs, shuffle=True) test_set = get_dataset(0, args.upscale_factor, phase='test') # 0 has no meaning test_loader = DataLoader(dataset=test_set, batch_size=1, shuffle=False) if args.model == 'srcnn': model = SRCNNTrainer(args, train_loader, test_loader) elif args.model == 'srdcn': model = SRDCNTrainer(args, train_loader, test_loader) else: raise Exception("the model does not exist") model.run()
def main(config): dataset = get_dataset(config.data_dir, config) dataloader = DataLoader(dataset=dataset, batch_size=config.batch_size, shuffle=False, num_workers=config.num_workers, drop_last=False) evaluator = Evaluation(config) if not os.path.exists(config.post_dir): os.makedirs(config.post_dir) evaluator.extract_post_to_hardisk( dataloader, 'ark,scp:{0}/posts.ark,{0}/posts.scp'.format(config.post_dir))
def pretrain(args): tf = get_transform(args, 'none') ds = get_dataset(args, tf, 'none') args, model, ckpt_available = get_model_ckpt(args) if ckpt_available: print("loaded checkpoint {} in pretraining stage".format(args.ckpt_name)) loss_fn = get_loss(args) sub_optimizer = get_sub_optimizer(args, model) optimizer = get_optimizer(args, sub_optimizer) scheduler = get_scheduler(args, optimizer) # setup nvidia/apex amp # model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level, num_losses=1) # model = idist.auto_model(model) trainer = get_trainer(args, model, loss_fn, optimizer, scheduler)
def pretrain(args): tf = get_transform(args, 'none') ds = get_dataset(args, tf, 'none') args, model, ckpt_available = get_model_ckpt(args) if ckpt_available: print("loaded checkpoint {} in pretraining stage".format( args.ckpt_name)) loss_fn = get_loss(args) sub_optimizer = get_sub_optimizer(args, model) optimizer = get_optimizer(args, sub_optimizer) scheduler = get_scheduler(args, optimizer) # setup nvidia/apex amp # model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level, num_losses=1) # model = idist.auto_model(model) trainer = get_trainer(args, model, loss_fn, optimizer, scheduler) metrics = get_metrics(args) logger = get_logger(args) @trainer.on(Events.STARTED) def on_training_started(engine): print("Begin Pretraining") # batch-wise @trainer.on(Events.ITERATION_COMPLETED) def log_iter_results(engine): log_results(logger, 'pretrain/iter', engine.state, engine.state.iteration) # epoch-wise (ckpt) @trainer.on(Events.EPOCH_COMPLETED) def save_epoch(engine): log_results(logger, 'pretrain/epoch', engine.state, engine.state.epoch) log_results_cmd(logger, 'pretrain/epoch', engine.state, engine.state.epoch) save_ckpt(args, engine.state.epoch, engine.state.metrics['loss'], model) trainer.run(ds, max_epochs=args.epoch)
def main(): parser = argparse.ArgumentParser(description='Partition a knowledge graph') parser.add_argument('--data_path', type=str, default='data', help='root path of all dataset') parser.add_argument('--dataset', type=str, default='FB15k', help='dataset name, under data_path') parser.add_argument('--data_files', type=str, default=None, nargs='+', help='a list of data files, e.g. entity relation train valid test') parser.add_argument('--format', type=str, default='built_in', help='the format of the dataset, it can be built_in,'\ 'raw_udd_{htr} and udd_{htr}') parser.add_argument('-k', '--num-parts', required=True, type=int, help='The number of partitions') args = parser.parse_args() num_parts = args.num_parts # load dataset and samplers dataset = get_dataset(args.data_path, args.dataset, args.format, args.data_files) src, etype_id, dst = dataset.train coo = sp.sparse.coo_matrix((np.ones(len(src)), (src, dst)), shape=[dataset.n_entities, dataset.n_entities]) g = dgl.DGLGraph(coo, readonly=True, sort_csr=True) g.edata['tid'] = F.tensor(etype_id, F.int64) part_dict = dgl.transform.metis_partition(g, num_parts, 1) tot_num_inner_edges = 0 for part_id in part_dict: part = part_dict[part_id] num_inner_nodes = len(np.nonzero(F.asnumpy(part.ndata['inner_node']))[0]) num_inner_edges = len(np.nonzero(F.asnumpy(part.edata['inner_edge']))[0]) print('part {} has {} nodes and {} edges. {} nodes and {} edges are inside the partition'.format( part_id, part.number_of_nodes(), part.number_of_edges(), num_inner_nodes, num_inner_edges)) tot_num_inner_edges += num_inner_edges part.copy_from_parent() save_graphs(args.data_path + '/part_' + str(part_id) + '.dgl', [part]) print('there are {} edges in the graph and {} edge cuts for {} partitions.'.format( g.number_of_edges(), g.number_of_edges() - tot_num_inner_edges, len(part_dict)))
def eval_linear(pretrain_args, args): # get pretrained model pt_args, pt_model, ckpt_available = get_model_ckpt(pretrain_args) tf = get_transform(args, 'train') ds = get_dataset(args, tf, 'train') if ckpt_available: print("loaded pretrained model {} in eval linear".format(args.ckpt_name)) model = get_linear(args, pt_model, args.num_classes) loss_fn = get_loss(args) optimizer = get_sub_optimizer(args, model) scheduler = get_scheduler(args, optimizer) trainer = get_trainer(args, model, loss_fn, optimizer, scheduler) evaluator = get_evaluator(args, model, loss_fn) # metrics = get_metrics(args) logger = get_logger(args) trainer.run(ds, max_epochs=args.epoch)
def main(): args = get_args() os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda_ids device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if not os.path.exists(args.save_path): os.makedirs(args.save_path) if not os.path.exists(args.tensorboard_dir): os.makedirs(args.tensorboard_dir) logging = get_logging() seed_everything(args.seed) transforms_train = Compose([ RandomResizedCrop(args.img_size, args.img_size), Transpose(p=0.5), HorizontalFlip(p=0.5), VerticalFlip(p=0.5), ShiftScaleRotate(p=0.5), HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5), RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5), Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0), CoarseDropout(p=0.5), ToTensorV2(p=1.0), ], p=1.) transforms_val = Compose([ CenterCrop(args.img_size, args.img_size, p=1.), Resize(args.img_size, args.img_size), Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0), ToTensorV2(p=1.0), ], p=1.) train_loader, val_loader = get_dataset(args.data_path, args.batch_size, args.batch_size_val, transforms_train, transforms_val) net = get_model(True, device) trainer = Trainer(net, train_loader, val_loader, args, device, logging) trainer.train()
def main(args): # load dataset and samplers dataset = get_dataset(args.data_path, args.dataset, args.format) args.pickle_graph = False args.train = False args.valid = False args.test = True args.batch_size_eval = args.batch_size logger = get_logger(args) # Here we want to use the regualr negative sampler because we need to ensure that # all positive edges are excluded. eval_dataset = EvalDataset(dataset, args) args.neg_sample_size_test = args.neg_sample_size if args.neg_sample_size < 0: args.neg_sample_size_test = args.neg_sample_size = eval_dataset.g.number_of_nodes( ) if args.num_proc > 1: test_sampler_tails = [] test_sampler_heads = [] for i in range(args.num_proc): test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size, args.neg_sample_size, mode='PBG-head', num_workers=args.num_worker, rank=i, ranks=args.num_proc) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size, args.neg_sample_size, mode='PBG-tail', num_workers=args.num_worker, rank=i, ranks=args.num_proc) test_sampler_heads.append(test_sampler_head) test_sampler_tails.append(test_sampler_tail) else: test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size, args.neg_sample_size, mode='PBG-head', num_workers=args.num_worker, rank=0, ranks=1) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size, args.neg_sample_size, mode='PBG-tail', num_workers=args.num_worker, rank=0, ranks=1) # load model n_entities = dataset.n_entities n_relations = dataset.n_relations ckpt_path = args.model_path model = load_model_from_checkpoint(logger, args, n_entities, n_relations, ckpt_path) if args.num_proc > 1: model.share_memory() # test args.step = 0 args.max_step = 0 if args.num_proc > 1: procs = [] for i in range(args.num_proc): proc = mp.Process(target=test, args=(args, model, [ test_sampler_heads[i], test_sampler_tails[i] ])) procs.append(proc) proc.start() for proc in procs: proc.join() else: test(args, model, [test_sampler_head, test_sampler_tail])
def main(args): args.eval_filter = not args.no_eval_filter if args.neg_deg_sample: assert not args.eval_filter, "if negative sampling based on degree, we can't filter positive edges." # load dataset and samplers dataset = get_dataset(args.data_path, args.dataset, args.format) args.pickle_graph = False args.train = False args.valid = False args.test = True args.batch_size_eval = args.batch_size logger = get_logger(args) # Here we want to use the regualr negative sampler because we need to ensure that # all positive edges are excluded. eval_dataset = EvalDataset(dataset, args) args.neg_sample_size_test = args.neg_sample_size args.neg_deg_sample_eval = args.neg_deg_sample if args.neg_sample_size < 0: args.neg_sample_size_test = args.neg_sample_size = eval_dataset.g.number_of_nodes( ) if args.neg_chunk_size < 0: args.neg_chunk_size = args.neg_sample_size num_workers = args.num_worker # for multiprocessing evaluation, we don't need to sample multiple batches at a time # in each process. if args.num_proc > 1: num_workers = 1 if args.num_proc > 1: test_sampler_tails = [] test_sampler_heads = [] for i in range(args.num_proc): test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size, args.neg_sample_size, args.neg_chunk_size, args.eval_filter, mode='chunk-head', num_workers=num_workers, rank=i, ranks=args.num_proc) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size, args.neg_sample_size, args.neg_chunk_size, args.eval_filter, mode='chunk-tail', num_workers=num_workers, rank=i, ranks=args.num_proc) test_sampler_heads.append(test_sampler_head) test_sampler_tails.append(test_sampler_tail) else: test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size, args.neg_sample_size, args.neg_chunk_size, args.eval_filter, mode='chunk-head', num_workers=num_workers, rank=0, ranks=1) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size, args.neg_sample_size, args.neg_chunk_size, args.eval_filter, mode='chunk-tail', num_workers=num_workers, rank=0, ranks=1) # load model n_entities = dataset.n_entities n_relations = dataset.n_relations ckpt_path = args.model_path model = load_model_from_checkpoint(logger, args, n_entities, n_relations, ckpt_path) if args.num_proc > 1: model.share_memory() # test args.step = 0 args.max_step = 0 start = time.time() if args.num_proc > 1: queue = mp.Queue(args.num_proc) procs = [] for i in range(args.num_proc): proc = mp.Process(target=test, args=(args, model, [ test_sampler_heads[i], test_sampler_tails[i] ], 'Test', queue)) procs.append(proc) proc.start() for proc in procs: proc.join() total_metrics = {} for i in range(args.num_proc): metrics = queue.get() for k, v in metrics.items(): if i == 0: total_metrics[k] = v / args.num_proc else: total_metrics[k] += v / args.num_proc for k, v in metrics.items(): print('Test average {} at [{}/{}]: {}'.format( k, args.step, args.max_step, v)) else: test(args, model, [test_sampler_head, test_sampler_tail]) print('Test takes {:.3f} seconds'.format(time.time() - start))
def run(args, logger): # load dataset and samplers dataset = get_dataset(args.data_path, args.dataset, args.format) n_entities = dataset.n_entities n_relations = dataset.n_relations if args.neg_sample_size_test < 0: args.neg_sample_size_test = n_entities train_data = TrainDataset(dataset, args, ranks=args.num_proc) if args.num_proc > 1: train_samplers = [] for i in range(args.num_proc): train_sampler_head = train_data.create_sampler( args.batch_size, args.neg_sample_size, mode='PBG-head', num_workers=args.num_worker, shuffle=True, exclude_positive=True, rank=i) train_sampler_tail = train_data.create_sampler( args.batch_size, args.neg_sample_size, mode='PBG-tail', num_workers=args.num_worker, shuffle=True, exclude_positive=True, rank=i) train_samplers.append( NewBidirectionalOneShotIterator(train_sampler_head, train_sampler_tail, True, n_entities)) else: train_sampler_head = train_data.create_sampler( args.batch_size, args.neg_sample_size, mode='PBG-head', num_workers=args.num_worker, shuffle=True, exclude_positive=True) train_sampler_tail = train_data.create_sampler( args.batch_size, args.neg_sample_size, mode='PBG-tail', num_workers=args.num_worker, shuffle=True, exclude_positive=True) train_sampler = NewBidirectionalOneShotIterator( train_sampler_head, train_sampler_tail, True, n_entities) if args.valid or args.test: eval_dataset = EvalDataset(dataset, args) if args.valid: # Here we want to use the regualr negative sampler because we need to ensure that # all positive edges are excluded. if args.num_proc > 1: valid_sampler_heads = [] valid_sampler_tails = [] for i in range(args.num_proc): valid_sampler_head = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_valid, mode='PBG-head', num_workers=args.num_worker, rank=i, ranks=args.num_proc) valid_sampler_tail = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_valid, mode='PBG-tail', num_workers=args.num_worker, rank=i, ranks=args.num_proc) valid_sampler_heads.append(valid_sampler_head) valid_sampler_tails.append(valid_sampler_tail) else: valid_sampler_head = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_valid, mode='PBG-head', num_workers=args.num_worker, rank=0, ranks=1) valid_sampler_tail = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_valid, mode='PBG-tail', num_workers=args.num_worker, rank=0, ranks=1) if args.test: # Here we want to use the regualr negative sampler because we need to ensure that # all positive edges are excluded. if args.num_proc > 1: test_sampler_tails = [] test_sampler_heads = [] for i in range(args.num_proc): test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, mode='PBG-head', num_workers=args.num_worker, rank=i, ranks=args.num_proc) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, mode='PBG-tail', num_workers=args.num_worker, rank=i, ranks=args.num_proc) test_sampler_heads.append(test_sampler_head) test_sampler_tails.append(test_sampler_tail) else: test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, mode='PBG-head', num_workers=args.num_worker, rank=0, ranks=1) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, mode='PBG-tail', num_workers=args.num_worker, rank=0, ranks=1) # We need to free all memory referenced by dataset. eval_dataset = None dataset = None # load model model = load_model(logger, args, n_entities, n_relations) if args.num_proc > 1: model.share_memory() # train start = time.time() if args.num_proc > 1: procs = [] for i in range(args.num_proc): valid_samplers = [valid_sampler_heads[i], valid_sampler_tails[i] ] if args.valid else None proc = mp.Process(target=train, args=(args, model, train_samplers[i], valid_samplers)) procs.append(proc) proc.start() for proc in procs: proc.join() else: valid_samplers = [valid_sampler_head, valid_sampler_tail ] if args.valid else None train(args, model, train_sampler, valid_samplers) print('training takes {} seconds'.format(time.time() - start)) if args.save_emb is not None: if not os.path.exists(args.save_emb): os.mkdir(args.save_emb) model.save_emb(args.save_emb, args.dataset) # test if args.test: if args.num_proc > 1: procs = [] for i in range(args.num_proc): proc = mp.Process(target=test, args=(args, model, [ test_sampler_heads[i], test_sampler_tails[i] ])) procs.append(proc) proc.start() for proc in procs: proc.join() else: test(args, model, [test_sampler_head, test_sampler_tail])
from dataloader import get_dataset, transform import torch from torch import nn from torch.nn import functional as F import numpy as np from matplotlib import pyplot as plt from torchvision import transforms train_iter = get_dataset('train', transform=transform, batch_size=1) for idx, item in enumerate(train_iter): # print(item) data, annotation = item data, _ = data annotation, _ = annotation if idx % 100 == 0: print(idx) print('finished')
def run(args, logger): # load dataset and samplers dataset = get_dataset(args.data_path, args.dataset, args.format) n_entities = dataset.n_entities n_relations = dataset.n_relations if args.neg_sample_size_test < 0: args.neg_sample_size_test = n_entities args.eval_filter = not args.no_eval_filter if args.neg_deg_sample_eval: assert not args.eval_filter, "if negative sampling based on degree, we can't filter positive edges." # When we generate a batch of negative edges from a set of positive edges, # we first divide the positive edges into chunks and corrupt the edges in a chunk # together. By default, the chunk size is equal to the negative sample size. # Usually, this works well. But we also allow users to specify the chunk size themselves. if args.neg_chunk_size < 0: args.neg_chunk_size = args.neg_sample_size if args.neg_chunk_size_valid < 0: args.neg_chunk_size_valid = args.neg_sample_size_valid if args.neg_chunk_size_test < 0: args.neg_chunk_size_test = args.neg_sample_size_test train_data = TrainDataset(dataset, args, ranks=args.num_proc) if args.num_proc > 1: train_samplers = [] for i in range(args.num_proc): train_sampler_head = train_data.create_sampler( args.batch_size, args.neg_sample_size, args.neg_chunk_size, mode='chunk-head', num_workers=args.num_worker, shuffle=True, exclude_positive=True, rank=i) train_sampler_tail = train_data.create_sampler( args.batch_size, args.neg_sample_size, args.neg_chunk_size, mode='chunk-tail', num_workers=args.num_worker, shuffle=True, exclude_positive=True, rank=i) train_samplers.append( NewBidirectionalOneShotIterator(train_sampler_head, train_sampler_tail, args.neg_chunk_size, True, n_entities)) else: train_sampler_head = train_data.create_sampler( args.batch_size, args.neg_sample_size, args.neg_chunk_size, mode='chunk-head', num_workers=args.num_worker, shuffle=True, exclude_positive=True) train_sampler_tail = train_data.create_sampler( args.batch_size, args.neg_sample_size, args.neg_chunk_size, mode='chunk-tail', num_workers=args.num_worker, shuffle=True, exclude_positive=True) train_sampler = NewBidirectionalOneShotIterator( train_sampler_head, train_sampler_tail, args.neg_chunk_size, True, n_entities) # for multiprocessing evaluation, we don't need to sample multiple batches at a time # in each process. num_workers = args.num_worker if args.num_proc > 1: num_workers = 1 if args.valid or args.test: eval_dataset = EvalDataset(dataset, args) if args.valid: # Here we want to use the regualr negative sampler because we need to ensure that # all positive edges are excluded. if args.num_proc > 1: valid_sampler_heads = [] valid_sampler_tails = [] for i in range(args.num_proc): valid_sampler_head = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_valid, args.neg_chunk_size_valid, args.eval_filter, mode='chunk-head', num_workers=num_workers, rank=i, ranks=args.num_proc) valid_sampler_tail = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_valid, args.neg_chunk_size_valid, args.eval_filter, mode='chunk-tail', num_workers=num_workers, rank=i, ranks=args.num_proc) valid_sampler_heads.append(valid_sampler_head) valid_sampler_tails.append(valid_sampler_tail) else: valid_sampler_head = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_valid, args.neg_chunk_size_valid, args.eval_filter, mode='chunk-head', num_workers=num_workers, rank=0, ranks=1) valid_sampler_tail = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_valid, args.neg_chunk_size_valid, args.eval_filter, mode='chunk-tail', num_workers=num_workers, rank=0, ranks=1) if args.test: # Here we want to use the regualr negative sampler because we need to ensure that # all positive edges are excluded. if args.num_proc > 1: test_sampler_tails = [] test_sampler_heads = [] for i in range(args.num_proc): test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, args.neg_chunk_size_test, args.eval_filter, mode='chunk-head', num_workers=num_workers, rank=i, ranks=args.num_proc) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, args.neg_chunk_size_test, args.eval_filter, mode='chunk-tail', num_workers=num_workers, rank=i, ranks=args.num_proc) test_sampler_heads.append(test_sampler_head) test_sampler_tails.append(test_sampler_tail) else: test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, args.neg_chunk_size_test, args.eval_filter, mode='chunk-head', num_workers=num_workers, rank=0, ranks=1) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, args.neg_chunk_size_test, args.eval_filter, mode='chunk-tail', num_workers=num_workers, rank=0, ranks=1) # We need to free all memory referenced by dataset. eval_dataset = None dataset = None # load model model = load_model(logger, args, n_entities, n_relations) if args.num_proc > 1: model.share_memory() # train start = time.time() if args.num_proc > 1: procs = [] for i in range(args.num_proc): rel_parts = train_data.rel_parts if args.rel_part else None valid_samplers = [valid_sampler_heads[i], valid_sampler_tails[i] ] if args.valid else None proc = mp.Process(target=train, args=(args, model, train_samplers[i], i, rel_parts, valid_samplers)) procs.append(proc) proc.start() for proc in procs: proc.join() else: valid_samplers = [valid_sampler_head, valid_sampler_tail ] if args.valid else None train(args, model, train_sampler, valid_samplers) print('training takes {} seconds'.format(time.time() - start)) if args.save_emb is not None: if not os.path.exists(args.save_emb): os.mkdir(args.save_emb) model.save_emb(args.save_emb, args.dataset) # test if args.test: start = time.time() if args.num_proc > 1: queue = mp.Queue(args.num_proc) procs = [] for i in range(args.num_proc): proc = mp.Process(target=test, args=(args, model, [ test_sampler_heads[i], test_sampler_tails[i] ], i, 'Test', queue)) procs.append(proc) proc.start() total_metrics = {} for i in range(args.num_proc): metrics = queue.get() for k, v in metrics.items(): if i == 0: total_metrics[k] = v / args.num_proc else: total_metrics[k] += v / args.num_proc for k, v in metrics.items(): print('Test average {} at [{}/{}]: {}'.format( k, args.step, args.max_step, v)) for proc in procs: proc.join() else: test(args, model, [test_sampler_head, test_sampler_tail]) print('test:', time.time() - start)
def dist_train_test(args, model, train_sampler, entity_pb, relation_pb, l2g, rank=0, rel_parts=None, cross_rels=None, barrier=None): if args.num_proc > 1: th.set_num_threads(args.num_thread) client = connect_to_kvstore(args, entity_pb, relation_pb, l2g) client.barrier() train_time_start = time.time() train(args, model, train_sampler, None, rank, rel_parts, cross_rels, barrier, client) client.barrier() print('Total train time {:.3f} seconds'.format(time.time() - train_time_start)) model = None if client.get_id() % args.num_client == 0: # pull full model from kvstore args.num_test_proc = args.num_client dataset_full = get_dataset(args.data_path, args.dataset, args.format) print('Full data n_entities: ' + str(dataset_full.n_entities)) print("Full data n_relations: " + str(dataset_full.n_relations)) model_test = load_model(None, args, dataset_full.n_entities, dataset_full.n_relations) eval_dataset = EvalDataset(dataset_full, args) if args.test: model_test.share_memory() if args.neg_sample_size_test < 0: args.neg_sample_size_test = dataset_full.n_entities args.eval_filter = not args.no_eval_filter if args.neg_deg_sample_eval: assert not args.eval_filter, "if negative sampling based on degree, we can't filter positive edges." if args.neg_chunk_size_valid < 0: args.neg_chunk_size_valid = args.neg_sample_size_valid if args.neg_chunk_size_test < 0: args.neg_chunk_size_test = args.neg_sample_size_test print("Pull relation_emb ...") relation_id = F.arange(0, model_test.n_relations) relation_data = client.pull(name='relation_emb', id_tensor=relation_id) model_test.relation_emb.emb[relation_id] = relation_data print("Pull entity_emb ... ") # split model into 100 small parts start = 0 percent = 0 entity_id = F.arange(0, model_test.n_entities) count = int(model_test.n_entities / 100) end = start + count while True: print("Pull %d / 100 ..." % percent) if end >= model_test.n_entities: end = -1 tmp_id = entity_id[start:end] entity_data = client.pull(name='entity_emb', id_tensor=tmp_id) model_test.entity_emb.emb[tmp_id] = entity_data if end == -1: break start = end end += count percent += 1 if args.save_emb is not None: if not os.path.exists(args.save_emb): os.mkdir(args.save_emb) model_test.save_emb(args.save_emb, args.dataset) if args.test: args.num_thread = 1 test_sampler_tails = [] test_sampler_heads = [] for i in range(args.num_test_proc): test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, args.neg_chunk_size_test, args.eval_filter, mode='chunk-head', num_workers=args.num_thread, rank=i, ranks=args.num_test_proc) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_test, args.neg_chunk_size_test, args.eval_filter, mode='chunk-tail', num_workers=args.num_thread, rank=i, ranks=args.num_test_proc) test_sampler_heads.append(test_sampler_head) test_sampler_tails.append(test_sampler_tail) eval_dataset = None dataset_full = None print("Run test, test processes: %d" % args.num_test_proc) queue = mp.Queue(args.num_test_proc) procs = [] for i in range(args.num_test_proc): proc = mp.Process(target=test_mp, args=(args, model_test, [ test_sampler_heads[i], test_sampler_tails[i] ], i, 'Test', queue)) procs.append(proc) proc.start() total_metrics = {} metrics = {} logs = [] for i in range(args.num_test_proc): log = queue.get() logs = logs + log for metric in logs[0].keys(): metrics[metric] = sum([log[metric] for log in logs]) / len(logs) for k, v in metrics.items(): print('Test average {} at [{}/{}]: {}'.format( k, args.step, args.max_step, v)) for proc in procs: proc.join() if client.get_id() == 0: client.shut_down()
required=True, help='kernel : linear | polynomial | gaussian') parser.add_argument('--q', type=int, default=2, help='parameter for polynomial kernel') parser.add_argument('--sigma', type=float, default=1, help='parameter for gaussian kernel') parser.add_argument('--normalize', action='store_true', help='toggle for normalizing the input data') opt = parser.parse_args() train, test_x = get_dataset(root=opt.dataroot, normalize=opt.normalize) train_x, train_y = train[:, 0:train.shape[1] - 1], train[:, train.shape[1] - 1] cross_valid = StratifiedKFold(n_splits=5) tr_accuracies = [] va_accuracies = [] tr_times = [] for tr, va in cross_valid.split(train_x, train_y): print('Started cross validation split: {0}'.format(len(tr_accuracies) + 1)) print('Ratio: {0}/{1} :: TR/VA'.format(tr.shape[0], va.shape[0])) tr_x, va_x = train_x[tr], train_x[va] tr_y, va_y = train_y[tr], train_y[va]
def run(args, logger): init_time_start = time.time() # load dataset and samplers dataset = get_dataset(args.data_path, args.dataset, args.format, args.data_files) if args.neg_sample_size_eval < 0: args.neg_sample_size_eval = dataset.n_entities args.batch_size = get_compatible_batch_size(args.batch_size, args.neg_sample_size) args.batch_size_eval = get_compatible_batch_size(args.batch_size_eval, args.neg_sample_size_eval) args.eval_filter = not args.no_eval_filter if args.neg_deg_sample_eval: assert not args.eval_filter, "if negative sampling based on degree, we can't filter positive edges." train_data = TrainDataset(dataset, args, ranks=args.num_proc) # if there is no cross partition relaiton, we fall back to strict_rel_part args.strict_rel_part = args.mix_cpu_gpu and (train_data.cross_part == False) args.soft_rel_part = args.mix_cpu_gpu and args.soft_rel_part and train_data.cross_part args.num_workers = 8 # fix num_worker to 8 if args.num_proc > 1: train_samplers = [] for i in range(args.num_proc): train_sampler_head = train_data.create_sampler( args.batch_size, args.neg_sample_size, args.neg_sample_size, mode='head', num_workers=args.num_workers, shuffle=True, exclude_positive=False, rank=i) train_sampler_tail = train_data.create_sampler( args.batch_size, args.neg_sample_size, args.neg_sample_size, mode='tail', num_workers=args.num_workers, shuffle=True, exclude_positive=False, rank=i) train_samplers.append( NewBidirectionalOneShotIterator(train_sampler_head, train_sampler_tail, args.neg_sample_size, args.neg_sample_size, True, dataset.n_entities)) train_sampler = NewBidirectionalOneShotIterator( train_sampler_head, train_sampler_tail, args.neg_sample_size, args.neg_sample_size, True, dataset.n_entities) else: # This is used for debug train_sampler_head = train_data.create_sampler( args.batch_size, args.neg_sample_size, args.neg_sample_size, mode='head', num_workers=args.num_workers, shuffle=True, exclude_positive=False) train_sampler_tail = train_data.create_sampler( args.batch_size, args.neg_sample_size, args.neg_sample_size, mode='tail', num_workers=args.num_workers, shuffle=True, exclude_positive=False) train_sampler = NewBidirectionalOneShotIterator( train_sampler_head, train_sampler_tail, args.neg_sample_size, args.neg_sample_size, True, dataset.n_entities) if args.valid or args.test: if len(args.gpu) > 1: args.num_test_proc = args.num_proc if args.num_proc < len( args.gpu) else len(args.gpu) else: args.num_test_proc = args.num_proc eval_dataset = EvalDataset(dataset, args) if args.valid: if args.num_proc > 1: valid_sampler_heads = [] valid_sampler_tails = [] for i in range(args.num_proc): valid_sampler_head = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_eval, args.neg_sample_size_eval, args.eval_filter, mode='chunk-head', num_workers=args.num_workers, rank=i, ranks=args.num_proc) valid_sampler_tail = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_eval, args.neg_sample_size_eval, args.eval_filter, mode='chunk-tail', num_workers=args.num_workers, rank=i, ranks=args.num_proc) valid_sampler_heads.append(valid_sampler_head) valid_sampler_tails.append(valid_sampler_tail) else: # This is used for debug valid_sampler_head = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_eval, args.neg_sample_size_eval, args.eval_filter, mode='chunk-head', num_workers=args.num_workers, rank=0, ranks=1) valid_sampler_tail = eval_dataset.create_sampler( 'valid', args.batch_size_eval, args.neg_sample_size_eval, args.neg_sample_size_eval, args.eval_filter, mode='chunk-tail', num_workers=args.num_workers, rank=0, ranks=1) if args.test: if args.num_test_proc > 1: test_sampler_tails = [] test_sampler_heads = [] for i in range(args.num_test_proc): test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_eval, args.neg_sample_size_eval, args.eval_filter, mode='chunk-head', num_workers=args.num_workers, rank=i, ranks=args.num_test_proc) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_eval, args.neg_sample_size_eval, args.eval_filter, mode='chunk-tail', num_workers=args.num_workers, rank=i, ranks=args.num_test_proc) test_sampler_heads.append(test_sampler_head) test_sampler_tails.append(test_sampler_tail) else: test_sampler_head = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_eval, args.neg_sample_size_eval, args.eval_filter, mode='chunk-head', num_workers=args.num_workers, rank=0, ranks=1) test_sampler_tail = eval_dataset.create_sampler( 'test', args.batch_size_eval, args.neg_sample_size_eval, args.neg_sample_size_eval, args.eval_filter, mode='chunk-tail', num_workers=args.num_workers, rank=0, ranks=1) # load model model = load_model(logger, args, dataset.n_entities, dataset.n_relations) if args.num_proc > 1 or args.async_update: model.share_memory() # We need to free all memory referenced by dataset. eval_dataset = None dataset = None print('Total initialize time {:.3f} seconds'.format(time.time() - init_time_start)) # train start = time.time() rel_parts = train_data.rel_parts if args.strict_rel_part or args.soft_rel_part else None cross_rels = train_data.cross_rels if args.soft_rel_part else None if args.num_proc > 1: procs = [] barrier = mp.Barrier(args.num_proc) for i in range(args.num_proc): valid_sampler = [valid_sampler_heads[i], valid_sampler_tails[i] ] if args.valid else None proc = mp.Process(target=train_mp, args=(args, model, train_samplers[i], valid_sampler, i, rel_parts, cross_rels, barrier)) procs.append(proc) proc.start() for proc in procs: proc.join() else: valid_samplers = [valid_sampler_head, valid_sampler_tail ] if args.valid else None train(args, model, train_sampler, valid_samplers, rel_parts=rel_parts) print('training takes {} seconds'.format(time.time() - start)) if args.save_emb is not None: if not os.path.exists(args.save_emb): os.mkdir(args.save_emb) model.save_emb(args.save_emb, args.dataset) # We need to save the model configurations as well. conf_file = os.path.join(args.save_emb, 'config.json') with open(conf_file, 'w') as outfile: json.dump( { 'dataset': args.dataset, 'model': args.model_name, 'emb_size': args.hidden_dim, 'max_train_step': args.max_step, 'batch_size': args.batch_size, 'neg_sample_size': args.neg_sample_size, 'lr': args.lr, 'gamma': args.gamma, 'double_ent': args.double_ent, 'double_rel': args.double_rel, 'neg_adversarial_sampling': args.neg_adversarial_sampling, 'adversarial_temperature': args.adversarial_temperature, 'regularization_coef': args.regularization_coef, 'regularization_norm': args.regularization_norm }, outfile, indent=4) # test if args.test: start = time.time() if args.num_test_proc > 1: queue = mp.Queue(args.num_test_proc) procs = [] for i in range(args.num_test_proc): proc = mp.Process(target=test_mp, args=(args, model, [ test_sampler_heads[i], test_sampler_tails[i] ], i, 'Test', queue)) procs.append(proc) proc.start() total_metrics = {} metrics = {} logs = [] for i in range(args.num_test_proc): log = queue.get() logs = logs + log for metric in logs[0].keys(): metrics[metric] = sum([log[metric] for log in logs]) / len(logs) for k, v in metrics.items(): print('Test average {} : {}'.format(k, v)) for proc in procs: proc.join() else: test(args, model, [test_sampler_head, test_sampler_tail]) print('testing takes {:.3f} seconds'.format(time.time() - start))
d = { "costs": costs, "Y_prediction_train": Y_prediction_train, "Y_prediction_test": Y_prediction_test, "w": w, "b": b, "epoch": epoch, "learning_rate": learning_rate } return d if __name__ == "__main__": train_x_orig, train_y, test_x_orig, test_y, classes = get_dataset() # Reshape train/test samples train_x_flatten = train_x_orig.reshape(train_x_orig.shape[0], -1).T test_x_flatten = test_x_orig.reshape(test_x_orig.shape[0], -1).T # # Example of a picture # index = 2 # plt.imshow(train_x_orig[index]) # plt.show() # print("y = " + str(train_y[:, index]) + ", it's a '" + classes[np.squeeze(train_y[:, index])].decode( # "utf-8") + "' picture.") # print(train_x_orig.shape) # print(train_x_flatten.shape) # 对图片数据集进行标准化处理 train_x = train_x_flatten / 255 test_x = test_x_flatten / 255