def main(): initialize_dataset(args.dataset) build_symbolic_dataset = get_symbolic_dataset_builder(args.dataset) dataset = build_symbolic_dataset(args) dataloader = dataset.make_dataloader(32, False, False, nr_workers=4) meters = GroupMeters() for idx, feed_dict in tqdm_gofor(dataloader): feed_dict = GView(feed_dict) for i, (p, s, gt) in enumerate( zip(feed_dict.program_seq, feed_dict.scene, feed_dict.answer)): _, pred = execute_program(p, s) if pred[0] == 'error': raise pred[1] if pred[1] != gt: print(p) print(s) from IPython import embed embed() from sys import exit exit() meters.update('accuracy', pred[1] == gt) get_current_tqdm().set_description( meters.format_simple('Exec:', 'val', compressed=True)) logger.critical( meters.format_simple('Symbolic execution test:', 'avg', compressed=False))
def main(): args.dump_dir = ensure_path( osp.join('dumps', args.dataset_name, args.desc_name, args.expr)) args.ckpt_dir = ensure_path(osp.join(args.dump_dir, 'checkpoints')) args.meta_dir = ensure_path(osp.join(args.dump_dir, 'meta')) args.vis_dir = osp.join(args.dump_dir, 'vis', args.run_name) initialize_dataset(args.dataset) build_dataset = get_dataset_builder(args.dataset) dataset = build_dataset(args, configs, args.data_image_root, args.data_scenes_json, args.data_questions_json) dataset_split = int(len(dataset) * args.data_split) if args.data_split <= 1 else int( args.data_split) train_dataset, validation_dataset = dataset.split_trainval(dataset_split) logger.critical('Building the model.') model = desc.make_model(args, train_dataset.unwrapped.vocab) if args.use_gpu: model.cuda() # Use the customized data parallel if applicable. if args.gpu_parallel: from jactorch.parallel import JacDataParallel # from jactorch.parallel import UserScatteredJacDataParallel as JacDataParallel model = JacDataParallel(model, device_ids=args.gpus).cuda() # Disable the cudnn benchmark. cudnn.benchmark = False if args.load: from jactorch.io import load_weights if load_weights(model, args.load): logger.critical( 'Loaded weights from pretrained model: "{}".'.format( args.load)) from jacinle.utils.meter import GroupMeters meters = GroupMeters() if args.embed: from IPython import embed embed() logger.critical('Building the data loader.') validation_dataloader = validation_dataset.make_dataloader( args.batch_size, shuffle=True, drop_last=False, nr_workers=args.data_workers) model.eval() validate_epoch(0, model, validation_dataloader, meters) logger.critical( meters.format_simple('Validation', {k: v for k, v in meters.avg.items() if v != 0}, compressed=False)) return meters
def _mining_epoch(self, mining_epoch_size, mining_dataset_size): """Take exam, collect and update positive dataset and negative dataset""" pos_data = RandomlyIterDataset() neg_data = RandomlyIterDataset() self.model.eval() meters = GroupMeters() with tqdm_pbar(total=mining_epoch_size) as pbar: for i in range(mining_epoch_size): message, result = self._get_result(i, meters, mode='mining') positive, number, backup = self._extract_info(result) dataset = pos_data if positive else neg_data if dataset.size < mining_dataset_size: dataset.append((number, backup)) pbar.set_description(message) pbar.update() # When both positive and negative dataset are full, break. if pos_data.size >= mining_dataset_size and \ neg_data.size >= mining_dataset_size: break logger.info(meters.format_simple('> Mining: ', compressed=False)) self._inherit_neg_data(neg_data, self.neg_data, meters, mining_dataset_size) self.pos_data = pos_data self.neg_data = neg_data self._dump_meters(meters, 'mining') return meters
def main(): scenes = io.load_json(args.scene_json)['scenes'] preds = io.load(args.preds_json) if isinstance(preds, dict): preds = list(preds.values()) if False: preds = [transpose_scene(s) for s in preds] # flattened_objs = [o for s in scenes for o in s['objects']] # flattened_preds = { # k: np.concatenate([np.array(p[k]) for p in preds], axis=0) # for k in preds[0] # } meter = GroupMeters() ''' for i, scene in tqdm_gofor(scenes, mininterval=0.5): for j in range(len(scene['objects'])): test(j, scene['objects'], preds[i], meter) ''' for i, pred in tqdm_gofor(preds, mininterval=0.5): scene = scenes[i] for j in range(len(scene['objects'])): test(j, scene['objects'], pred, meter) print(meter.format_simple('Results:', compressed=False))
def main(): scenes = io.load_json(args.scene_json)['scenes'] preds = io.load(args.preds_json) if isinstance(preds, dict): preds = list(preds.values()) if False: preds = [transpose_scene(s) for s in preds] meter = GroupMeters() flattened_objs = [o for s in scenes for o in s['objects']] flattened_preds = { k: np.concatenate([np.array(p[k]) for p in preds], axis=0) for k in preds[0] } for k, preds in flattened_preds.items(): kk = def_.word2lemma.get(k, k) for i, o in tqdm_gofor(flattened_objs, desc='{}(lemma: {})'.format(k, kk), leave=False): meter.update( 'acc', (preds[i] > 0) == (kk == o[def_.concept2attribute[kk]])) meter.update( f'acc/{k}', (preds[i] > 0) == (kk == o[def_.concept2attribute[kk]])) print(meter.format_simple('Results:', compressed=False))
def _mining_epoch(self, mining_epoch_size, mining_dataset_size): """Take exam, collect and update positive dataset and negative dataset""" pos_data = RandomlyIterDataset() neg_data = RandomlyIterDataset() self.model.eval() meters_deter = GroupMeters() meters_stoch = GroupMeters() disable_pbar = False if os.getenv("ONCLUSTER") is not None: disable_pbar = True with tqdm_pbar(total=mining_epoch_size, disable=disable_pbar) as pbar: for i in range(mining_epoch_size): if i % 2 == 0: message, result = self._get_result(i, meters_deter, mode='mining-deter') else: message, result = self._get_result(i, meters_stoch, mode='mining-stoch') positive, number, backup = self._extract_info(result) dataset = pos_data if positive else neg_data if dataset.size < mining_dataset_size: dataset.append((number, backup)) pbar.set_description(message) pbar.update() # When both positive and negative dataset are full, break. if pos_data.size >= mining_dataset_size and \ neg_data.size >= mining_dataset_size: break logger.info( meters_deter.format_simple('> Mining (deter): ', compressed=False)) logger.info( meters_stoch.format_simple('> Mining (stoch): ', compressed=False)) meters = self.best_meters(meters_deter, meters_stoch) self._inherit_neg_data(neg_data, self.neg_data, meters, mining_dataset_size) self.pos_data = pos_data self.neg_data = neg_data self._dump_meters(meters_deter, 'mining-deter') self._dump_meters(meters_stoch, 'mining-stoch') return meters, meters_deter, meters_stoch
def _test_epoch(self, epoch_size): meters = GroupMeters() self._prepare_dataset(epoch_size, mode='test') def test_func(index): message, _ = self._get_result(index, meters, mode='test') return message tqdm_for(epoch_size, test_func) logger.info(meters.format_simple('> Evaluation: ', compressed=False)) self._dump_meters(meters, 'test') return meters
def main_train(validation_dataset): logger.critical('Building the model.') model = desc.make_model(args) if args.use_gpu: model.cuda() # Use the customized data parallel if applicable. if args.gpu_parallel: from jactorch.parallel import JacDataParallel # from jactorch.parallel import UserScatteredJacDataParallel as JacDataParallel model = JacDataParallel(model, device_ids=args.gpus).cuda() # Disable the cudnn benchmark. cudnn.benchmark = False trainer = TrainerEnv(model, None) if args.load: if trainer.load_weights(args.load): logger.critical( 'Loaded weights from pretrained model: "{}".'.format( args.load)) from jacinle.utils.meter import GroupMeters meters = GroupMeters() logger.critical('Building the data loader.') validation_dataloader = validation_dataset.make_dataloader( args.batch_size, shuffle=False, drop_last=False, nr_workers=args.data_workers) meters.reset() model.eval() if not os.path.isdir(args.output_attr_path): os.makedirs(args.output_attr_path) validate_attribute(model, validation_dataloader, meters, args.setname, logger, args.output_attr_path) logger.critical( meters.format_simple(args.setname, {k: v for k, v in meters.avg.items() if v != 0}, compressed=False)) return meters
def train(self, data_loader, nr_epochs, verbose=True, meters=None, early_stop=None, print_interval=1): if meters is None: meters = GroupMeters() for epoch in range(1, 1 + nr_epochs): meters.reset() self.train_epoch(data_loader, meters=meters) if verbose and epoch % print_interval == 0: caption = 'Epoch: {}:'.format(epoch) logger.info(meters.format_simple(caption)) if early_stop is not None: flag = early_stop(self._model) if flag: break
def _train_epoch(self, epoch_size): model = self.model meters = GroupMeters() self._prepare_dataset(epoch_size, mode='train') def train_func(index): model.eval() feed_dict = self._get_train_data(index, meters) model.train() message, _ = self._train_step(feed_dict, meters) return message # For $epoch_size times, do train_func with tqdm progress bar. tqdm_for(epoch_size, train_func) logger.info( meters.format_simple( '> Train Epoch {:5d}: '.format(self.current_epoch), compressed=False)) self._dump_meters(meters, 'train') return meters
def main(): scenes = io.load_json(args.scene_json)['scenes'] preds = io.load(args.preds_json) if isinstance(preds, dict): preds = list(preds.values()) if False: preds = [transpose_scene(s) for s in preds] scenes = scenes[:1000] preds = preds[:1000] flattened_objs = [o for s in scenes for o in s['objects']] flattened_preds = { k: np.concatenate([np.array(p[k]) for p in preds], axis=0) for k in preds[0] } meter = GroupMeters() for i, obj in tqdm_gofor(flattened_objs, mininterval=0.5): test(i, flattened_objs, flattened_preds, meter) print(meter.format_simple('Results:', compressed=False))