def main(): args = get_args() data_path = os.path.join(args.iobasedir, 'processed/downloads', args.data_set) log_path = os.path.join(args.iobasedir, 'logs') log_file = os.path.join(args.iobasedir, 'logs', 'UB.log') mkdirp(log_path) set_logger(log_file) for filename in os.listdir(data_path): data_file = os.path.join(data_path, filename) topic = filename[:-5] docs, refs = load_data(data_file) if not refs: continue if not args.summary_size: summary_size = len(' '.join(refs[0]).split(' ')) else: summary_size = int(args.summary_size) logger.info('Topic ID: %s ', topic) logger.info('###') logger.info('Summmary_len: %d', summary_size) algos = ['UB1', 'UB2'] for algo in algos: get_summary_scores(algo, docs, refs, summary_size, language, rouge) logger.info('###')
def main(): args = get_args() rouge_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'rouge/RELEASE-1.5.5/') data_path = os.path.join(args.iobasedir, 'processed/', args.dataset, args.domain, args.split) log_path = os.path.join(args.iobasedir, 'logs') log_file = os.path.join( args.iobasedir, 'logs', 'baselines_rsumm_%s_%s_%s_%s.log' % (args.dataset, args.domain, args.split, str(args.summary_size))) mkdirp(log_path) set_logger(log_file) data_file = os.path.join(data_path, 'test0.csv') df = pd.read_csv(data_file, sep=",", quotechar='"', engine='python', header=None, skiprows=1, names=[ "user_id", "product_id", "rating", "review", "nouns", "summary", 'time' ]) # check_index = 1099 for index, row in df.iterrows(): # if index != check_index: # continue topic = row['user_id'] + '_' + row['product_id'] docs = [[sent] for sent in sent_tokenize(row['review'].strip())] refs = [sent_tokenize(row['summary'].strip())] if not refs: continue if not args.summary_size: summary_size = len(" ".join(refs[0]).split(' ')) else: summary_size = int(args.summary_size) logger.info('Topic ID: %s', topic) logger.info('###') logger.info('Summmary_len: %d', summary_size) rouge = Rouge(rouge_dir) algos = [ 'Luhn', 'LexRank', 'TextRank', 'LSA', 'KL', "ICSI", 'UB1', 'UB2' ] best_summary = [] best_score = 0.0 for algo in algos: best_summary, best_score = get_summary_scores( algo, docs, refs, summary_size, args.language, rouge, best_summary, best_score) rouge._cleanup() logger.info('###')
def setup(self): # Setup random seed for the experiment if none provided use 42 torch.manual_seed(self.parameters.get('random_seed', 42)) np.random.seed(self.parameters.get('random_seed', 42)) if torch.cuda.is_available(): torch.cuda.manual_seed(self.parameters.get('random_seed', 42)) torch.cuda.manual_seed_all(self.parameters.get('random_seed', 42)) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True # Create directories to store results for directory in ['log', 'results', 'models', 'tensorboard']: directory_path = os.path.join(self.experiment_path, directory) if not os.path.exists(directory_path): os.mkdir(directory_path) # Set the logger set_logger(os.path.join(self.experiment_path, 'log', 'experiment.log'))
def main(): args = get_args() rouge_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'rouge/RELEASE-1.5.5/') data_path = os.path.join(args.iobasedir, args.data_setpath) log_path = os.path.join(args.iobasedir, 'logs') log_file = os.path.join( args.iobasedir, 'logs', 'baselines_%s_%s.log' % (args.data_set, args.summary_size)) mkdirp(log_path) set_logger(log_file) for filename in os.listdir(data_path): data_file = os.path.join(data_path, filename) topic = filename[:-5] try: docs, refs = load_data(data_file) except: pass if not refs: continue if not args.summary_size: summary_size = len(" ".join(refs[0]).split(' ')) else: summary_size = int(args.summary_size) logger.info('Topic ID: %s', topic) logger.info('###') logger.info('Summmary_len: %d', summary_size) rouge = Rouge(rouge_dir) algos = ['UB1', 'UB2', 'ICSI', 'Luhn', 'LexRank', 'LSA', 'KL'] for algo in algos: get_summary_scores(algo, docs, refs, summary_size, args.language, rouge) rouge._cleanup() logger.info('###')
def main(): args = get_args_from_command_line() # Set GPU to use os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id # update config from configs.base_config import cfg, cfg_from_file, cfg_update if args.gan: cfg_from_file("configs/" + args.model + "_gan.yaml") else: cfg_from_file("configs/" + args.model + ".yaml") if args.test_mode is not None: cfg.TEST.mode = args.test_mode output_dir = cfg_update(args) # Set up folders for logs and checkpoints if not os.path.exists(cfg.DIR.logs): os.makedirs(cfg.DIR.logs) from utils.misc import set_logger logger = set_logger(os.path.join(cfg.DIR.logs, "log.txt")) logger.info("save into dir: %s" % cfg.DIR.logs) if "weights" not in cfg.CONST or not os.path.exists(cfg.CONST.weights): logger.error("Please specify the file path of checkpoint.") sys.exit(2) # Start inference process if args.gan: runners = __import__("runners." + args.model + "_gan_runner") module = getattr(runners, args.model + "_gan_runner") model = getattr(module, args.model + "GANRunner")(cfg, logger) else: runners = __import__("runners." + args.model + "_runner") module = getattr(runners, args.model + "_runner") model = getattr(module, args.model + "Runner")(cfg, logger) model.test()
def main(): args = get_args_from_command_line() # Set GPU to use os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id # update config from configs.base_config import cfg, cfg_from_file, cfg_update if args.gan: cfg_from_file("configs/" + args.model + "_gan.yaml") else: cfg_from_file("configs/" + args.model + ".yaml") output_dir = cfg_update(args) # Set up folders for logs and checkpoints if not os.path.exists(cfg.DIR.logs): os.makedirs(cfg.DIR.logs) from utils.misc import set_logger logger = set_logger(os.path.join(cfg.DIR.logs, "log.txt")) logger.info("save into dir: %s" % cfg.DIR.logs) # Start train/inference process if args.gan: runners = __import__("runners." + args.model + "_gan_runner") module = getattr(runners, args.model + "_gan_runner") model = getattr(module, args.model + "GANRunner")(cfg, logger) else: runners = __import__("runners." + args.model + "_runner") module = getattr(runners, args.model + "_runner") model = getattr(module, args.model + "Runner")(cfg, logger) model.runner()
# set run directory logs_dir = os.path.join(model_dir, 'runs') if args.run: log = args.run log_dir = os.path.join(logs_dir, args.run) else: log = sorted(glob.glob(os.path.join(logs_dir, '*')))[-1].split('/')[-1] log_dir = os.path.join(logs_dir, log) # Set the random seed for reproducible experiments torch.manual_seed(230) if params.cuda: torch.cuda.manual_seed(230) # Set the logger misc.set_logger(os.path.join(root, args.model_dir, 'test.log')) # Create the input data pipeline logging.info("\nLoading the datasets...") # fetch dataloaders data_dir = os.path.join(root, args.data_dir) # fetch dataloaders if args.dataloader == 'mnist': import data_loaders.mnist_data_loader as data_loader dataloaders = data_loader.fetch_dataloader(types=['test'], data_dir=data_dir, download=False, params=params) # If output needs to be reshaped into an image reshape = True
model_dir = os.path.join(root, args.model_dir) json_path = os.path.join(model_dir, 'params.json') assert os.path.isfile( json_path), "No json configuration file found at {}".format(json_path) params = Params(json_path) # use GPU if available params.cuda = torch.cuda.is_available() # Set the random seed for reproducible experiments torch.manual_seed(230) if params.cuda: torch.cuda.manual_seed(230) # Set the logger set_logger(os.path.join(root, args.model_dir, 'train.log')) # print out the arguments in a nice table tab_printer(args, "Argument Parameters") tab_printer(params, "Hyperparameters") # Create the input data pipeline logging.info("\nLoading the datasets...") data_dir = os.path.join(root, args.data_dir) # fetch dataloaders if args.dataloader == 'mnist': import data_loaders.mnist_data_loader as data_loader dataloaders = data_loader.fetch_dataloader(types=['train'], data_dir=data_dir, download=False, params=params)