def main(): config_help = '\n\nConfig parameters:\n\n' + '\n'.join(ConfigSchema.help()) parser = argparse.ArgumentParser( epilog=config_help, # Needed to preserve line wraps in epilog. formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('config', help='Path to config file') parser.add_argument('-p', '--param', action='append', nargs='*') parser.add_argument('edge_paths', type=Path, nargs='*', help='Input file paths') parser.add_argument('-l', '--lhs-col', type=int, required=True, help='Column index for source entity') parser.add_argument('-r', '--rhs-col', type=int, required=True, help='Column index for target entity') parser.add_argument('--rel-col', type=int, help='Column index for relation entity') parser.add_argument('--relation-type-min-count', type=int, default=1, help='Min count for relation types') parser.add_argument('--entity-min-count', type=int, default=1, help='Min count for entities') opt = parser.parse_args() loader = ConfigFileLoader() config_dict = loader.load_raw_config(opt.config) if opt.param is not None: overrides = chain.from_iterable(opt.param) # flatten config_dict = override_config_dict(config_dict, overrides) entity_configs, relation_configs, entity_path, dynamic_relations = \ parse_config_partial(config_dict) convert_input_data( entity_configs, relation_configs, entity_path, opt.edge_paths, opt.lhs_col, opt.rhs_col, opt.rel_col, opt.entity_min_count, opt.relation_type_min_count, dynamic_relations, )
def main(): parser = argparse.ArgumentParser(description='Example on Livejournal') parser.add_argument('--config', default=DEFAULT_CONFIG, help='Path to config file') parser.add_argument('-p', '--param', action='append', nargs='*') parser.add_argument('--data_dir', default='data', help='where to save processed data') args = parser.parse_args() if args.param is not None: overrides = chain.from_iterable(args.param) # flatten else: overrides = None # download data data_dir = args.data_dir os.makedirs(data_dir, exist_ok=True) fpath = utils.download_url(URL, data_dir) fpath = utils.extract_gzip(fpath) print('Downloaded and extracted file.') # random split file for train and test random_split_file(fpath) loader = ConfigFileLoader() config = loader.load_config(args.config, overrides) edge_paths = [os.path.join(data_dir, name) for name in FILENAMES.values()] convert_input_data( config.entities, config.relations, config.entity_path, edge_paths, lhs_col=0, rhs_col=1, rel_col=None, dynamic_relations=config.dynamic_relations, ) train_path = [convert_path(os.path.join(data_dir, FILENAMES['train']))] train_config = attr.evolve(config, edge_paths=train_path) train( train_config, subprocess_init=partial(add_to_sys_path, loader.config_dir.name), ) eval_path = [convert_path(os.path.join(data_dir, FILENAMES['test']))] eval_config = attr.evolve(config, edge_paths=eval_path) do_eval( eval_config, subprocess_init=partial(add_to_sys_path, loader.config_dir.name), )
def main(): setup_logging() parser = argparse.ArgumentParser(description='Example on Livejournal') parser.add_argument('--config', default=DEFAULT_CONFIG, help='Path to config file') parser.add_argument('-p', '--param', action='append', nargs='*') parser.add_argument('--data_dir', type=Path, default='data', help='where to save processed data') args = parser.parse_args() if args.param is not None: overrides = chain.from_iterable(args.param) # flatten else: overrides = None # download data data_dir = args.data_dir data_dir.mkdir(parents=True, exist_ok=True) fpath = download_url(URL, data_dir) fpath = extract_gzip(fpath) print('Downloaded and extracted file.') # random split file for train and test random_split_file(fpath) loader = ConfigFileLoader() config = loader.load_config(args.config, overrides) set_logging_verbosity(config.verbose) subprocess_init = SubprocessInitializer() subprocess_init.register(setup_logging, config.verbose) subprocess_init.register(add_to_sys_path, loader.config_dir.name) edge_paths = [data_dir / name for name in FILENAMES.values()] convert_input_data( config.entities, config.relations, config.entity_path, edge_paths, lhs_col=0, rhs_col=1, rel_col=None, dynamic_relations=config.dynamic_relations, ) train_path = [str(convert_path(data_dir / FILENAMES['train']))] train_config = attr.evolve(config, edge_paths=train_path) train(train_config, subprocess_init=subprocess_init) eval_path = [str(convert_path(data_dir / FILENAMES['test']))] eval_config = attr.evolve(config, edge_paths=eval_path) do_eval(eval_config, subprocess_init=subprocess_init)
def main(): setup_logging() parser = argparse.ArgumentParser(description='Example on Livejournal') parser.add_argument('--config', default=DEFAULT_CONFIG, help='Path to config file') parser.add_argument('-p', '--param', action='append', nargs='*') parser.add_argument('--data_dir', type=Path, default='data', help='where to save processed data') args = parser.parse_args() # download data data_dir = args.data_dir data_dir.mkdir(parents=True, exist_ok=True) fpath = download_url(URL, data_dir) fpath = extract_gzip(fpath) print('Downloaded and extracted file.') # random split file for train and test random_split_file(fpath) loader = ConfigFileLoader() config = loader.load_config(args.config, args.param) set_logging_verbosity(config.verbose) subprocess_init = SubprocessInitializer() subprocess_init.register(setup_logging, config.verbose) subprocess_init.register(add_to_sys_path, loader.config_dir.name) input_edge_paths = [data_dir / name for name in FILENAMES] output_train_path, output_test_path = config.edge_paths convert_input_data( config.entities, config.relations, config.entity_path, config.edge_paths, input_edge_paths, TSVEdgelistReader(lhs_col=0, rhs_col=1, rel_col=None), dynamic_relations=config.dynamic_relations, ) train_config = attr.evolve(config, edge_paths=[output_train_path]) train(train_config, subprocess_init=subprocess_init) eval_config = attr.evolve(config, edge_paths=[output_test_path]) do_eval(eval_config, subprocess_init=subprocess_init)
def main(): config_help = '\n\nConfig parameters:\n\n' + '\n'.join(ConfigSchema.help()) parser = argparse.ArgumentParser( epilog=config_help, # Needed to preserve line wraps in epilog. formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('config', help="Path to config file") parser.add_argument('-p', '--param', action='append', nargs='*') opt = parser.parse_args() if opt.param is not None: overrides = chain.from_iterable(opt.param) # flatten else: overrides = None loader = ConfigFileLoader() config = loader.load_config(opt.config, overrides) do_eval(config, subprocess_init=partial(add_to_sys_path, loader.config_dir.name))
def main(): setup_logging() config_help = '\n\nConfig parameters:\n\n' + '\n'.join(ConfigSchema.help()) parser = argparse.ArgumentParser( epilog=config_help, # Needed to preserve line wraps in epilog. formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('config', help="Path to config file") parser.add_argument('-p', '--param', action='append', nargs='*') opt = parser.parse_args() loader = ConfigFileLoader() config = loader.load_config(opt.config, opt.param) set_logging_verbosity(config.verbose) subprocess_init = SubprocessInitializer() subprocess_init.register(setup_logging, config.verbose) subprocess_init.register(add_to_sys_path, loader.config_dir.name) do_eval(config, subprocess_init=subprocess_init)