def cli_main(): import argparse parser = argparse.ArgumentParser( description= "Downloading/Decompressing CodeSearchNet dataset(s) or Tree-Sitter Library(ies)" ) parser.add_argument("--yaml_file", "-f", type=str, help="load {yaml_file}.yml for train", default='config/python_wan/python') parser.add_argument( '--out_file', '-o', type=str, help='output generated file', default=None, ) args = parser.parse_args() yaml_file = os.path.join(os.path.dirname(__file__), '{}.yml'.format(args.yaml_file)) out_file = args.out_file if out_file: dirname = os.path.dirname(out_file) assert os.path.isdir(dirname) os.makedirs(dirname, exist_ok=True) LOGGER.info('Load arguments in {}, output gnerated sentences at {}(if None, it won\'t record prediction).' \ .format(yaml_file, out_file)) args = load_yaml(yaml_file) LOGGER.info(args) torch.cuda.set_device(args['distributed_training']['device_id']) main(args, out_file)
def cli_main(): import argparse parser = argparse.ArgumentParser( description= "Downloading/Decompressing CodeSearchNet dataset(s) or Tree-Sitter Library(ies)" ) parser.add_argument("--yaml_file", "-f", type=str, help="load {language}.yml for train", default='config/csn_feng/ruby') parser.add_argument( '--out_file', '-o', type=str, help='output generated file', default=None, ) args = parser.parse_args() yaml_file = os.path.join(os.path.dirname(__file__), f"{args.yaml_file}.yml") out_file = None if args.out_file is None else recursive_expanduser( args.out_file) LOGGER.info('Load arguments in {}'.format(yaml_file)) args = load_yaml(yaml_file) LOGGER.info(args) main(args, out_file)
def cli_main(): Argues = namedtuple('Argues', 'yaml') args_ = Argues('preprocess_multilingual.yml') # train_sl LOGGER.info(args_) yaml_file = os.path.join(os.path.dirname(__file__), 'config', args_.yaml) LOGGER.info('Load arguments in {}'.format(yaml_file)) args = load_yaml(yaml_file) LOGGER.info(args) main(args)
def cli_main(): import argparse parser = argparse.ArgumentParser( description= "Downloading/Decompressing code_search_net dataset(s) or Tree-Sitter Library(ies)" ) parser.add_argument( "--yaml_file", "-f", type=str, help="load {yaml_file}.yml for train", default='config/csn_feng/python', ) args = parser.parse_args() yaml_file = os.path.join(os.path.dirname(__file__), '{}.yml'.format(args.yaml_file)) LOGGER.info('Load arguments in {}'.format(yaml_file)) args = load_yaml(yaml_file) LOGGER.info(args) if args['distributed_training']['distributed_init_method'] is None: distributed_utils.infer_init_method(args) if args['distributed_training']['distributed_init_method'] is not None: # distributed training if torch.cuda.device_count( ) > 1 and not args['distributed_training']['distributed_no_spawn']: start_rank = args['distributed_training']['distributed_rank'] args['distributed_training'][ 'distributed_rank'] = None # assign automatically torch.multiprocessing.spawn( fn=distributed_main, args=(args, start_rank), nprocs=torch.cuda.device_count(), ) else: distributed_main(args['distributed_training']['device_id'], args) elif args['distributed_training']['distributed_world_size'] > 1: # fallback for single node with multiple GPUs assert args['distributed_training'][ 'distributed_world_size'] <= torch.cuda.device_count() port = random.randint(10000, 20000) args['distributed_training'][ 'distributed_init_method'] = 'tcp://localhost:{port}'.format( port=port) args['distributed_training'][ 'distributed_rank'] = None # set based on device id torch.multiprocessing.spawn( fn=distributed_main, args=(args, ), nprocs=args['distributed_training']['distributed_world_size'], ) else: LOGGER.info('single GPU training...') single_main(args)
def cli_main(): import argparse parser = argparse.ArgumentParser( description="Downloading/Decompressing CodeSearchNet dataset(s) or Tree-Sitter Library(ies)") parser.add_argument( "--yaml_file", "-f", help="load {yaml_file}.yml for train", type=str, ) args = parser.parse_args() yaml_file = os.path.join(os.path.dirname(__file__), '{}.yml'.format(args.yaml_file)) LOGGER.info('Load arguments in {}'.format(yaml_file)) args = load_yaml(yaml_file) LOGGER.info(args) main(args)
def cli_main(): import argparse parser = argparse.ArgumentParser( description= "Downloading/Decompressing CodeSearchNet dataset(s) or Tree-Sitter Library(ies)" ) parser.add_argument( "--language", "-l", default='ruby', type=str, help="load {language}.yml for train", ) args = parser.parse_args() LOGGER.info(args) yaml_file = os.path.join( os.path.dirname(__file__), 'config', 'preprocess_multiattrs_{}.yml'.format(args.language)) LOGGER.info('Load arguments in {}'.format(yaml_file)) args = load_yaml(yaml_file) LOGGER.info(args) main(args)