def cli_main():
import argparse
parser = argparse.ArgumentParser(
description=
"Downloading/Decompressing CodeSearchNet dataset(s) or Tree-Sitter Library(ies)"
)
parser.add_argument("--yaml_file",
"-f",
type=str,
help="load {yaml_file}.yml for train",
default='config/python_wan/python')
parser.add_argument(
'--out_file',
'-o',
type=str,
help='output generated file',
default=None,
)
args = parser.parse_args()
yaml_file = os.path.join(os.path.dirname(__file__),
'{}.yml'.format(args.yaml_file))
out_file = args.out_file
if out_file:
dirname = os.path.dirname(out_file)
assert os.path.isdir(dirname)
os.makedirs(dirname, exist_ok=True)
LOGGER.info('Load arguments in {}, output gnerated sentences at {}(if None, it won\'t record prediction).' \
.format(yaml_file, out_file))
args = load_yaml(yaml_file)
LOGGER.info(args)
torch.cuda.set_device(args['distributed_training']['device_id'])
main(args, out_file)
def cli_main():
import argparse
parser = argparse.ArgumentParser(
description=
"Downloading/Decompressing CodeSearchNet dataset(s) or Tree-Sitter Library(ies)"
)
parser.add_argument("--yaml_file",
"-f",
type=str,
help="load {language}.yml for train",
default='config/csn_feng/ruby')
parser.add_argument(
'--out_file',
'-o',
type=str,
help='output generated file',
default=None,
)
args = parser.parse_args()
yaml_file = os.path.join(os.path.dirname(__file__),
f"{args.yaml_file}.yml")
out_file = None if args.out_file is None else recursive_expanduser(
args.out_file)
LOGGER.info('Load arguments in {}'.format(yaml_file))
args = load_yaml(yaml_file)
LOGGER.info(args)
main(args, out_file)
def cli_main():
Argues = namedtuple('Argues', 'yaml')
args_ = Argues('preprocess_multilingual.yml') # train_sl
LOGGER.info(args_)
yaml_file = os.path.join(os.path.dirname(__file__), 'config', args_.yaml)
LOGGER.info('Load arguments in {}'.format(yaml_file))
args = load_yaml(yaml_file)
LOGGER.info(args)
main(args)
def cli_main():
import argparse
parser = argparse.ArgumentParser(
description=
"Downloading/Decompressing code_search_net dataset(s) or Tree-Sitter Library(ies)"
)
parser.add_argument(
"--yaml_file",
"-f",
type=str,
help="load {yaml_file}.yml for train",
default='config/csn_feng/python',
)
args = parser.parse_args()
yaml_file = os.path.join(os.path.dirname(__file__),
'{}.yml'.format(args.yaml_file))
LOGGER.info('Load arguments in {}'.format(yaml_file))
args = load_yaml(yaml_file)
LOGGER.info(args)
if args['distributed_training']['distributed_init_method'] is None:
distributed_utils.infer_init_method(args)
if args['distributed_training']['distributed_init_method'] is not None:
# distributed training
if torch.cuda.device_count(
) > 1 and not args['distributed_training']['distributed_no_spawn']:
start_rank = args['distributed_training']['distributed_rank']
args['distributed_training'][
'distributed_rank'] = None # assign automatically
torch.multiprocessing.spawn(
fn=distributed_main,
args=(args, start_rank),
nprocs=torch.cuda.device_count(),
)
else:
distributed_main(args['distributed_training']['device_id'], args)
elif args['distributed_training']['distributed_world_size'] > 1:
# fallback for single node with multiple GPUs
assert args['distributed_training'][
'distributed_world_size'] <= torch.cuda.device_count()
port = random.randint(10000, 20000)
args['distributed_training'][
'distributed_init_method'] = 'tcp://localhost:{port}'.format(
port=port)
args['distributed_training'][
'distributed_rank'] = None # set based on device id
torch.multiprocessing.spawn(
fn=distributed_main,
args=(args, ),
nprocs=args['distributed_training']['distributed_world_size'],
)
else:
LOGGER.info('single GPU training...')
single_main(args)
def cli_main():
import argparse
parser = argparse.ArgumentParser(
description="Downloading/Decompressing CodeSearchNet dataset(s) or Tree-Sitter Library(ies)")
parser.add_argument(
"--yaml_file", "-f", help="load {yaml_file}.yml for train", type=str,
)
args = parser.parse_args()
yaml_file = os.path.join(os.path.dirname(__file__), '{}.yml'.format(args.yaml_file))
LOGGER.info('Load arguments in {}'.format(yaml_file))
args = load_yaml(yaml_file)
LOGGER.info(args)
main(args)
def cli_main():
import argparse
parser = argparse.ArgumentParser(
description=
"Downloading/Decompressing CodeSearchNet dataset(s) or Tree-Sitter Library(ies)"
)
parser.add_argument(
"--language",
"-l",
default='ruby',
type=str,
help="load {language}.yml for train",
)
args = parser.parse_args()
LOGGER.info(args)
yaml_file = os.path.join(
os.path.dirname(__file__), 'config',
'preprocess_multiattrs_{}.yml'.format(args.language))
LOGGER.info('Load arguments in {}'.format(yaml_file))
args = load_yaml(yaml_file)
LOGGER.info(args)
main(args)
