def from_file(cls, filename: str, run_number: int, reset=False, seed: int=None, device: str='cuda', params=None):
""" Create model config from file """
with open(filename, 'r') as fp:
model_config_contents = Parser.parse(fp)
project_config_path = ModelConfig.find_project_directory(os.path.dirname(os.path.abspath(filename)))
with open(os.path.join(project_config_path, cls.PROJECT_FILE_NAME), 'r') as fp:
project_config_contents = Parser.parse(fp)
aggregate_dictionary = {
**project_config_contents,
**model_config_contents
}
# Options that should exist for every config
try:
model_name = model_config_contents['name']
except KeyError:
raise VelInitializationException("Model configuration must have a 'name' key")
return ModelConfig(
model_name=model_name,
filename=filename,
configuration=aggregate_dictionary,
run_number=run_number,
project_dir=project_config_path,
reset=reset,
seed=seed,
device=device,
parameters=params
)
def main():
""" Paperboy entry point - parse the arguments and run a command """
parser = argparse.ArgumentParser(
description='Paperboy deep learning launcher')
parser.add_argument('config',
metavar='FILENAME',
help='Configuration file for the run')
parser.add_argument('command', metavar='COMMAND', help='A command to run')
parser.add_argument('varargs',
nargs='*',
metavar='VARARGS',
help='Extra options to the command')
parser.add_argument('-r',
'--run_number',
type=int,
default=0,
help="A run number")
parser.add_argument('-d',
'--device',
default='cuda',
help="A device to run the model on")
parser.add_argument('-s',
'--seed',
type=int,
default=None,
help="Random seed for the project")
parser.add_argument('-p',
'--param',
type=str,
metavar='NAME=VALUE',
action='append',
default=[],
help="Configuration parameters")
parser.add_argument('--reset',
action='store_true',
default=False,
help="Overwrite existing model storage")
args = parser.parse_args()
model_config = ModelConfig.from_file(
args.config,
args.run_number,
reset=args.reset,
device=args.device,
seed=args.seed,
params={
k: v
for (k, v) in (Parser.parse_equality(eq) for eq in args.param)
})
# Set seed already in the launcher
set_seed(model_config.seed)
model_config.banner(args.command)
model_config.run_command(args.command, args.varargs)
model_config.quit_banner()
def main():
""" Paperboy entry point - parse the arguments and run a command """
parser = argparse.ArgumentParser(
description='Paperboy deep learning launcher')
parser.add_argument('config',
metavar='FILENAME',
help='Configuration file for the run')
parser.add_argument('command', metavar='COMMAND', help='A command to run')
parser.add_argument('varargs',
nargs='*',
metavar='VARARGS',
help='Extra options to the command')
parser.add_argument('-r',
'--run_number',
type=int,
default=0,
help="A run number")
parser.add_argument('-d',
'--device',
default='cuda',
help="A device to run the model on")
parser.add_argument('-s',
'--seed',
type=int,
default=None,
help="Random seed for the project")
parser.add_argument('-p',
'--param',
type=str,
metavar='NAME=VALUE',
action='append',
default=[],
help="Configuration parameters")
parser.add_argument('--continue',
action='store_true',
default=False,
help="Continue previously started learning process")
parser.add_argument('--profile',
type=str,
default=None,
help="Profiler output")
args = parser.parse_args()
model_config = ModelConfig.from_file(
args.config,
args.run_number,
continue_training=getattr(args, 'continue'),
device=args.device,
seed=args.seed,
params={
k: v
for (k, v) in (Parser.parse_equality(eq) for eq in args.param)
})
if model_config.project_dir not in sys.path:
sys.path.append(model_config.project_dir)
multiprocessing_setting = model_config.provide_with_default(
'multiprocessing', default=None)
if multiprocessing_setting:
# This needs to be called before any of PyTorch module is imported
multiprocessing.set_start_method(multiprocessing_setting)
# Set seed already in the launcher
from vel.util.random import set_seed
set_seed(model_config.seed)
model_config.banner(args.command)
if args.profile:
print("[PROFILER] Running Vel in profiling mode, output filename={}".
format(args.profile))
import cProfile
import pstats
profiler = cProfile.Profile()
profiler.enable()
model_config.run_command(args.command, args.varargs)
profiler.disable()
profiler.dump_stats(args.profile)
profiler.print_stats(sort='tottime')
print(
"======================================================================"
)
pstats.Stats(profiler).strip_dirs().sort_stats('tottime').print_stats(
30)
print(
"======================================================================"
)
pstats.Stats(profiler).strip_dirs().sort_stats('cumtime').print_stats(
30)
else:
model_config.run_command(args.command, args.varargs)
model_config.quit_banner()