def dsc_run(args, workflow_args, content, verbosity = 1, jobs = None,
run_mode = 'run', queue = None):
env.verbosity = verbosity
env.max_jobs = args.__max_jobs__ if jobs is None else jobs
# kill all remaining processes when the master process is killed.
atexit.register(env.cleanup)
if args.__rerun__ or run_mode != 'run':
env.sig_mode = 'ignore'
elif run_mode == 'construct':
env.sig_mode = 'construct'
else:
env.sig_mode = 'default'
try:
script = SoS_Script(content=content, transcript = None)
workflow = script.workflow(args.workflow)
if queue is None or env.max_jobs == 1:
# single process executor
executor = Base_Executor(workflow, args=workflow_args, config_file=args.__config__)
elif queue is None:
executor = MP_Executor(workflow, args=workflow_args, config_file=args.__config__)
else:
executor = RQ_Executor(workflow, args=workflow_args, config_file=args.__config__)
if run_mode == 'dryrun':
executor.dryrun()
else:
executor.run()
except Exception as e:
if verbosity and verbosity > 2:
sys.stderr.write(get_traceback())
env.logger.error(e)
sys.exit(1)
env.verbosity = args.verbosity
def sos_run(args, workflow_args):
env.max_jobs = args.__max_jobs__
env.verbosity = args.verbosity
# kill all remainging processes when the master process is killed.
atexit.register(env.cleanup)
env.run_mode = 'run'
if args.__rerun__:
env.sig_mode = 'ignore'
try:
script = SoS_Script(content=args.script)
executor = Sequential_Executor(script.workflow(args.workflow))
executor.run(workflow_args, cmd_name=args.dsc_file, config_file = args.__config__)
except Exception as e:
if args.verbosity and args.verbosity > 2:
sys.stderr.write(get_traceback())
env.logger.error(e)
sys.exit(1)
def main():
def add_common_args(obj):
obj.add_argument('-v', '--verbosity', type = int, choices = list(range(5)), default = 2,
help='''Output error (0), warning (1), info (2), debug (3) and trace (4)
information.''')
obj.add_argument('--debug', action='store_true', help = argparse.SUPPRESS)
#
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('--version', action = 'version', version = '{} {}'.format(PACKAGE, VERSION))
subparsers = parser.add_subparsers(dest = 'subcommands')
subparsers.required = True
p = subparsers.add_parser('exec', help = 'Execute DSC',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('dsc_file', metavar = "dsc_file", help = 'DSC file')
p.add_argument('-s', '--sequence', metavar = "str", nargs = '+',
help = '''DSC sequence to be executed. It will override the DSC::run
entry when specified. Multiple sequences are allowed. Each input should be
a quoted string defining a valid DSC sequence. Multiple such strings should be
separated by space.''')
p.add_argument('-j', type=int, metavar='jobs', default=1, dest='__max_jobs__',
help='''Number of concurrent processes allowed.''')
p.add_argument('-d', action='store_true', dest='__dryrun__', help = argparse.SUPPRESS)
p.add_argument('-f', action='store_true', dest='__rerun__',
help='''Force executing DSC afresh regardless of already created results.''')
add_common_args(p)
p.set_defaults(func = execute)
p = subparsers.add_parser('rm', help = 'Remove output of given steps',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
p.add_argument('dsc_file', metavar = "dsc_file", help = 'DSC file')
p.add_argument('-s', '--step', metavar = "str", nargs = '+',
help = '''DSC steps whose output are to be removed. Multiple steps are allowed.
Each step should be a quoted string defining a valid DSC step, in the format of
"block_name[step_index]". Multiple such steps should be separated by space.''')
add_common_args(p)
p.set_defaults(func = remove)
args, argv = parser.parse_known_args()
try:
with Timer(verbose = True if ('verbosity' in vars(args) and args.verbosity > 0) else False):
args.func(args, argv)
except Exception as e:
if args.verbosity > 2:
sys.stderr.write(get_traceback())
else:
env.logger.error(e)
sys.exit(1)
def main():
p = argparse.ArgumentParser(description = __doc__)
p.add_argument('--version', action = 'version', version = '{} {}'.format(PACKAGE, VERSION))
p.add_argument('-v', '--verbosity', type = int, choices = list(range(5)), default = 2,
help='''Output error (0), warning (1), info (2), debug (3) and trace (4)
information.''')
p.add_argument('--debug', action='store_true', help = argparse.SUPPRESS)
p.add_argument('dsc_file', metavar = "DSC file", help = '')
p.add_argument('-s', '--sequence', metavar = "str", nargs = '+',
help = '''DSC sequence to be executed. It will override the DSC::run
entry when specified. Multiple sequences are allowed. Each input should be
a quoted string defining a valid DSC sequence. Multiple such strings should be
separated by space.''')
p.add_argument('-o', metavar = "str", dest = 'output',
help = '''DSC output filename/directory. When used, it will override the
specification in DSC script''')
p_execute = p.add_argument_group("Execute DSC")
p_execute.add_argument('-d', action='store_true', dest='__dryrun__', help = argparse.SUPPRESS)
p_execute.add_argument('-f', action='store_true', dest='__rerun__',
help='''Force executing DSC ignoring existing results.''')
p_execute.add_argument('--recover', action='store_true', dest='__construct__',
help='''Recover DSC based on names (not contents) of existing files.''')
p_execute.add_argument('-j', type=int, metavar='N', default=1, dest='__max_jobs__',
help='''Number of concurrent processes allowed.''')
p_execute.add_argument('--host', metavar='str',
help='''URL of Redis server for distributed computation.''')
p_remove = p.add_argument_group("Remove DSC")
p_remove.add_argument('-r', '--to_remove', metavar = "str", nargs = '+',
help = '''DSC steps whose output are to be removed. Multiple steps are allowed.
Each step should be a quoted string defining a valid DSC step, in the format of
"block_name[step_index]". Multiple such steps should be separated by space.''')
p.set_defaults(func = execute)
args, argv = p.parse_known_args()
try:
with Timer(verbose = True if ('verbosity' in vars(args) and args.verbosity > 0) else False):
args.func(args, argv)
except Exception as e:
if 'verbosity' in args and args.verbosity > 2:
sys.stderr.write(get_traceback())
else:
logger.error(e)
sys.exit(1)
