def testDynamicNestedWorkflow(self): '''Test nested workflow controlled by command line option''' script = SoS_Script(''' if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] [parameters] wf='a' [a_1] [a_2] [a_3] [b_1] [b_2] [b_3] [default] sos_run(wf) ''') env.shared_vars = ['executed'] env.run_mode = 'dryrun' wf = script.workflow() wf.run(args=['--wf', 'b']) self.assertEqual(env.sos_dict['executed'], ['default_parameters', 'default_0', 'b_parameters', 'b_1', 'b_2', 'b_3']) # wf.run(args=['--wf', 'a']) self.assertEqual(env.sos_dict['executed'], ['default_parameters', 'default_0', 'a_parameters', 'a_1', 'a_2', 'a_3'])
def testDescriptions(self): '''Test script and workflow descriptions''' script = SoS_Script('''# first block # global # description # human # description of human # description of human continued [human_1] a = '1' # mouse # mouse description # [mouse_1] ''') self.assertEqual(script.description, 'global\ndescription\n\n') self.assertEqual(script.workflow('human').description, 'description of human\ndescription of human continued\n') self.assertEqual(script.workflow('mouse').description, 'mouse description\n')
def testInput(self): '''Test input directive''' script = SoS_Script(''' [0] files = ['a.txt', 'b.txt'] input: 'a.pdf', files, skip=False ''') env.run_mode = 'dryrun' script.workflow('default').run() # # test input types script = SoS_Script(''' [0:alias='test'] files = ('a${i}' for i in range(2)) input: {'a.txt', 'b.txt'}, files output: ('a${x}' for x in _input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(sorted(env.sos_dict['test'].input), ['a.txt', 'a0', 'a1', 'b.txt']) self.assertEqual(sorted(env.sos_dict['test'].output), ['aa.txt', 'aa0', 'aa1', 'ab.txt'])
def dsc_run(args, workflow_args, content, verbosity = 1, jobs = None, run_mode = 'run', queue = None): env.verbosity = verbosity env.max_jobs = args.__max_jobs__ if jobs is None else jobs # kill all remaining processes when the master process is killed. atexit.register(env.cleanup) if args.__rerun__ or run_mode != 'run': env.sig_mode = 'ignore' elif run_mode == 'construct': env.sig_mode = 'construct' else: env.sig_mode = 'default' try: script = SoS_Script(content=content, transcript = None) workflow = script.workflow(args.workflow) if queue is None or env.max_jobs == 1: # single process executor executor = Base_Executor(workflow, args=workflow_args, config_file=args.__config__) elif queue is None: executor = MP_Executor(workflow, args=workflow_args, config_file=args.__config__) else: executor = RQ_Executor(workflow, args=workflow_args, config_file=args.__config__) if run_mode == 'dryrun': executor.dryrun() else: executor.run() except Exception as e: if verbosity and verbosity > 2: sys.stderr.write(get_traceback()) env.logger.error(e) sys.exit(1) env.verbosity = args.verbosity
def testStringLiteral(self): '''Test string literals of SoS''' env.shared_vars = ['a', 'b', 'c'] script = SoS_Script(r""" [0] a = 'a\n' b = "a\n" c = '''a\n b''' """) wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['a'], 'a\n') self.assertEqual(env.sos_dict['b'], 'a\n') # MAYJOR difference self.assertEqual(env.sos_dict['c'], 'a\\n\nb') script = SoS_Script(r''' [0] c = """a\n b""" ''') wf = script.workflow() wf.run() # Note the difference between """ and ''' quotes self.assertEqual(env.sos_dict['c'], 'a\n\nb')
def testGlobalVariables(self): '''Test definition of variables''' # global section cannot have directive self.assertRaises(ParsingError, SoS_Script, '''input: 'filename' ''') # or unrecognized directive self.assertRaises(ParsingError, SoS_Script, '''inputs: 'filename' ''') # allow definition SoS_Script('''a = '1' ''') SoS_Script('''a = ['a', 'b'] ''') # but this one has incorrect syntax self.assertRaises(ParsingError, SoS_Script, '''a = 'b ''') # This one also does not work because b is not defined. delattr(env, 'sos_dict') script = SoS_Script('''a = b\n[0] ''') wf = script.workflow() self.assertRaises(RuntimeError, wf.run) # multi-line string literal SoS_Script('''a = """ this is a multi line string """ ''') # multi-line list literal, even with newline in between SoS_Script('''a = [ 'a', 'b' ] ''') # script = SoS_Script(filename='scripts/section1.sos')
def testYAMLConfig(self): '''Test config file in yaml format''' with open('config.yaml', 'w') as config: config.write(''' # Lines beginning with # are skipped when the JSON is parsed, so we can # put comments into our JSON configuration files { StoreOwner : "John Doe", # List of items that we sell Fruits: [ "apple", "banana", "pear" ], Price: 1.05 } ''') with open('config.sos', 'w') as sos: sos.write(''' [0] print(CONFIG['StoreOwner']) print(CONFIG.get('StoreOwner', 'something')) print(CONFIG.get('StoreOwnerSpouse', 'someone else')) print(CONFIG.StoreOwner) ''' ) # run the command self.assertEqual(subprocess.call('sos run config.sos -c config.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 0) # now test the value script = SoS_Script(filename='config.sos') wf = script.workflow() wf.run(config_file='config.yaml') self.assertEqual(env.sos_dict['CONFIG']['Price'], 1.05) self.assertEqual(env.sos_dict['CONFIG']['StoreOwner'], 'John Doe') self.assertEqual(env.sos_dict['CONFIG']['Fruits'], ['apple', 'banana', 'pear']) # configuration items should be readonly with open('config.sos', 'w') as sos: sos.write(''' [0] CONFIG['a'] = 'b' ''' ) # the command would fail with error message # ERROR: Failed to process statement CONFIG['a'] = 'b' # : Cannot modify a readonly dictionary. self.assertEqual(subprocess.call('sos run config.sos -c config.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1) # with open('config.sos', 'w') as sos: sos.write(''' [0] CONFIG.a = 'b' ''' ) # the command would fail with error message # ERROR: Failed to process statement CONFIG['a'] = 'b' # : Cannot modify a readonly dictionary. self.assertEqual(subprocess.call('sos run config.sos -c config.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1) # for filename in ['config.sos', 'config.yaml']: os.remove(filename)
def install_r_libs(libs): if libs is None: return verbosity = env.verbosity for value in libs: groups = re.search('(.*?)\((.*?)\)', value) if groups is not None: value = groups.group(1).strip() versions = [x.strip() for x in groups.group(2).split(',')] else: versions = None env.logger.info("Checking R library ``{}`` ...".format(value)) env.verbosity = 0 script = SoS_Script('[0]\ncheck_R_library({}, {})'.format(repr(value), repr(versions))) Base_Executor(script.workflow()).prepare() env.verbosity = verbosity
def sos_run(args, workflow_args): env.max_jobs = args.__max_jobs__ env.verbosity = args.verbosity # kill all remainging processes when the master process is killed. atexit.register(env.cleanup) env.run_mode = 'run' if args.__rerun__: env.sig_mode = 'ignore' try: script = SoS_Script(content=args.script) executor = Sequential_Executor(script.workflow(args.workflow)) executor.run(workflow_args, cmd_name=args.dsc_file, config_file = args.__config__) except Exception as e: if args.verbosity and args.verbosity > 2: sys.stderr.write(get_traceback()) env.logger.error(e) sys.exit(1)
def sos_run(args, workflow_args): env.verbosity = args.verbosity env.max_jobs = args.__max_jobs__ # kill all remainging processes when the master process is killed. atexit.register(env.cleanup) try: script = SoS_Script(content=args.script) workflow = script.workflow(args.workflow) if args.__dryrun__: env.run_mode = 'dryrun' if args.__rerun__: env.sig_mode = 'ignore' workflow.run(workflow_args, cmd_name=args.dsc_file) except Exception as e: if args.verbosity and args.verbosity > 2: print_traceback() env.logger.error(e) sys.exit(1)
def testSourceOption(self): '''Test the source section option''' # nested subworkflow with step option and others env.shared_vars = ['executed', 'GLB', 'parB'] if not os.path.isdir('temp'): os.mkdir('temp') with open('temp/test.sos', 'w') as sos: sos.write(''' # test sos script # global definition GLB = 5 if 'executed' in locals(): executed.append('t.' + step_name) else: executed = ['t.' + step_name] [parameters] parB = 10 [A_1] output: _input[0] + 'a1' [A_2] output: _input[0] + 'a2' ''') script = SoS_Script(''' if 'executed' in locals(): executed.append('g.' + step_name) else: executed = ['g.' + step_name] [b_1: skip=False] input: 'a.txt', 'b.txt', group_by='single' sos_run('A', source='temp/test.sos') ''') env.run_mode = 'dryrun' wf = script.workflow('b') wf.run() self.assertEqual(env.sos_dict['GLB'], 5) self.assertEqual(env.sos_dict['parB'], 10) self.assertEqual(env.sos_dict['executed'], ['g.b_1', 't.A_parameters', 't.A_1', 't.A_2', 't.A_parameters', 't.A_1', 't.A_2']) # shutil.rmtree('temp')
def testSkipStep(self): '''Test the skip option to skip certain steps''' script = SoS_Script(''' [parameters] skip = 0 [0: alias='a', skip=skip==0] var = 0 [1: alias='b', skip=skip==1] var = 1 ''') wf = script.workflow() wf.run(args=['--skip', '0']) self.assertEqual(env.sos_dict['b'].var, 1) # wf.run(args=['--skip', '1']) self.assertEqual(env.sos_dict['a'].var, 0)
def testScriptFormat(self): '''Test putting scripts directly under action''' script = SoS_Script(''' [0] input: 'filename', 'filename2', opt=value==1 R: open('something') save.put() ''') script = SoS_Script(''' [0] input: 'filename', 'filename2', opt=value==1 R: concurrent = True open('something') save.put() ''') script = SoS_Script(''' [0] input: 'filename', 'filename2', opt=value==1 R: concurrent = True, workdir = 'someelse else' open('something') save.put() ''') # test dedent script = SoS_Script(''' [0] python3: from pysos import logger logger.warning('I am from a dented text') if 1: logger.warning('Another warning') ''') wf = script.workflow() wf.run()
def testLongerCode(self): '''Test definition of classes (with intermediate newlines) in step.''' script = SoS_Script('''# first block [0] class A: def __init__(self): pass # the newline above should be fine because SoS treat this as # regular lines def __call__(self): return 0 b = A()() ''') env.shared_vars=['b'] wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['b'], 0)
def testCombinedWorkflow(self): '''Test the creation and execution of combined workfow''' env.shared_vars = ['executed', 'input_b1', 'a'] script = SoS_Script(''' a0 = 0 if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] [parameters] a = a0 + 1 [a_1] [a_2] [a_3] [a_4] output: 'out_a_4' [b_1] input_b1 = input [b_2] [b_3] [b_4] [c] [d] ''') env.run_mode = 'dryrun' wf = script.workflow('a+b') wf.run() self.assertEqual(env.sos_dict['executed'], ['a_parameters', 'a_1', 'a_2', 'a_3', 'a_4', 'b_parameters', 'b_1', 'b_2', 'b_3', 'b_4']) self.assertEqual(env.sos_dict['a'], 1) self.assertEqual(env.sos_dict['input_b1'], ['out_a_4']) # wf = script.workflow('a_ 1-2 + a_4 + b_3-') wf.run() self.assertEqual(env.sos_dict['executed'], ['a_parameters', 'a_1', 'a_2', 'a_parameters', 'a_4', 'b_parameters', 'b_3', 'b_4']) # wf = script.workflow('a+c+d') wf.run() self.assertEqual(env.sos_dict['executed'], ['a_parameters', 'a_1', 'a_2', 'a_3', 'a_4', 'c_parameters', 'c_0', 'd_parameters', 'd_0'])
def testProgressBar(self): '''Test progress bar''' env.verbosity = 1 prog = ProgressBar('test', 100) for i in range(100): prog.update(i) prog.done() prog = ProgressBar('test', 100) for i in range(20): prog.progress(5) prog.done() # script = SoS_Script(''' [1] [2] [3] [4] [5] ''') wf = script.workflow() wf.run() # progress bar with nested workflow script = SoS_Script(''' import time time.sleep(0.5) [sub_1] [sub_2] [sub_3] [sub_4] [a_1] [a_2] [a_3] sos_run('sub') [a_4] [a_5] ''') wf = script.workflow('a') wf.run()
def testGroupBy(self): '''Test group_by parameter of step input''' # group_by = 'all' env.shared_vars = ['executed'] script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='all' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['executed'], [['a1.txt', 'a2.txt', 'a3.txt', 'a4.txt']]) # group_by = 'single' script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='single' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['executed'], [['a1.txt'], ['a2.txt'], ['a3.txt'], ['a4.txt']]) # group_by = 'pairs' script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='pairs' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['executed'], [['a1.txt', 'a3.txt'], ['a2.txt', 'a4.txt']]) # group_by = 'pairwise' script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='pairwise' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual( env.sos_dict['executed'], [['a1.txt', 'a2.txt'], ['a2.txt', 'a3.txt'], ['a3.txt', 'a4.txt']]) # group_by = 'combinations' script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='combinations' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual( env.sos_dict['executed'], [['a1.txt', 'a2.txt'], ['a1.txt', 'a3.txt'], ['a1.txt', 'a4.txt'], ['a2.txt', 'a3.txt'], ['a2.txt', 'a4.txt'], ['a3.txt', 'a4.txt']])
def testSectionDirectives(self): '''Test directives of sections''' # cannot be in the global section self.assertRaises(ParsingError, SoS_Script, '''input: 'filename' ''') # multi-line OK SoS_Script(''' [0] input: 'filename', 'filename1' ''') # An abusive case with multi-line OK, from first column ok, blank line ok SoS_Script(''' [0] input: 'filename', 'filename1', filename4, opt1=value output: blah depends: 'something else' ''') # option with expression ok SoS_Script(''' [0] input: 'filename', 'filename2', opt=value==1 ''') # unrecognized directive, allowed now SoS_Script(''' [0] something: 'filename', filename2, opt=value==1 ''') # need commma self.assertRaises(ParsingError, SoS_Script, ''' [0] input: 'filename' filename2 ''') # can be after action SoS_Script(''' [0] func() input: 'filename', 'filename2', opt=value==1 ''') # assignments between directives are allowed SoS_Script(''' [0] input: 'filename', 'filename2', opt=value==1 a = 'some text' output: 'filename', 'filename2', opt=value==1 ''') # can have action between directives SoS_Script(''' [0] input: 'filename', 'filename2', opt=value==1 abc output: 'filename', 'filename2', opt=value==1 ''')
def testParameters(self): '''Test parameters section''' # directive not allowed in parameters self.assertRaises(ParsingError, SoS_Script, ''' [parameters] input: 'filename' ''') self.assertRaises(ParsingError, SoS_Script, ''' [parameters] func() ''') script = SoS_Script(filename='scripts/section1.sos') self.assertRaises(ArgumentError, script.workflow('chapter_0').run, args=['--not_exist']) self.assertRaises(ArgumentError, script.workflow('chapter_0').run, args=['--par1', 'a', 'b']) # script = SoS_Script(''' [parameters] a = [1, 2] [0] ''') script.workflow().run() self.assertEqual(env.sos_dict['a'], [1, 2]) script.workflow().run(args=['--a', '3']) self.assertEqual(env.sos_dict['a'], [3]) script.workflow().run(args=['--a', '3', '5']) self.assertEqual(env.sos_dict['a'], [3, 5]) # script = SoS_Script(''' [parameters] a = ['a.txt', 'b.txt'] [0] ''') script.workflow().run() self.assertEqual(env.sos_dict['a'], ['a.txt', 'b.txt']) script.workflow().run(args=['--a', '3']) self.assertEqual(env.sos_dict['a'], ['3']) script.workflow().run(args=['--a', '3', '5']) self.assertEqual(env.sos_dict['a'], ['3', '5']) # # test parameter using global definition script = SoS_Script(''' a="100" [parameters] b=str(int(a)+1) ''') script.workflow().run() self.assertEqual(env.sos_dict['b'], '101') # env.sos_dict.clear() script = SoS_Script(''' a=100 [parameters] b=a+1 ''') script.workflow().run() self.assertEqual(env.sos_dict['b'], 101) # script = SoS_Script(''' a=100 [parameters] b=a+1 ''') wf = script.workflow() self.assertRaises(ArgumentError, wf.run, args=['--b', 'a']) # script = SoS_Script(''' a=100 [parameters] b=a+1. ''') wf = script.workflow() wf.run(args=['--b', '1000']) # self.assertEqual(env.sos_dict['b'], 1000) # # test string interpolation of the parameter section script = SoS_Script(''' a=100 [parameters] b='${a+1}' ''') wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['b'], '101') # test alternative sigil script = SoS_Script(''' a=100 [parameters: sigil='[ ]'] b='[a+1]' ''') wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['b'], '101') # # argument has hve a value self.assertRaises(ParsingError, SoS_Script, ''' [parameters] b= ''') # if it is a type, must provide value script = SoS_Script(''' [parameters] b = int ''') self.assertRaises(ArgumentError, script.workflow().run) # script = SoS_Script(''' [parameters] b = list ''') self.assertRaises(ArgumentError, script.workflow().run) # also require the type script = SoS_Script(''' [parameters] b = int ''') self.assertRaises(ArgumentError, script.workflow().run, args=['--b', 'file']) # script = SoS_Script(''' [parameters] b = int ''') wf = script.workflow() wf.run(args=['--b', '5']) self.assertEqual(env.sos_dict['b'], 5) # list is ok script = SoS_Script(''' [parameters] b = list ''') wf = script.workflow() wf.run(args=['--b', '5']) self.assertEqual(env.sos_dict['b'], ['5']) # bool script = SoS_Script(''' [parameters] b = bool ''') wf = script.workflow() wf.run(args=['--b', 't']) self.assertEqual(env.sos_dict['b'], True) script = SoS_Script(''' [parameters] b = True ''') wf = script.workflow() wf.run(args=['--b', 'False']) self.assertEqual(env.sos_dict['b'], False) script = SoS_Script(''' [parameters] b = True ''') wf = script.workflow() wf.run(args=['--b', '1']) self.assertEqual(env.sos_dict['b'], True) script = SoS_Script(''' [parameters] b = bool ''') wf = script.workflow() wf.run(args=['--b', 'no']) self.assertEqual(env.sos_dict['b'], False) # # should fail for undefined variables script = SoS_Script(''' [parameters] a = 5 ''') self.assertRaises(ArgumentError, script.workflow().run, args=['--b', 'file']) # and for cases without parameter section script = SoS_Script(''' ''') self.assertRaises(ArgumentError, script.workflow().run, args=['--b', 'file'])
def testParameters(self): '''Test parameters section''' # directive not allowed in parameters self.assertRaises(ParsingError, SoS_Script, ''' [parameters] input: 'filename' ''') self.assertRaises(ParsingError, SoS_Script, ''' [parameters] func() ''') script = SoS_Script(filename='scripts/section1.sos') self.assertRaises(ArgumentError, script.workflow('chapter_0').run, args=['--not_exist']) self.assertRaises(ArgumentError, script.workflow('chapter_0').run, args=['--par1', 'a', 'b']) # script = SoS_Script(''' [parameters] a = [1, 2] [0] ''') script.workflow().run() self.assertEqual(env.sos_dict['a'], [1,2]) script.workflow().run(args=['--a', '3']) self.assertEqual(env.sos_dict['a'], [3]) script.workflow().run(args=['--a', '3', '5']) self.assertEqual(env.sos_dict['a'], [3, 5]) # script = SoS_Script(''' [parameters] a = ['a.txt', 'b.txt'] [0] ''') script.workflow().run() self.assertEqual(env.sos_dict['a'], ['a.txt', 'b.txt']) script.workflow().run(args=['--a', '3']) self.assertEqual(env.sos_dict['a'], ['3']) script.workflow().run(args=['--a', '3', '5']) self.assertEqual(env.sos_dict['a'], ['3', '5']) # # test parameter using global definition script = SoS_Script(''' a="100" [parameters] b=str(int(a)+1) ''') script.workflow().run() self.assertEqual(env.sos_dict['b'], '101') # env.sos_dict.clear() script = SoS_Script(''' a=100 [parameters] b=a+1 ''') script.workflow().run() self.assertEqual(env.sos_dict['b'], 101) # script = SoS_Script(''' a=100 [parameters] b=a+1 ''') wf = script.workflow() self.assertRaises(ArgumentError, wf.run, args=['--b', 'a']) # script = SoS_Script(''' a=100 [parameters] b=a+1. ''') wf = script.workflow() wf.run(args=['--b', '1000']) # self.assertEqual(env.sos_dict['b'], 1000) # # test string interpolation of the parameter section script = SoS_Script(''' a=100 [parameters] b='${a+1}' ''') wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['b'], '101') # test alternative sigil script = SoS_Script(''' a=100 [parameters: sigil='[ ]'] b='[a+1]' ''') wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['b'], '101') # # argument has hve a value self.assertRaises(ParsingError, SoS_Script, ''' [parameters] b= ''') # if it is a type, must provide value script = SoS_Script(''' [parameters] b = int ''') self.assertRaises(ArgumentError, script.workflow().run) # script = SoS_Script(''' [parameters] b = list ''') self.assertRaises(ArgumentError, script.workflow().run) # also require the type script = SoS_Script(''' [parameters] b = int ''') self.assertRaises(ArgumentError, script.workflow().run, args=['--b', 'file']) # script = SoS_Script(''' [parameters] b = int ''') wf = script.workflow() wf.run(args=['--b', '5']) self.assertEqual(env.sos_dict['b'], 5) # list is ok script = SoS_Script(''' [parameters] b = list ''') wf = script.workflow() wf.run(args=['--b', '5']) self.assertEqual(env.sos_dict['b'], ['5']) # bool script = SoS_Script(''' [parameters] b = bool ''') wf = script.workflow() wf.run(args=['--b', 't']) self.assertEqual(env.sos_dict['b'], True) script = SoS_Script(''' [parameters] b = True ''') wf = script.workflow() wf.run(args=['--b', 'False']) self.assertEqual(env.sos_dict['b'], False) script = SoS_Script(''' [parameters] b = True ''') wf = script.workflow() wf.run(args=['--b', '1']) self.assertEqual(env.sos_dict['b'], True) script = SoS_Script(''' [parameters] b = bool ''') wf = script.workflow() wf.run(args=['--b', 'no']) self.assertEqual(env.sos_dict['b'], False) # # should fail for undefined variables script = SoS_Script(''' [parameters] a = 5 ''') self.assertRaises(ArgumentError, script.workflow().run, args=['--b', 'file']) # and for cases without parameter section script = SoS_Script(''' ''') self.assertRaises(ArgumentError, script.workflow().run, args=['--b', 'file'])
def testGroupBy(self): '''Test group_by parameter of step input''' # group_by = 'all' env.shared_vars = ['executed'] script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='all' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['executed'], [['a1.txt', 'a2.txt', 'a3.txt', 'a4.txt']]) # group_by = 'single' script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='single' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['executed'], [['a1.txt'], ['a2.txt'], ['a3.txt'], ['a4.txt']]) # group_by = 'pairs' script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='pairs' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['executed'], [['a1.txt', 'a3.txt'], ['a2.txt', 'a4.txt']]) # group_by = 'pairwise' script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='pairwise' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['executed'], [['a1.txt', 'a2.txt'], ['a2.txt', 'a3.txt'], ['a3.txt', 'a4.txt']]) # group_by = 'combinations' script = SoS_Script(''' [0] executed = [] input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='combinations' executed.append(_input) ''') env.run_mode = 'dryrun' wf = script.workflow() wf.run() self.assertEqual(env.sos_dict['executed'], [['a1.txt', 'a2.txt'], ['a1.txt', 'a3.txt'], ['a1.txt', 'a4.txt'], ['a2.txt', 'a3.txt'], ['a2.txt', 'a4.txt'], ['a3.txt', 'a4.txt']])
def testNestedWorkflow(self): '''Test the creation and execution of combined workfow''' env.shared_vars = ['executed', 'inputs'] script = SoS_Script(''' if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] if 'inputs' not in locals(): inputs = [] [paramters] [a_1] inputs.append(input) [a_2] inputs.append(input) [a_3] inputs.append(input) [a_4] output: 'a.done' inputs.append(input) [b_1] input: 'b.begin' inputs.append(input) [b_2] inputs.append(input) [b_3] inputs.append(input) [b_4] output: 'b.txt' inputs.append(input) [c] input: 'a.txt' output: 'b.txt' inputs.append(input) sos_run('a+b') ''') env.run_mode = 'dryrun' wf = script.workflow('c') wf.run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'a_1', 'a_2', 'a_3', 'a_4', 'b_1', 'b_2', 'b_3', 'b_4']) self.assertEqual(env.sos_dict['inputs'], [['a.txt'], ['a.txt'], [], [], [], ['b.begin'], [], [], []]) # step will be looped script = SoS_Script(''' if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] if 'inputs' not in locals(): inputs = [] [a_1] output: _input[0] + '.a1' inputs.append(input) [a_2] output: _input[0] + '.a2' inputs.append(input) [c] input: 'a.txt', 'b.txt', group_by='single' inputs.append(_input) sos_run('a') ''') env.run_mode = 'dryrun' wf = script.workflow('c') wf.run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'a_1', 'a_2', 'a_1', 'a_2']) #self.assertEqual(env.sos_dict['inputs'], [['a.txt'], ['a.txt'], ['a.txt.a1'], ['b.txt'], ['b.txt'], ['b.txt.a1']]) # # allow specifying a single step # step will be looped script = SoS_Script(''' if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] [a_1] [a_2] [c_0] [c_1] input: 'a.txt', 'b.txt', group_by='single' sos_run('a_2') ''') env.run_mode = 'dryrun' wf = script.workflow('c') wf.run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_2', 'a_2']) # allow specifying a single step # step will be looped script = SoS_Script(''' if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] [a_1] [a_2] [c_0] [c_1] input: 'a.txt', 'b.txt', group_by='single' sos_run('a_2') ''') env.run_mode = 'dryrun' wf = script.workflow('c') wf.run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_2', 'a_2']) # # recursive subworkflow not allowed script = SoS_Script(''' if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] [a_1] [a_2] [c_0] [c_1] input: 'a.txt', 'b.txt', group_by='single' sos_run('a_2+c') ''') env.run_mode = 'dryrun' wf = script.workflow('c') self.assertRaises((ExecuteError, RuntimeError), wf.run) # # nested subworkflow is allowed script = SoS_Script(''' if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] [a_1] [a_2] [a_3] [b_1] [b_2] sos_run('a_1+a_2') [c_0] [c_1] input: 'a.txt' sos_run('a+b') ''') env.run_mode = 'dryrun' wf = script.workflow('c') wf.run() self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_1', 'a_2', 'a_3', 'b_1', 'b_2', 'a_1', 'a_2']) # # # nested subworkflow with step option and others script = SoS_Script(''' if 'executed' in locals(): executed.append(step_name) else: executed = [step_name] [a_1] [a_2] [a_3] [b] input: 'a.txt', 'b.txt', group_by='single' sos_run('a_3+a_1') [d] input: 'a.txt', 'b.txt', group_by='single' sos_run('a_2') [e2_2] input: 'a.txt', 'b.txt', group_by='single' ''') env.run_mode = 'dryrun' wf = script.workflow('b') wf.run() self.assertEqual(env.sos_dict['executed'], ['b_0', 'a_3', 'a_1', 'a_3', 'a_1']) wf = script.workflow('d') wf.run() self.assertEqual(env.sos_dict['executed'], ['d_0', 'a_2', 'a_2']) wf = script.workflow('e2') wf.run() self.assertEqual(env.sos_dict['executed'], ['e2_2'])