Esempio n. 1
0
    def testDynamicNestedWorkflow(self):
        '''Test nested workflow controlled by command line option'''
        script = SoS_Script('''
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
[parameters]
wf='a'

[a_1]
[a_2]
[a_3]
[b_1]
[b_2]
[b_3]

[default]
sos_run(wf)
''')
        env.shared_vars = ['executed']
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run(args=['--wf', 'b'])
        self.assertEqual(env.sos_dict['executed'], ['default_parameters', 'default_0', 'b_parameters', 'b_1', 'b_2', 'b_3'])
        #
        wf.run(args=['--wf', 'a'])
        self.assertEqual(env.sos_dict['executed'], ['default_parameters', 'default_0', 'a_parameters', 'a_1', 'a_2', 'a_3'])
Esempio n. 2
0
    def testDescriptions(self):
        '''Test script and workflow descriptions'''
        script = SoS_Script('''# first block

# global
# description

# human
# description of human

# description of human continued

[human_1]

a = '1'

# mouse
# mouse description
#

[mouse_1]
''')
        self.assertEqual(script.description, 'global\ndescription\n\n')
        self.assertEqual(script.workflow('human').description, 'description of human\ndescription of human continued\n')
        self.assertEqual(script.workflow('mouse').description, 'mouse description\n')
Esempio n. 3
0
    def testInput(self):
        '''Test input directive'''
        script = SoS_Script('''
[0]
files = ['a.txt', 'b.txt']

input: 'a.pdf', files, skip=False

''')
        env.run_mode = 'dryrun'
        script.workflow('default').run()
        #
        # test input types
        script = SoS_Script('''
[0:alias='test']
files = ('a${i}' for i in range(2))
input: {'a.txt', 'b.txt'}, files
output: ('a${x}' for x in _input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(sorted(env.sos_dict['test'].input), ['a.txt', 'a0', 'a1', 'b.txt'])
        self.assertEqual(sorted(env.sos_dict['test'].output), ['aa.txt', 'aa0', 'aa1', 'ab.txt'])
Esempio n. 4
0
def dsc_run(args, workflow_args, content, verbosity = 1, jobs = None,
            run_mode = 'run', queue = None):
    env.verbosity = verbosity
    env.max_jobs = args.__max_jobs__ if jobs is None else jobs
    # kill all remaining processes when the master process is killed.
    atexit.register(env.cleanup)
    if args.__rerun__ or run_mode != 'run':
        env.sig_mode = 'ignore'
    elif run_mode == 'construct':
        env.sig_mode = 'construct'
    else:
        env.sig_mode = 'default'
    try:
        script = SoS_Script(content=content, transcript = None)
        workflow = script.workflow(args.workflow)
        if queue is None or env.max_jobs == 1:
            # single process executor
            executor = Base_Executor(workflow, args=workflow_args, config_file=args.__config__)
        elif queue is None:
            executor = MP_Executor(workflow, args=workflow_args, config_file=args.__config__)
        else:
            executor = RQ_Executor(workflow, args=workflow_args, config_file=args.__config__)
        if run_mode == 'dryrun':
            executor.dryrun()
        else:
            executor.run()
    except Exception as e:
        if verbosity and verbosity > 2:
            sys.stderr.write(get_traceback())
        env.logger.error(e)
        sys.exit(1)
    env.verbosity = args.verbosity
Esempio n. 5
0
    def testStringLiteral(self):
        '''Test string literals of SoS'''
        env.shared_vars = ['a', 'b', 'c']
        script = SoS_Script(r"""
[0]
a = 'a\n'
b = "a\n"
c = '''a\n

b'''
""")
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['a'], 'a\n')
        self.assertEqual(env.sos_dict['b'], 'a\n')
        # MAYJOR difference
        self.assertEqual(env.sos_dict['c'], 'a\\n\nb')
        script = SoS_Script(r'''
[0]
c = """a\n

b"""
''')
        wf = script.workflow()
        wf.run()
        # Note the difference between """ and ''' quotes
        self.assertEqual(env.sos_dict['c'], 'a\n\nb')
Esempio n. 6
0
    def testGlobalVariables(self):
        '''Test definition of variables'''
        # global section cannot have directive
        self.assertRaises(ParsingError, SoS_Script,
            '''input: 'filename' ''')
        # or unrecognized directive
        self.assertRaises(ParsingError, SoS_Script,
            '''inputs: 'filename' ''')
        # allow definition
        SoS_Script('''a = '1' ''')
        SoS_Script('''a = ['a', 'b'] ''')
        # but this one has incorrect syntax
        self.assertRaises(ParsingError, SoS_Script,
            '''a = 'b  ''')
        # This one also does not work because b is not defined.
        delattr(env, 'sos_dict')
        script = SoS_Script('''a = b\n[0] ''')
        wf = script.workflow()
        self.assertRaises(RuntimeError, wf.run)
        # multi-line string literal
        SoS_Script('''a = """
this is a multi line
string """
''')
        # multi-line list literal, even with newline in between
        SoS_Script('''a = [
'a',

'b'
]
''')
        #
        script = SoS_Script(filename='scripts/section1.sos')
Esempio n. 7
0
    def testYAMLConfig(self):
        '''Test config file in yaml format'''
        with open('config.yaml', 'w') as config:
            config.write('''
# Lines beginning with # are skipped when the JSON is parsed, so we can
# put comments into our JSON configuration files
{
    StoreOwner : "John Doe",

    # List of items that we sell
    Fruits: [ "apple", "banana", "pear" ],
    Price: 1.05
}            
''')
        with open('config.sos', 'w') as sos:
            sos.write('''
[0]
print(CONFIG['StoreOwner'])
print(CONFIG.get('StoreOwner', 'something'))
print(CONFIG.get('StoreOwnerSpouse', 'someone else'))
print(CONFIG.StoreOwner)
'''
)
        # run the command
        self.assertEqual(subprocess.call('sos run config.sos -c config.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 0)
        # now test the value
        script = SoS_Script(filename='config.sos')
        wf = script.workflow()
        wf.run(config_file='config.yaml')
        self.assertEqual(env.sos_dict['CONFIG']['Price'], 1.05)
        self.assertEqual(env.sos_dict['CONFIG']['StoreOwner'], 'John Doe')
        self.assertEqual(env.sos_dict['CONFIG']['Fruits'], ['apple', 'banana', 'pear'])
        # configuration items should be readonly
        with open('config.sos', 'w') as sos:
            sos.write('''
[0]
CONFIG['a'] = 'b'
'''
)
        # the command would fail with error message
        # ERROR: Failed to process statement CONFIG['a'] = 'b'
        # : Cannot modify a readonly dictionary.
        self.assertEqual(subprocess.call('sos run config.sos -c config.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1)
        #
        with open('config.sos', 'w') as sos:
            sos.write('''
[0]
CONFIG.a = 'b'
'''
)
        # the command would fail with error message
        # ERROR: Failed to process statement CONFIG['a'] = 'b'
        # : Cannot modify a readonly dictionary.
        self.assertEqual(subprocess.call('sos run config.sos -c config.yaml', stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, shell=True), 1)
        #
        for filename in ['config.sos', 'config.yaml']:
            os.remove(filename)
Esempio n. 8
0
def install_r_libs(libs):
    if libs is None:
        return
    verbosity = env.verbosity
    for value in libs:
        groups = re.search('(.*?)\((.*?)\)', value)
        if groups is not None:
            value = groups.group(1).strip()
            versions = [x.strip() for x in groups.group(2).split(',')]
        else:
            versions = None
        env.logger.info("Checking R library ``{}`` ...".format(value))
        env.verbosity = 0
        script = SoS_Script('[0]\ncheck_R_library({}, {})'.format(repr(value), repr(versions)))
        Base_Executor(script.workflow()).prepare()
        env.verbosity = verbosity
Esempio n. 9
0
def sos_run(args, workflow_args):
    env.max_jobs = args.__max_jobs__
    env.verbosity = args.verbosity
    # kill all remainging processes when the master process is killed.
    atexit.register(env.cleanup)
    env.run_mode = 'run'
    if args.__rerun__:
        env.sig_mode = 'ignore'
    try:
        script = SoS_Script(content=args.script)
        executor = Sequential_Executor(script.workflow(args.workflow))
        executor.run(workflow_args, cmd_name=args.dsc_file, config_file = args.__config__)
    except Exception as e:
        if args.verbosity and args.verbosity > 2:
            sys.stderr.write(get_traceback())
        env.logger.error(e)
        sys.exit(1)
Esempio n. 10
0
def sos_run(args, workflow_args):
    env.verbosity = args.verbosity
    env.max_jobs = args.__max_jobs__
    # kill all remainging processes when the master process is killed.
    atexit.register(env.cleanup)
    try:
        script = SoS_Script(content=args.script)
        workflow = script.workflow(args.workflow)
        if args.__dryrun__:
            env.run_mode = 'dryrun'
        if args.__rerun__:
            env.sig_mode = 'ignore'
        workflow.run(workflow_args, cmd_name=args.dsc_file)
    except Exception as e:
        if args.verbosity and args.verbosity > 2:
            print_traceback()
        env.logger.error(e)
        sys.exit(1)
Esempio n. 11
0
    def testSourceOption(self):
        '''Test the source section option'''
        # nested subworkflow with step option and others
        env.shared_vars = ['executed', 'GLB', 'parB']
        if not os.path.isdir('temp'):
            os.mkdir('temp')
        with open('temp/test.sos', 'w') as sos:
            sos.write('''
# test sos script

# global definition
GLB = 5
if 'executed' in locals():
    executed.append('t.' + step_name)
else:
    executed = ['t.' + step_name]

[parameters]
parB = 10

[A_1]
output: _input[0] + 'a1'

[A_2]
output: _input[0] + 'a2'

''')
        script = SoS_Script('''
if 'executed' in locals():
    executed.append('g.' + step_name)
else:
    executed = ['g.' + step_name]
[b_1: skip=False]
input: 'a.txt', 'b.txt', group_by='single'
sos_run('A', source='temp/test.sos')
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('b')
        wf.run()
        self.assertEqual(env.sos_dict['GLB'], 5)
        self.assertEqual(env.sos_dict['parB'], 10)
        self.assertEqual(env.sos_dict['executed'], ['g.b_1', 't.A_parameters', 't.A_1', 't.A_2', 't.A_parameters', 't.A_1', 't.A_2'])
        #
        shutil.rmtree('temp')
Esempio n. 12
0
    def testSkipStep(self):
        '''Test the skip option to skip certain steps'''
        script = SoS_Script('''
[parameters]
skip = 0

[0: alias='a', skip=skip==0]
var = 0

[1: alias='b', skip=skip==1]
var = 1

''')
        wf = script.workflow()
        wf.run(args=['--skip', '0'])
        self.assertEqual(env.sos_dict['b'].var, 1)
        #
        wf.run(args=['--skip', '1'])
        self.assertEqual(env.sos_dict['a'].var, 0)
Esempio n. 13
0
    def testScriptFormat(self):
        '''Test putting scripts directly under action'''
        script = SoS_Script('''
[0]
input: 'filename',  'filename2', opt=value==1
R:

open('something')
save.put()

''')
        script = SoS_Script('''
[0]
input: 'filename',  'filename2', opt=value==1
R: concurrent = True

open('something')
save.put()

''')
        script = SoS_Script('''
[0]
input: 'filename',  'filename2', opt=value==1
R: concurrent = True,
    workdir = 'someelse else'

open('something')
save.put()

''')
        # test dedent
        script = SoS_Script('''
[0]
python3:
    from pysos import logger
    logger.warning('I am from a dented text')
    if 1:
        logger.warning('Another warning')
''')
        wf = script.workflow()
        wf.run()
Esempio n. 14
0
    def testLongerCode(self):
        '''Test definition of classes (with intermediate newlines) in step.'''
        script = SoS_Script('''# first block

[0]
class A:
    def __init__(self):
        pass

    # the newline above should be fine because SoS treat this as
    # regular lines
    def __call__(self):
        return 0

b = A()()

''')
        env.shared_vars=['b']
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['b'], 0)
Esempio n. 15
0
    def testCombinedWorkflow(self):
        '''Test the creation and execution of combined workfow'''
        env.shared_vars = ['executed', 'input_b1', 'a']
        script = SoS_Script('''
a0 = 0
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
[parameters]
a = a0 + 1
[a_1]
[a_2]
[a_3]
[a_4]
output: 'out_a_4'
[b_1]
input_b1 = input
[b_2]
[b_3]
[b_4]
[c]
[d]
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('a+b')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['a_parameters', 'a_1', 'a_2', 'a_3', 'a_4', 'b_parameters', 'b_1', 'b_2', 'b_3', 'b_4'])
        self.assertEqual(env.sos_dict['a'], 1)
        self.assertEqual(env.sos_dict['input_b1'], ['out_a_4'])
        #
        wf = script.workflow('a_ 1-2 + a_4 + b_3-')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['a_parameters', 'a_1', 'a_2', 'a_parameters', 'a_4', 
            'b_parameters', 'b_3', 'b_4'])
        #
        wf = script.workflow('a+c+d')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['a_parameters', 'a_1', 'a_2', 'a_3', 'a_4', 'c_parameters', 'c_0', 'd_parameters', 'd_0'])
Esempio n. 16
0
    def testProgressBar(self):
        '''Test progress bar'''
        env.verbosity = 1
        prog = ProgressBar('test', 100)
        for i in range(100):
            prog.update(i)
        prog.done()
        prog = ProgressBar('test', 100)
        for i in range(20):
            prog.progress(5)
        prog.done()
        #
        script = SoS_Script('''

[1]
[2]
[3]
[4]
[5]
''')
        wf = script.workflow()
        wf.run()
        # progress bar with nested workflow
        script = SoS_Script('''
import time
time.sleep(0.5)
[sub_1]
[sub_2]
[sub_3]
[sub_4]
[a_1]
[a_2]
[a_3]
sos_run('sub')
[a_4]
[a_5]
''')
        wf = script.workflow('a')
        wf.run()
Esempio n. 17
0
    def testGroupBy(self):
        '''Test group_by parameter of step input'''
        # group_by = 'all'
        env.shared_vars = ['executed']
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='all'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['executed'],
                         [['a1.txt', 'a2.txt', 'a3.txt', 'a4.txt']])
        # group_by = 'single'
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='single'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['executed'],
                         [['a1.txt'], ['a2.txt'], ['a3.txt'], ['a4.txt']])
        # group_by = 'pairs'
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='pairs'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['executed'],
                         [['a1.txt', 'a3.txt'], ['a2.txt', 'a4.txt']])
        # group_by = 'pairwise'
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='pairwise'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(
            env.sos_dict['executed'],
            [['a1.txt', 'a2.txt'], ['a2.txt', 'a3.txt'], ['a3.txt', 'a4.txt']])
        # group_by = 'combinations'
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='combinations'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(
            env.sos_dict['executed'],
            [['a1.txt', 'a2.txt'], ['a1.txt', 'a3.txt'], ['a1.txt', 'a4.txt'],
             ['a2.txt', 'a3.txt'], ['a2.txt', 'a4.txt'], ['a3.txt', 'a4.txt']])
Esempio n. 18
0
    def testSectionDirectives(self):
        '''Test directives of sections'''
        # cannot be in the global section
        self.assertRaises(ParsingError, SoS_Script, '''input: 'filename' ''')
        # multi-line OK
        SoS_Script('''
[0]
input: 'filename',
    'filename1'

''')
        # An abusive case with multi-line OK, from first column ok, blank line ok
        SoS_Script('''
[0]
input: 'filename',
'filename1',

filename4,
opt1=value
output: 
    blah

depends:
'something else'
''')
        # option with expression ok
        SoS_Script('''
[0]
input: 'filename',  'filename2', opt=value==1

''')
        # unrecognized directive, allowed now
        SoS_Script('''
[0]
something: 'filename',  filename2, opt=value==1
''')
        # need commma
        self.assertRaises(ParsingError, SoS_Script, '''
[0]
input: 'filename'  filename2
''')
        # can be after action
        SoS_Script('''
[0]
func()        
input: 'filename',  'filename2', opt=value==1
''')
        # assignments between directives are allowed
        SoS_Script('''
[0]
input: 'filename',  'filename2', opt=value==1
a = 'some text'
output: 'filename',  'filename2', opt=value==1
''')
        # can have action between directives
        SoS_Script('''
[0]
input: 'filename',  'filename2', opt=value==1
abc
output: 'filename',  'filename2', opt=value==1
''')
Esempio n. 19
0
    def testParameters(self):
        '''Test parameters section'''
        # directive not allowed in parameters
        self.assertRaises(ParsingError, SoS_Script, '''
[parameters]
input: 'filename' 
''')
        self.assertRaises(ParsingError, SoS_Script, '''
[parameters]
func()
''')
        script = SoS_Script(filename='scripts/section1.sos')
        self.assertRaises(ArgumentError,
                          script.workflow('chapter_0').run,
                          args=['--not_exist'])
        self.assertRaises(ArgumentError,
                          script.workflow('chapter_0').run,
                          args=['--par1', 'a', 'b'])
        #
        script = SoS_Script('''
[parameters]
a = [1, 2]
[0]
''')
        script.workflow().run()
        self.assertEqual(env.sos_dict['a'], [1, 2])
        script.workflow().run(args=['--a', '3'])
        self.assertEqual(env.sos_dict['a'], [3])
        script.workflow().run(args=['--a', '3', '5'])
        self.assertEqual(env.sos_dict['a'], [3, 5])
        #
        script = SoS_Script('''
[parameters]
a = ['a.txt', 'b.txt']
[0]
''')
        script.workflow().run()
        self.assertEqual(env.sos_dict['a'], ['a.txt', 'b.txt'])
        script.workflow().run(args=['--a', '3'])
        self.assertEqual(env.sos_dict['a'], ['3'])
        script.workflow().run(args=['--a', '3', '5'])
        self.assertEqual(env.sos_dict['a'], ['3', '5'])
        #
        # test parameter using global definition
        script = SoS_Script('''
a="100"

[parameters]
b=str(int(a)+1)
''')
        script.workflow().run()
        self.assertEqual(env.sos_dict['b'], '101')
        #
        env.sos_dict.clear()
        script = SoS_Script('''
a=100

[parameters]
b=a+1
''')
        script.workflow().run()
        self.assertEqual(env.sos_dict['b'], 101)
        #
        script = SoS_Script('''
a=100

[parameters]
b=a+1
''')
        wf = script.workflow()
        self.assertRaises(ArgumentError, wf.run, args=['--b', 'a'])
        #
        script = SoS_Script('''
a=100

[parameters]
b=a+1.
''')
        wf = script.workflow()
        wf.run(args=['--b', '1000'])
        #
        self.assertEqual(env.sos_dict['b'], 1000)
        #
        # test string interpolation of the parameter section
        script = SoS_Script('''
a=100

[parameters]
b='${a+1}'
''')
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['b'], '101')
        # test alternative sigil
        script = SoS_Script('''
a=100

[parameters: sigil='[ ]']
b='[a+1]'
''')
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['b'], '101')
        #
        # argument has hve a value
        self.assertRaises(ParsingError, SoS_Script, '''
[parameters]
b=
''')
        # if it is a type, must provide value
        script = SoS_Script('''
[parameters]
b = int
''')
        self.assertRaises(ArgumentError, script.workflow().run)
        #
        script = SoS_Script('''
[parameters]
b = list
''')
        self.assertRaises(ArgumentError, script.workflow().run)
        # also require the type
        script = SoS_Script('''
[parameters]
b = int
''')
        self.assertRaises(ArgumentError,
                          script.workflow().run,
                          args=['--b', 'file'])
        #
        script = SoS_Script('''
[parameters]
b = int
''')
        wf = script.workflow()
        wf.run(args=['--b', '5'])
        self.assertEqual(env.sos_dict['b'], 5)
        # list is ok
        script = SoS_Script('''
[parameters]
b = list
''')
        wf = script.workflow()
        wf.run(args=['--b', '5'])
        self.assertEqual(env.sos_dict['b'], ['5'])
        # bool
        script = SoS_Script('''
[parameters]
b = bool
''')
        wf = script.workflow()
        wf.run(args=['--b', 't'])
        self.assertEqual(env.sos_dict['b'], True)
        script = SoS_Script('''
[parameters]
b = True
''')
        wf = script.workflow()
        wf.run(args=['--b', 'False'])
        self.assertEqual(env.sos_dict['b'], False)
        script = SoS_Script('''
[parameters]
b = True
''')
        wf = script.workflow()
        wf.run(args=['--b', '1'])
        self.assertEqual(env.sos_dict['b'], True)
        script = SoS_Script('''
[parameters]
b = bool
''')
        wf = script.workflow()
        wf.run(args=['--b', 'no'])
        self.assertEqual(env.sos_dict['b'], False)
        #
        # should fail for undefined variables
        script = SoS_Script('''
[parameters]
a = 5
''')
        self.assertRaises(ArgumentError,
                          script.workflow().run,
                          args=['--b', 'file'])
        # and for cases without parameter section
        script = SoS_Script('''
''')
        self.assertRaises(ArgumentError,
                          script.workflow().run,
                          args=['--b', 'file'])
Esempio n. 20
0
    def testParameters(self):
        '''Test parameters section'''
        # directive not allowed in parameters
        self.assertRaises(ParsingError, SoS_Script,
            '''
[parameters]
input: 'filename' 
''')
        self.assertRaises(ParsingError, SoS_Script,
            '''
[parameters]
func()
''')    
        script = SoS_Script(filename='scripts/section1.sos')
        self.assertRaises(ArgumentError, script.workflow('chapter_0').run,
            args=['--not_exist'])
        self.assertRaises(ArgumentError, script.workflow('chapter_0').run,
            args=['--par1', 'a', 'b'])
        # 
        script = SoS_Script('''
[parameters]
a = [1, 2]
[0]
''')    
        script.workflow().run()
        self.assertEqual(env.sos_dict['a'], [1,2])
        script.workflow().run(args=['--a', '3'])
        self.assertEqual(env.sos_dict['a'], [3])
        script.workflow().run(args=['--a', '3', '5'])
        self.assertEqual(env.sos_dict['a'], [3, 5])
        #
        script = SoS_Script('''
[parameters]
a = ['a.txt', 'b.txt']
[0]
''')    
        script.workflow().run()
        self.assertEqual(env.sos_dict['a'], ['a.txt', 'b.txt'])
        script.workflow().run(args=['--a', '3'])
        self.assertEqual(env.sos_dict['a'], ['3'])
        script.workflow().run(args=['--a', '3', '5'])
        self.assertEqual(env.sos_dict['a'], ['3', '5'])
        #
        # test parameter using global definition
        script = SoS_Script('''
a="100"

[parameters]
b=str(int(a)+1)
''')
        script.workflow().run()
        self.assertEqual(env.sos_dict['b'], '101')
        #
        env.sos_dict.clear()
        script = SoS_Script('''
a=100

[parameters]
b=a+1
''')
        script.workflow().run()
        self.assertEqual(env.sos_dict['b'], 101)
        #
        script = SoS_Script('''
a=100

[parameters]
b=a+1
''')
        wf = script.workflow()
        self.assertRaises(ArgumentError, wf.run, args=['--b', 'a'])
        #
        script = SoS_Script('''
a=100

[parameters]
b=a+1.
''')
        wf = script.workflow()
        wf.run(args=['--b', '1000'])
        #
        self.assertEqual(env.sos_dict['b'], 1000)
        #
        # test string interpolation of the parameter section
        script = SoS_Script('''
a=100

[parameters]
b='${a+1}'
''')
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['b'], '101')
        # test alternative sigil
        script = SoS_Script('''
a=100

[parameters: sigil='[ ]']
b='[a+1]'
''')
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['b'], '101')
        #
        # argument has hve a value
        self.assertRaises(ParsingError, SoS_Script, '''
[parameters]
b=
''')
        # if it is a type, must provide value
        script = SoS_Script('''
[parameters]
b = int
''')
        self.assertRaises(ArgumentError, script.workflow().run)
        #
        script = SoS_Script('''
[parameters]
b = list
''')
        self.assertRaises(ArgumentError, script.workflow().run)
        # also require the type
        script = SoS_Script('''
[parameters]
b = int
''')
        self.assertRaises(ArgumentError, script.workflow().run, args=['--b', 'file'])
        #
        script = SoS_Script('''
[parameters]
b = int
''')
        wf = script.workflow()
        wf.run(args=['--b', '5'])
        self.assertEqual(env.sos_dict['b'], 5)
        # list is ok
        script = SoS_Script('''
[parameters]
b = list
''')
        wf = script.workflow()
        wf.run(args=['--b', '5'])
        self.assertEqual(env.sos_dict['b'], ['5'])
        # bool
        script = SoS_Script('''
[parameters]
b = bool
''')
        wf = script.workflow()
        wf.run(args=['--b', 't'])
        self.assertEqual(env.sos_dict['b'], True)
        script = SoS_Script('''
[parameters]
b = True
''')
        wf = script.workflow()
        wf.run(args=['--b', 'False'])
        self.assertEqual(env.sos_dict['b'], False)
        script = SoS_Script('''
[parameters]
b = True
''')
        wf = script.workflow()
        wf.run(args=['--b', '1'])
        self.assertEqual(env.sos_dict['b'], True)
        script = SoS_Script('''
[parameters]
b = bool
''')
        wf = script.workflow()
        wf.run(args=['--b', 'no'])
        self.assertEqual(env.sos_dict['b'], False)
        #
        # should fail for undefined variables
        script = SoS_Script('''
[parameters]
a = 5
''')
        self.assertRaises(ArgumentError, script.workflow().run,
            args=['--b', 'file'])
        # and for cases without parameter section
        script = SoS_Script('''
''')        
        self.assertRaises(ArgumentError, script.workflow().run,
            args=['--b', 'file'])
Esempio n. 21
0
    def testGroupBy(self):
        '''Test group_by parameter of step input'''
        # group_by = 'all'
        env.shared_vars = ['executed']
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='all'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['executed'],  [['a1.txt', 'a2.txt', 'a3.txt', 'a4.txt']])
        # group_by = 'single'
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='single'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['executed'],  [['a1.txt'], ['a2.txt'], ['a3.txt'], ['a4.txt']])
        # group_by = 'pairs'
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='pairs'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['executed'],  [['a1.txt', 'a3.txt'], ['a2.txt', 'a4.txt']])
        # group_by = 'pairwise'
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='pairwise'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['executed'],  [['a1.txt', 'a2.txt'], ['a2.txt', 'a3.txt'], ['a3.txt', 'a4.txt']])
        # group_by = 'combinations'
        script = SoS_Script('''
[0]

executed = []
input: ['a{}.txt'.format(x) for x in range(1, 5)], group_by='combinations'

executed.append(_input)

''')
        env.run_mode = 'dryrun'
        wf = script.workflow()
        wf.run()
        self.assertEqual(env.sos_dict['executed'],  [['a1.txt', 'a2.txt'], ['a1.txt', 'a3.txt'], 
            ['a1.txt', 'a4.txt'], ['a2.txt', 'a3.txt'], ['a2.txt', 'a4.txt'], ['a3.txt', 'a4.txt']])
Esempio n. 22
0
    def testNestedWorkflow(self):
        '''Test the creation and execution of combined workfow'''
        env.shared_vars = ['executed', 'inputs']
        script = SoS_Script('''
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
if 'inputs' not in locals():
    inputs = []

[paramters]
[a_1]
inputs.append(input)
[a_2]
inputs.append(input)
[a_3]
inputs.append(input)
[a_4]
output: 'a.done'
inputs.append(input)
[b_1]
input: 'b.begin'
inputs.append(input)
[b_2]
inputs.append(input)
[b_3]
inputs.append(input)
[b_4]
output: 'b.txt'
inputs.append(input)
[c]
input: 'a.txt'
output: 'b.txt'
inputs.append(input)
sos_run('a+b')
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('c')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'a_1', 'a_2', 'a_3', 'a_4',
            'b_1', 'b_2', 'b_3', 'b_4'])
        self.assertEqual(env.sos_dict['inputs'], [['a.txt'], ['a.txt'], [], [], [], ['b.begin'], [], [], []])
        # step will be looped
        script = SoS_Script('''
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
if 'inputs' not in locals():
    inputs = []

[a_1]
output: _input[0] + '.a1'
inputs.append(input)
[a_2]
output: _input[0] + '.a2'
inputs.append(input)
[c]
input: 'a.txt', 'b.txt', group_by='single'
inputs.append(_input)
sos_run('a')
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('c')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'a_1', 'a_2', 'a_1', 'a_2'])
        #self.assertEqual(env.sos_dict['inputs'], [['a.txt'], ['a.txt'], ['a.txt.a1'], ['b.txt'], ['b.txt'], ['b.txt.a1']])
        #
        # allow specifying a single step
        # step will be looped
        script = SoS_Script('''
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
[a_1]
[a_2]
[c_0]
[c_1]
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a_2')
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('c')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_2',  'a_2'])
        # allow specifying a single step
        # step will be looped
        script = SoS_Script('''
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
[a_1]
[a_2]
[c_0]
[c_1]
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a_2')
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('c')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_2', 'a_2'])
        #
        # recursive subworkflow not allowed
        script = SoS_Script('''
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
[a_1]
[a_2]
[c_0]
[c_1]
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a_2+c')
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('c')
        self.assertRaises((ExecuteError, RuntimeError), wf.run)
        #
        # nested subworkflow is allowed
        script = SoS_Script('''
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
[a_1]
[a_2]
[a_3]
[b_1]
[b_2]
sos_run('a_1+a_2')
[c_0]
[c_1]
input: 'a.txt'
sos_run('a+b')
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('c')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['c_0', 'c_1', 'a_1', 'a_2', 'a_3',
            'b_1', 'b_2', 'a_1', 'a_2'])
        #
        #
        # nested subworkflow with step option and others
        script = SoS_Script('''
if 'executed' in locals():
    executed.append(step_name)
else:
    executed = [step_name]
[a_1]
[a_2]
[a_3]
[b]
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a_3+a_1')
[d]
input: 'a.txt', 'b.txt', group_by='single'
sos_run('a_2')
[e2_2]
input: 'a.txt', 'b.txt', group_by='single'
''')
        env.run_mode = 'dryrun'
        wf = script.workflow('b')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['b_0', 'a_3', 'a_1', 'a_3', 'a_1'])
        wf = script.workflow('d')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['d_0', 'a_2', 'a_2'])
        wf = script.workflow('e2')
        wf.run()
        self.assertEqual(env.sos_dict['executed'], ['e2_2'])