def testToStrWithScriptArgsSkipAndStartAfter(self, subprocessMock):
"""
The toStr method must return a complete summary of the status
specification when scriptArgs, skip, and startAfter are specified.
"""
status = {
'firstStep':
None,
'force':
False,
'lastStep':
None,
'scheduledAt':
1481379658.5455897,
'scriptArgs': ['hey', 'you'],
'skip': ['skip', 'this'],
'startAfter': ['34', '56'],
'nice':
'None',
'steps': [
{
'cwd': '00-start',
'name': 'start',
'scheduledAt': 1481379659.1530972,
'script': '00-start/start.sh',
'simulate': True,
'skip': False,
'stdout': '',
'taskDependencies': {},
'tasks': {}
},
]
}
subprocessMock.return_value = ('JOBID ST TIME NODELIST(REASON)\n'
'4417616 R 4:27 cpu-3\n')
sps = SlurmPipelineStatus(status)
self.assertEqual(
'''\
Scheduled at: 2016-12-10 14:20:58
Scheduling arguments:
First step: None
Force: False
Last step: None
Nice: None
Script arguments: hey you
Skip: skip, this
Start after: 34, 56
0 jobs emitted in total, of which 0 (100.00%) are finished
Step summary:
start: no jobs emitted
Step 1: start
No dependencies.
No tasks emitted by this step
Working directory: 00-start
Scheduled at: 2016-12-10 14:20:59
Script: 00-start/start.sh
Simulate: True
Skip: False''', sps.toStr())
def testToStr(self, subprocessMock):
"""
The toStr method must return a complete summary of the status
specification.
"""
status = {
'firstStep':
'panel',
'force':
False,
'lastStep':
None,
'nice':
3,
'user':
'******',
'scheduledAt':
1481379658.5455897,
'scriptArgs': [],
'skip': [],
'startAfter':
None,
'steps': [{
'cwd': '00-start',
'environ': {
'SP_FORCE': '1',
'SP_SKIP': '0',
},
'name': 'start',
'scheduledAt': 1481379659.1530972,
'script': '00-start/start.sh',
'simulate': True,
'skip': False,
'stdout': '',
'taskDependencies': {},
'tasks': {}
}, {
'cwd': '01-split',
'dependencies': ['start'],
'environ': {
'SP_FORCE': '1',
'SP_SKIP': '0',
},
'name': 'split',
'scheduledAt': 1481379664.184737,
'script': '01-split/sbatch.sh',
'simulate': True,
'skip': False,
'stdout': '',
'taskDependencies': {},
'tasks': {
'chunk-aaaaa': [],
'chunk-aaaab': [],
'chunk-aaaac': [],
}
}, {
'cwd': '02-blastn',
'dependencies': ['split'],
'environ': {
'SP_FORCE': '1',
'SP_SKIP': '0',
},
'name': 'blastn',
'scheduledAt': 1481379722.3996398,
'script': '02-blastn/sbatch.sh',
'simulate': True,
'skip': False,
'stdout': '',
'taskDependencies': {
'chunk-aaaaa': [],
'chunk-aaaab': [],
'chunk-aaaac': [],
},
"tasks": {
"chunk-aaaaa": [4416231],
"chunk-aaaab": [4416232],
"chunk-aaaac": [4416233],
},
}, {
'collect': True,
'cwd': '03-panel',
'dependencies': ['blastn'],
'environ': {
'SP_FORCE': '1',
'SP_SKIP': '0',
},
'name': 'panel',
'scheduledAt': 1481379722.5036008,
'script': '03-panel/sbatch.sh',
'simulate': False,
'skip': False,
'stdout': 'TASK: panel 4417615\n',
'taskDependencies': {
'chunk-aaaaa': [4416231],
'chunk-aaaab': [4416232],
'chunk-aaaac': [4416233],
},
'tasks': {
'panel': [4417615]
}
}, {
'cwd': '04-stop',
'dependencies': ['panel'],
'environ': {
'SP_FORCE': '1',
'SP_SKIP': '0',
},
'name': 'stop',
'scheduledAt': 1481379722.5428307,
'script': '04-stop/sbatch.sh',
'simulate': False,
'skip': False,
'stdout': 'TASK: stop 4417616\n',
'taskDependencies': {
'panel': [4417615]
},
'tasks': {
'stop': [4417616]
}
}]
}
subprocessMock.return_value = (
'JobID|JobName|State|Elapsed|Nodelist\n'
'4416231|name1|COMPLETED|04:32:00|cpu-3\n'
'4416232|name2|COMPLETED|04:02:00|cpu-6\n'
'4416233|name3|COMPLETED|04:12:00|cpu-7\n'
'4417615|name5|COMPLETED|04:11:00|cpu-8\n'
'4417616|name6|RUNNING|04:32:00|cpu-3\n')
sps = SlurmPipelineStatus(status)
self.assertEqual(
'''\
Scheduled by: sw463
Scheduled at: 2016-12-10 14:20:58
Scheduling arguments:
First step: panel
Force: False
Last step: None
Nice: 3
Sleep: 0.00
Script arguments: <None>
Skip: <None>
Start after: <None>
Steps summary:
Number of steps: 5
Jobs emitted in total: 5
Jobs finished: 4 (80.00%)
start: no jobs emitted
split: no jobs emitted
blastn: 3 jobs emitted, 3 (100.00%) finished
panel: 1 job emitted, 1 (100.00%) finished
stop: 1 job emitted, 0 (0.00%) finished
Step 1: start
No dependencies.
No tasks emitted by this step
Collect step: False
Error step: False
Working directory: 00-start
Scheduled at: 2016-12-10 14:20:59
Script: 00-start/start.sh
Simulate: True
Skip: False
Slurm pipeline environment variables:
SP_FORCE: 1
SP_SKIP: 0
Step 2: split
1 step dependency: start
Dependent on 0 tasks emitted by the dependent step
3 tasks emitted by this step
Summary: 0 jobs started by these tasks
Tasks:
chunk-aaaaa
chunk-aaaab
chunk-aaaac
Collect step: False
Error step: False
Working directory: 01-split
Scheduled at: 2016-12-10 14:21:04
Script: 01-split/sbatch.sh
Simulate: True
Skip: False
Slurm pipeline environment variables:
SP_FORCE: 1
SP_SKIP: 0
Step 3: blastn
1 step dependency: split
Dependent on 3 tasks emitted by the dependent step
Summary: 0 jobs started by the dependent tasks
Dependent tasks:
chunk-aaaaa
chunk-aaaab
chunk-aaaac
3 tasks emitted by this step
Summary: 3 jobs started by these tasks, of which 3 (100.00%) are finished
Tasks:
chunk-aaaaa
Job 4416231: JobName=name1, State=COMPLETED, Elapsed=04:32:00, \
Nodelist=cpu-3
chunk-aaaab
Job 4416232: JobName=name2, State=COMPLETED, Elapsed=04:02:00, \
Nodelist=cpu-6
chunk-aaaac
Job 4416233: JobName=name3, State=COMPLETED, Elapsed=04:12:00, \
Nodelist=cpu-7
Collect step: False
Error step: False
Working directory: 02-blastn
Scheduled at: 2016-12-10 14:22:02
Script: 02-blastn/sbatch.sh
Simulate: True
Skip: False
Slurm pipeline environment variables:
SP_FORCE: 1
SP_SKIP: 0
Step 4: panel
1 step dependency: blastn
Dependent on 3 tasks emitted by the dependent step
Summary: 3 jobs started by the dependent task, of which 3 (100.00%) are \
finished
Dependent tasks:
chunk-aaaaa
Job 4416231: JobName=name1, State=COMPLETED, Elapsed=04:32:00, \
Nodelist=cpu-3
chunk-aaaab
Job 4416232: JobName=name2, State=COMPLETED, Elapsed=04:02:00, \
Nodelist=cpu-6
chunk-aaaac
Job 4416233: JobName=name3, State=COMPLETED, Elapsed=04:12:00, \
Nodelist=cpu-7
1 task emitted by this step
Summary: 1 job started by this task, of which 1 (100.00%) are finished
Tasks:
panel
Job 4417615: JobName=name5, State=COMPLETED, Elapsed=04:11:00, \
Nodelist=cpu-8
Collect step: True
Error step: False
Working directory: 03-panel
Scheduled at: 2016-12-10 14:22:02
Script: 03-panel/sbatch.sh
Simulate: False
Skip: False
Slurm pipeline environment variables:
SP_FORCE: 1
SP_SKIP: 0
Step 5: stop
1 step dependency: panel
Dependent on 1 task emitted by the dependent step
Summary: 1 job started by the dependent task, of which 1 (100.00%) are \
finished
Dependent tasks:
panel
Job 4417615: JobName=name5, State=COMPLETED, Elapsed=04:11:00, \
Nodelist=cpu-8
1 task emitted by this step
Summary: 1 job started by this task, of which 0 (0.00%) are finished
Tasks:
stop
Job 4417616: JobName=name6, State=RUNNING, Elapsed=04:32:00, \
Nodelist=cpu-3
Collect step: False
Error step: False
Working directory: 04-stop
Scheduled at: 2016-12-10 14:22:02
Script: 04-stop/sbatch.sh
Simulate: False
Skip: False
Slurm pipeline environment variables:
SP_FORCE: 1
SP_SKIP: 0''', sps.toStr())
def testToStr(self, subprocessMock):
"""
The toStr method must return a complete summary of the status
specification.
"""
status = {
'firstStep':
'panel',
'force':
False,
'lastStep':
None,
'nice':
3,
'scheduledAt':
1481379658.5455897,
'scriptArgs': [],
'skip': [],
'startAfter':
None,
'steps': [{
'cwd': '00-start',
'name': 'start',
'scheduledAt': 1481379659.1530972,
'script': '00-start/start.sh',
'simulate': True,
'skip': False,
'stdout': '',
'taskDependencies': {},
'tasks': {}
}, {
'cwd': '01-split',
'dependencies': ['start'],
'name': 'split',
'scheduledAt': 1481379664.184737,
'script': '01-split/sbatch.sh',
'simulate': True,
'skip': False,
'stdout': '',
'taskDependencies': {},
'tasks': {
'chunk-aaaaa': [],
'chunk-aaaab': [],
'chunk-aaaac': [],
}
}, {
'cwd': '02-blastn',
'dependencies': ['split'],
'name': 'blastn',
'scheduledAt': 1481379722.3996398,
'script': '02-blastn/sbatch.sh',
'simulate': True,
'skip': False,
'stdout': '',
'taskDependencies': {
'chunk-aaaaa': [],
'chunk-aaaab': [],
'chunk-aaaac': [],
},
"tasks": {
"chunk-aaaaa": [4416231],
"chunk-aaaab": [4416232],
"chunk-aaaac": [4416233],
},
}, {
'collect': True,
'cwd': '03-panel',
'dependencies': ['blastn'],
'name': 'panel',
'scheduledAt': 1481379722.5036008,
'script': '03-panel/sbatch.sh',
'simulate': False,
'skip': False,
'stdout': 'TASK: panel 4417615\n',
'taskDependencies': {
'chunk-aaaaa': [4416231],
'chunk-aaaab': [4416232],
'chunk-aaaac': [4416233],
},
'tasks': {
'panel': [4417615]
}
}, {
'cwd': '04-stop',
'dependencies': ['panel'],
'name': 'stop',
'scheduledAt': 1481379722.5428307,
'script': '04-stop/sbatch.sh',
'simulate': False,
'skip': False,
'stdout': 'TASK: stop 4417616\n',
'taskDependencies': {
'panel': [4417615]
},
'tasks': {
'stop': [4417616]
}
}]
}
subprocessMock.return_value = ('JOBID ST TIME NODELIST(REASON)\n'
'4417616 R 4:27 cpu-3\n')
sps = SlurmPipelineStatus(status)
self.assertEqual(
'''\
Scheduled at: 2016-12-10 14:20:58
Scheduling arguments:
First step: panel
Force: False
Last step: None
Nice: 3
Script arguments: <None>
Skip: <None>
Start after: <None>
5 jobs emitted in total, of which 4 (80.00%) are finished
Step summary:
start: no jobs emitted
split: no jobs emitted
blastn: 3 jobs emitted, 3 (100.00%) finished
panel: 1 job emitted, 1 (100.00%) finished
stop: 1 job emitted, 0 (0.00%) finished
Step 1: start
No dependencies.
No tasks emitted by this step
Working directory: 00-start
Scheduled at: 2016-12-10 14:20:59
Script: 00-start/start.sh
Simulate: True
Skip: False
Step 2: split
1 step dependency: start
Dependent on 0 tasks emitted by the dependent step
3 tasks emitted by this step
0 jobs started by these tasks
Tasks:
chunk-aaaaa
chunk-aaaab
chunk-aaaac
Working directory: 01-split
Scheduled at: 2016-12-10 14:21:04
Script: 01-split/sbatch.sh
Simulate: True
Skip: False
Step 3: blastn
1 step dependency: split
Dependent on 3 tasks emitted by the dependent step
0 jobs started by the dependent tasks
Dependent tasks:
chunk-aaaaa
chunk-aaaab
chunk-aaaac
3 tasks emitted by this step
3 jobs started by this task, of which 3 (100.00%) are finished
Tasks:
chunk-aaaaa
Job 4416231: Finished
chunk-aaaab
Job 4416232: Finished
chunk-aaaac
Job 4416233: Finished
Working directory: 02-blastn
Scheduled at: 2016-12-10 14:22:02
Script: 02-blastn/sbatch.sh
Simulate: True
Skip: False
Step 4: panel
1 step dependency: blastn
Dependent on 3 tasks emitted by the dependent step
3 jobs started by the dependent task, of which 3 (100.00%) are finished
Dependent tasks:
chunk-aaaaa
Job 4416231: Finished
chunk-aaaab
Job 4416232: Finished
chunk-aaaac
Job 4416233: Finished
1 task emitted by this step
1 job started by this task, of which 1 (100.00%) are finished
Tasks:
panel
Job 4417615: Finished
Working directory: 03-panel
Scheduled at: 2016-12-10 14:22:02
Script: 03-panel/sbatch.sh
Simulate: False
Skip: False
Step 5: stop
1 step dependency: panel
Dependent on 1 task emitted by the dependent step
1 job started by the dependent task, of which 1 (100.00%) are finished
Dependent tasks:
panel
Job 4417615: Finished
1 task emitted by this step
1 job started by this task, of which 0 (0.00%) are finished
Tasks:
stop
Job 4417616: Status=R Time=4:27 Nodelist=cpu-3
Working directory: 04-stop
Scheduled at: 2016-12-10 14:22:02
Script: 04-stop/sbatch.sh
Simulate: False
Skip: False''', sps.toStr())
def testToStrWithScriptArgsSkipAndStartAfter(self, subprocessMock):
"""
The toStr method must return a complete summary of the status
specification when scriptArgs, skip, and startAfter are specified.
"""
status = {
'user':
'******',
'firstStep':
None,
'force':
False,
'lastStep':
None,
'nice':
'None',
'scheduledAt':
1481379658.5455897,
'scriptArgs': ['hey', 'you'],
'skip': ['start-step'],
'sleep':
5.0,
'startAfter': [34, 56],
'steps': [
{
'cwd': '00-start',
'environ': {
'SP_FORCE': '1',
'SP_SKIP': '0',
},
'name': 'start-step',
'scheduledAt': 1481379659.1530972,
'script': '00-start/start.sh',
'simulate': True,
'skip': False,
'stdout': '',
'taskDependencies': {},
'tasks': {}
},
]
}
subprocessMock.return_value = ('JobID|JobName|State|Elapsed|Nodelist\n'
'34|name1|RUNNING|01:32:00|cpu-2\n'
'56|name2|COMPLETED|04:32:00|cpu-3\n')
sps = SlurmPipelineStatus(status)
self.assertEqual(
'''\
Scheduled by: sally
Scheduled at: 2016-12-10 14:20:58
Scheduling arguments:
First step: None
Force: False
Last step: None
Nice: None
Sleep: 5.00
Script arguments: hey you
Skip: start-step
Start after the following 2 jobs, of which 1 (50.00%) is finished:
Job 34: JobName=name1, State=RUNNING, Elapsed=01:32:00, Nodelist=cpu-2
Job 56: JobName=name2, State=COMPLETED, Elapsed=04:32:00, Nodelist=cpu-3
Steps summary:
Number of steps: 1
Jobs emitted in total: 0
Jobs finished: 0 (100.00%)
start-step: no jobs emitted
Step 1: start-step
No dependencies.
No tasks emitted by this step
Collect step: False
Error step: False
Working directory: 00-start
Scheduled at: 2016-12-10 14:20:59
Script: 00-start/start.sh
Simulate: True
Skip: False
Slurm pipeline environment variables:
SP_FORCE: 1
SP_SKIP: 0''', sps.toStr())