This test is a copy of the e2e_io_tasks.sh test with additional checks
on the objects returned by dsub.call().
"""
import os
import sys
import test_setup_e2e as test
import test_util
if not os.environ.get('CHECK_RESULTS_ONLY'):
print 'Launching pipeline...'
# pyformat: disable
launched_job = test.run_dsub([
'--script', '%s/script_io_test.sh' % test.TEST_DIR,
'--tasks', test.TASKS_FILE,
'--wait'])
# pyformat: enable
# Sanity check launched_jobs - should have a single record with 3 tasks
if not launched_job:
print >> sys.stderr, 'No launched jobs returned'
sys.exit(1)
if not launched_job.get('job-id'):
print >> sys.stderr, 'Launched job contains no job id'
sys.exit(1)
if not launched_job.get('user-id'):
print >> sys.stderr, 'Launched job contains no user-id.'
sys.exit(1)
# Because this may be invoked from another directory (treated as a library) or
# invoked localy (treated as a binary) both import styles need to be supported.
# pylint: disable=g-import-not-at-top
try:
from . import test_setup_e2e as test
except ImportError:
import test_setup_e2e as test
if not os.environ.get('CHECK_RESULTS_ONLY'):
# (1) Launch a job to test command execution failure
print('Launch a job that should fail (--wait for it)...')
try:
# pyformat: disable
bad_job_wait = test.run_dsub([
'--command', 'exit 1',
'--wait'])
# pyformat: enable
print('Expected to throw dsub_errors.JobExecutionError', file=sys.stderr)
sys.exit(1)
except dsub_errors.JobExecutionError as e:
if len(e.error_list) != 1:
print('Expected 1 error during wait, got: %s' % e.error_list,
file=sys.stderr)
sys.exit(1)
# (2) Launch a bad job to allow the next call to detect its failure
print('Launch a job that should fail (don\'t --wait)')
# pyformat: disable
bad_job_previous = test.run_dsub(['--command', 'sleep 5s && exit 1'])
# Build up an array of lines for the TSV.
with open(test.TASKS_FILE, 'w') as f:
f.write('--env TASK_ID\t--input INPUT_PATH\t--output OUTPUT_PATH\n')
for i in range(len(INPUT_BAMS)):
f.write(
'TASK_{task}\t{input}\t{output_path}/{task}/*.md5\n'.format(
task=i + 1, input=INPUT_BAMS[i], output_path=test.OUTPUTS))
print 'Launching pipeline...'
# pyformat: disable
launched_job = test.run_dsub([
'--script',
'%s/script_io_test.sh' % test.TEST_DIR, '--tasks', test.TASKS_FILE,
'--env',
'TEST_NAME=%s' % test.TEST_NAME, '--input',
'POPULATION_FILE=%s' % POPULATION_FILE, '--output',
'OUTPUT_POPULATION_FILE=%s/*' % test.OUTPUTS, '--wait'
])
# pyformat: enable
# Sanity check launched_jobs - should have a single record with 3 tasks
if not launched_job:
print >> sys.stderr, 'No launched jobs returned'
sys.exit(1)
if not launched_job.get('job-id'):
print >> sys.stderr, 'Launched job contains no job id'
sys.exit(1)
if not launched_job.get('user-id'):
# invoked localy (treated as a binary) both import styles need to be supported.
# pylint: disable=g-import-not-at-top
try:
from . import test_setup_e2e as test
from . import test_util
except SystemError:
import test_setup_e2e as test
import test_util
if not os.environ.get('CHECK_RESULTS_ONLY'):
print('Launching pipeline...')
# pyformat: disable
launched_job = test.run_dsub([
'--script', '%s/script_env_test.sh' % test.TEST_DIR,
'--env', 'VAR1=VAL1', 'VAR2=VAL2', 'VAR3=VAL3',
'--env', 'VAR4=VAL4',
'--env', 'VAR5=VAL5',
'--wait'])
# pyformat: enable
# Sanity check launched_jobs - should have a single record with no tasks
if not launched_job:
print('No launched jobs returned.', file=sys.stderr)
sys.exit(1)
if not launched_job.get('job-id'):
print('Launched job contains no job-id.', file=sys.stderr)
sys.exit(1)
if not launched_job.get('user-id'):
print('Launched job contains no user-id.', file=sys.stderr)
from dsub.lib import dsub_errors
import test_setup_e2e as test
import test_util
TEST_FILE_PATH_1 = test.OUTPUTS + '/testfile1.txt'
TEST_FILE_PATH_2 = test.OUTPUTS + '/testfile2.txt'
if not os.environ.get('CHECK_RESULTS_ONLY'):
print 'Launching pipeline...'
# (1) Launch a simple job that should succeed after a short wait
# pyformat: disable
launched_job = test.run_dsub([
'--command', 'sleep 5s && echo "hello world" > "${OUT}"', '--output',
'OUT=%s' % TEST_FILE_PATH_1
])
# pyformat: enable
# (2) Wait for the previous job and then launch a new one that blocks
# until exit
# pyformat: disable
next_job = test.run_dsub([
'--after', launched_job['job-id'], '--input',
'IN=%s' % TEST_FILE_PATH_1, '--output',
'OUT=%s' % TEST_FILE_PATH_2, '--wait', '--command',
'cat "${IN}" > "${OUT}"'
])
# pyformat: enable
# (3) Launch a job to test command execution failure
from . import test_setup_e2e as test
from . import test_util
except SystemError:
import test_setup_e2e as test
import test_util
TEST_FILE_PATH_1 = test.OUTPUTS + '/testfile1.txt'
TEST_FILE_PATH_2 = test.OUTPUTS + '/testfile2.txt'
if not os.environ.get('CHECK_RESULTS_ONLY'):
# (1) Launch a simple job that should succeed after a short wait
print('Launch a job (and don\'t --wait)...')
# pyformat: disable
launched_job = test.run_dsub([
'--command', 'sleep 5s && echo "hello world" > "${OUT}"', '--output',
'OUT=%s' % TEST_FILE_PATH_1
])
# pyformat: enable
# (2) Wait for the previous job and then launch a new one that blocks
# until exit
print('Launch a job (--after the previous, and then --wait)...')
# pyformat: disable
next_job = test.run_dsub([
'--after', launched_job['job-id'], '--input',
'IN=%s' % TEST_FILE_PATH_1, '--output',
'OUT=%s' % TEST_FILE_PATH_2, '--wait', '--command',
'cat "${IN}" > "${OUT}"'
])
# pyformat: enable