def test_write_loads_pbs_template_on_initial_call(self):
self.setup_mock_read()
bp = BasePipe(job_name='foo')
bp.write_script('pbs_file')
self.assertIsNotNone(bp.pbs_template)
self.assertEqual(bp.pbs_template, 'Hello world')
def create_pipe_to_run(self):
bp = BasePipe(job_name='job_name')
bp.pbs_file = 'script.pbs'
patcher = patch.object(os.path, 'isfile', return_value=True)
self.mock_isfile = patcher.start()
self.addCleanup(patcher.stop)
return bp
def test_add_supports_chaining(self):
bp = BasePipe()
a = self.mock_cmd()
b = self.mock_cmd()
bp.add(a).add(b)
self.assertEqual(bp.cmds, [a, b])
def test_add_accepts_multiple_commands(self):
bp = BasePipe()
a = self.mock_cmd()
b = self.mock_cmd()
bp.add(a, b)
self.assertEqual(bp.cmds, [a, b])
def test_add_links_commands(self):
bp = BasePipe()
a = self.mock_cmd()
b = self.mock_cmd()
bp.add(a, b)
a.link.assert_called_once_with(b)
def test_add_links_output_of_first_command_to_input_of_next(self):
bp = BasePipe()
a = self.CMD()
b = self.CMD()
bp.add(a, b)
self.assertEqual(a.output, b.input)
def test_write_does_not_load_pbs_template_on_subsequent_calls(self):
m_open = self.setup_mock_read()
bp = BasePipe(job_name='foo')
with patch.object(bp, '_BasePipe__write_pbs'):
bp.write_script('pbs_file')
bp.write_script('pbs_file')
self.assertEqual(m_open.call_count, 1, 'PBS template init > 1x')
def test_add_extends_command_list(self):
bp = BasePipe()
a = self.mock_cmd()
b = self.mock_cmd()
bp.add(a)
bp.add(b)
self.assertEqual(bp.cmds, [a, b])
def setUp(self):
super().setUp()
a = self.mock_cmd()
b = self.mock_cmd()
cmd = BasePipe()
cmd.add(a, b)
self.cmd = cmd
def test_write_script_sets_pbs_file_directory(self):
bp = BasePipe(job_name='foo')
bp.write_script(directory='~')
self.assertEqual(
bp.pbs_file, os.path.join(
path.protect('~'), 'foo_{}.pbs'.format(bp.timestamp)
)
)
def test_add_sets_timestamp_on_each_cmd(self):
bp = BasePipe()
a = self.mock_cmd()
b = self.mock_cmd()
bp.add(a, b)
self.assertEqual(bp.timestamp, a.timestamp)
self.assertEqual(bp.timestamp, b.timestamp)
def test_link_integrity_maintained_through_cmd_output(self):
bp = BasePipe()
a = self.CMD(foo='meh')
b = self.CMD()
bp.add(a, b)
a.kwargs['-foo'] = 'hehe.txt'
self.assertEqual(b.cmd(readable=False), '{} {} {}'.format(
self.CMD.INVOKE_STR, '-foo', 'hehe.txt'))
def test_output_returns_unique_combined_list_from_all_if_passed_true(self):
pipe = BasePipe()
a = Mock(output=lambda: list('abc'))
b = Mock(output=lambda: list('def'))
c = Mock(output=lambda: list('beg'))
pipe.add(a, b, c)
self.assertEqual(pipe.output(from_all=True), list('abcdefg'))
def test_add_double_check_link_integrity(self):
bp = BasePipe()
a = self.CMD(foo='meh')
b = self.CMD()
bp.add(a, b)
self.assertEqual(a.output(), ['meh'])
self.assertEqual(b.input(), ['meh'])
a.kwargs['-foo'] = 'hehe'
self.assertEqual(b.input(), ['hehe'])
def test_subpipe_should_return_expected_output(self):
a = Mock(output=lambda: list('abc'))
b = Mock(output=lambda: list('def'))
c = Mock(output=lambda: list('beg'))
subpipe = BasePipe()
subpipe.add(a, b)
pipe = BasePipe()
pipe.add(c, subpipe) # should not raise
self.assertEqual(pipe.output(), subpipe.output())
def test_pipe_may_be_used_as_command(self):
a = self.mock_cmd()
b = self.mock_cmd()
subpipe = BasePipe()
subpipe.add(a, b)
pipe = BasePipe()
pipe.add(a, subpipe) # should not raise
def test_subpipe_input_set_as_expected(self):
a = Mock(output=lambda: list('abc'))
b = Mock(output=lambda: list('def'))
c = Mock(output=lambda: list('beg'))
def ugh(x):
x.input = c.output
c.link = Mock(side_effect=ugh)
subpipe = BasePipe()
subpipe.add(a, b)
pipe = BasePipe()
pipe.add(c, subpipe) # should not raise
c.link.assert_called_once_with(subpipe)
self.assertEqual(subpipe.input(), c.output())
def test_write_attempts_to_update_permissions_on_pbs_script(self):
bp = BasePipe(job_name='foo')
bp.write_script('pbs_file')
self.mock_oschmod.assert_called_once_with(
bp.pbs_file, self.mock_osstat().st_mode.__or__())
def test_write_script_sets_pbs_file_using_job_name_and_timestamp(self):
bp = BasePipe(job_name='foo')
bp.write_script()
self.assertEqual(bp.pbs_file, 'foo_{}.pbs'.format(bp.timestamp))
def sample_pipe(self):
a = self.mock_cmd()
b = self.mock_cmd()
pipe = BasePipe()
pipe.add(a, b)
return pipe
def test_do_run_calls_has_output_cmd(self):
self.mock_cmd._has_output = Mock(return_value=True)
bp = BasePipe()
bp._do_run(self.mock_cmd)
self.mock_cmd._has_output.assert_called_once_with()
def test_run_raises_AttributeError_if_pbs_file_not_set(self):
bp = BasePipe()
with self.assertRaisesRegex(AttributeError, r'file.*not set'):
bp.run()
def test_do_run_returns_false_if_cmd_output_exists(self):
self.mock_cmd._has_output = Mock(return_value=True)
bp = BasePipe()
self.assertFalse(bp._do_run(self.mock_cmd))
def test_do_run_returns_true_if_cmd_output_does_not_exist(self):
self.mock_cmd._has_output = Mock(return_value=False)
bp = BasePipe()
self.assertTrue(bp._do_run(self.mock_cmd), 'Will not run cmd')
def test_do_run_returns_true_if_force_is_true_even_if_no_output(self):
self.mock_cmd._has_output = Mock(return_value=True)
bp = BasePipe(force=True)
self.assertTrue(bp._do_run(self.mock_cmd))
def test_run_raises_OSError_if_pbs_file_not_found(self):
bp = BasePipe()
bp.pbs_file = 'script.pbs'
with patch.object(os.path, 'isfile', return_value=False):
with self.assertRaises(OSError):
bp.run()
def pipe(file_list, genome, project_dir, force=False):
timestamp = time.strftime("%y%m%d-%H%M%S")
for f in file_list:
name = f[0]
files = f[1:]
out_dir = os.path.join(project_dir, name)
path.makedirs(out_dir)
# # 1st fast qc
# fastqc_1 = FastqcCmd(*files, o=out_dir)
#
# # trimming
# out_prefix = os.path.join(out_dir, name)
# trim = SkewerCmd(*files, o=out_prefix)
# trimmed_fastq = trim.output()
#
# # 2nd fastqc
# fastqc_2 = FastqcCmd(*trimmed_fastq, o=out_dir)
#
# # setup alignment
# # NOTE: need to check for encoding
# align_kwargs = {
# '-x': genome,
# '-S': '{}_{}.sam'.format(
# out_prefix,
# os.path.basename(genome),
# ),
# '-p': 3, # set for local (should use pbs paramters on qsub)
# }
# if len(trimmed_fastq) == 1:
# align_kwargs['U'] = trimmed_fastq[0]
# else:
# align_kwargs['1'], align_kwargs['2'] = trimmed_fastq
# align = HisatCmd(timestamp=timestamp, **align_kwargs)
# # human_kw = [m for m in ['human', 'sapien', 'G37RCh'] if m in genome]
# # if human_kw:
# # align = HisatCmd(timestamp=timestamp, **align_kwargs)
# # else:
# # align = Bowtie2Cmd(timestamp=timestamp, **align_kwargs)
#
# # samtools
# sam_sort = SamtoolsSortCmd(*(align.output()))
# sam_index = SamtoolsIndexCmd(*(sam_sort.output()))
# UPDATED
fastqc_1 = FastqcCmd(*files, o=out_dir)
# trimming
out_prefix = os.path.join(out_dir, name)
trim = SkewerCmd(*files, o=out_prefix)
# 2nd fastqc
fastqc_2 = FastqcCmd(o=out_dir)
# setup alignment
# NOTE: need to check for encoding
align_kwargs = {
'-x': genome,
'-S': '{}_{}.sam'.format(
out_prefix,
os.path.basename(genome),
),
'-p': 3, # set for local (should use pbs paramters on qsub)
}
align = HisatCmd(timestamp=timestamp, **align_kwargs)
# samtools
sam_sort = SamtoolsSortCmd()
sam_index = SamtoolsIndexCmd()
# count
kwargs = {'-bed': genome}
bedtools_multicov = BedtoolsMulticovCmd(**kwargs)
# Setup pipe
# NOTE: This is the alpha test of the pipe class.
job_name = name + '_' + os.path.basename(genome)
pipe = BasePipe(job_name=job_name, force=force)
pipe.add(
fastqc_1, trim, fastqc_2, align, sam_sort, sam_index,
bedtools_multicov,
)
# write pbs file & run
# pbs_file = '{} , timestamp, os.path.basename(genome))
pipe.write_script()
pipe.run()
