def test_graph(self):
"""Validates graph call on pipelines."""
p = Pipe("test")
# Validate graph call on empty pipe.
try:
p.graph()
except Exception:
self.fail("graph() on empty pipe should not throw any exception")
# Validate graph on a pipe with jobs.
def dummy():
pass
def dummy1():
pass
p1 = Pipe("test1")
p1.add_jobs([Job(dummy)])
p1.add_jobs([Job(dummy), Job(dummy1)], run_in_parallel=True)
p.add_jobs([p1])
p.add_jobs([Job(dummy), Job(dummy1)], run_in_parallel=True)
p.add_jobs([BashJob(["ls"])])
try:
p.graph()
except Exception:
self.fail("graph() should not throw any exception")
def test_add_stages(self):
"""Validates the flow created due to submission of jobs to pipeline."""
def dummy():
pass
j1 = Job(dummy)
b1 = BashJob(["ls"])
p = Pipe("test")
# Validate a new pipe contains no jobs by default.
self.assertEqual([], list(p.job_map.values()))
# Validate empty job list does nothing but doesn't throw an error
# either.
p.add_stage()
self.assertEqual([], list(p.job_map.values()))
# Validate the structure of the jobs submittted.
p.add_stage(j1)
p.add_stage(b1)
p.add_stage(b1)
p.add_stage(j1)
# Builder pattern
p.add_stage(j1, b1).add_stage(b1, j1)
expected = [[j1], [b1], [b1], [j1], [j1, b1], [b1, j1]]
for (a, b) in zip(p.job_map.values(), expected):
self.assertEqual(
a, b, "Jobset in the current stage of pipeline is different")
def test_validation(self):
"""Tests the validation of job list submitted to Pipe object."""
def dummy():
pass
job = Job(dummy)
bashjob = BashJob(["ls"])
pipe = Pipe("dummy")
try:
# Validate job object list is valid list to submit.
Pipe._validate([job])
# Validate bash job object list is valid list to submit.
Pipe._validate([bashjob])
# Validate pipe object list is valid list to submit.
Pipe._validate([pipe])
# Validate lists including both are valid list to submit.
Pipe._validate([job, bashjob, pipe])
except AssertionError:
self.fail("Submit validation raised AssertionError unexpectedly!")
# Validate wrong types of input raise AssertionError.
with self.assertRaises(AssertionError):
Pipe._validate([1])
with self.assertRaises(AssertionError):
Pipe._validate(["test_string"])
def test_string_representation(self):
"""Tests printable representation of Jobs."""
def dummy(msg):
pass
self.assertEqual(
"Job(function=dummy, args=('a',), kwargs={'msg': 'a'})",
Job(dummy, ('a', ), {
'msg': 'a'
}).__repr__(), 'Job representation not as expected')
self.assertEqual(
"Job(function=dummy, args=('a',), kwargs={'msg': 'a'})",
Job(dummy, ('a', ), {
'msg': 'a'
}).__str__(), 'Job representation not as expected')
def test_basic_flow(self):
"""Validates the flow created due to submission of jobs to pipeline."""
def dummy():
pass
j1 = Job(dummy)
b1 = BashJob(['ls'])
p = Pipe('test')
# Validate a new pipe contains no jobs by default.
self.assertEqual([], list(p.job_map.values()))
# Validate empty job list does nothing but doesn't throw an error
# either.
p.add_jobs([])
self.assertEqual([], list(p.job_map.values()))
# Validate the structure of the jobs submittted.
p.add_jobs([j1, b1])
p.add_jobs([b1, j1])
p.add_jobs([j1, b1], True)
p.add_jobs([b1, j1], True)
expected = [[j1], [b1], [b1], [j1], [j1, b1], [b1, j1]]
for (a, b) in zip(p.job_map.values(), expected):
self.assertEqual(
a, b, 'Jobset in the current stage of pipeline is different')
def test_safe_start(self):
"""Tests start() thread method can be called safely."""
def dummy():
pass
def corrupt():
raise Exception("Corrupt")
tw = ThreadWrapper(Job(function=dummy))
self.assertEqual(tw.state, ThreadState.INIT)
tw.start()
self.assertEqual(tw.state, ThreadState.SUCCESS)
tw = ThreadWrapper(Job(function=corrupt))
self.assertEqual(tw.state, ThreadState.INIT)
tw.start()
self.assertEqual(tw.state, ThreadState.FAILED)
def test_default_args(self):
"""Validates the default arguments set for jobs."""
def dummy():
pass
j = Job(dummy)
self.assertEqual((), j.args, 'args should be set to `()` by default')
self.assertEqual({}, j.kwargs,
'kwargs should be set to `\{\}` by default')
def test_run(self):
"""Validates run call on pipelines."""
p = Pipe("test")
# Validate run call on empty pipe.
try:
p.run()
except Exception:
self.fail("run() on empty pipe should not throw any exception")
def dummy():
pass
p.add_jobs([Job(dummy), BashJob(["pwd"])])
try:
p.run()
except Exception:
self.fail("run() should not throw any exception")
def test_create_thread_for(self):
"""Tests creation of threads for various jobs."""
# Validate thread creation for python callable.
def dummy(msg):
pass
t = Pipe._create_thread_for(Job(dummy, args=('a', )))
self.assertEqual(type(t), Thread,
'Thread for Job created successfully')
# Validate thread creation for Pipe object.
p = Pipe('test')
t = Pipe._create_thread_for(p)
self.assertEqual(type(t), Thread,
'Thread for Pipe created successfully')
# Validate thread creation for bash job.
t = Pipe._create_thread_for(BashJob(['ls']))
self.assertEqual(type(t), Thread,
'Thread for BashJob created successfully')
from pypette import BashJob, Job, Pipe
def print_job(message):
"""Sample method which takes a message to print."""
print(threading.currentThread().getName(), 'Starting')
sleep(1)
print('From within print_job : ' + str(message))
print(threading.currentThread().getName(), 'Ending')
# Create pipeline p2
p2 = Pipe('p2')
p2.add_jobs([
Job(print_job, ('p2 1', )),
Job(print_job, ('p2 2', )),
])
p2.add_jobs([
Job(print_job, ('p2 3.1', )),
Job(print_job, ('p2 3.2', )),
],
run_in_parallel=True)
p2.add_jobs([
Job(print_job, ('p2 4', )),
])
# Create pipeline p1
p1 = Pipe('p1')
p1.add_jobs([
Job(print_job, ('p1 1', )),
def test_function_validation(self):
"""Tests the function validation mechanism."""
with self.assertRaises(AssertionError):
Job(1)
with self.assertRaises(AssertionError):
Job("test")
def test_equality(self):
"""Validates equality of jobs."""
def dummy(msg):
pass
def dummy1(msg):
pass
self.assertEqual(Job(dummy, args=('a', )), Job(dummy, args=('a', )))
self.assertNotEqual(Job(dummy, args=('a', )), Job(dummy1,
args=('a', )))
self.assertNotEqual(Job(dummy, args=('a', )), Job(dummy, args=('b', )))
self.assertEqual(Job(dummy, kwargs={'msg': 'a'}),
Job(dummy, kwargs={'msg': 'a'}))
self.assertNotEqual(Job(dummy, kwargs={'msg': 'a'}),
Job(dummy1, kwargs={'msg': 'a'}))
self.assertNotEqual(Job(dummy, kwargs={'msg': 'a'}),
Job(dummy, kwargs={'msg': 'b'}))
sleep(1)
print("From within print_job : " + str(message))
print(threading.currentThread().getName(), "Ending")
def print_corrupt(message):
print(threading.currentThread().getName(), "Starting")
sleep(1)
print("From within print_corrupt : " + str(message))
print(threading.currentThread().getName(), "Ending")
raise Exception("lal")
# Create pipeline p2
p2 = Pipe("p2")
p2.add_jobs([Job(print_corrupt, ("p2 1", )), Job(print_job, ("p2 2", ))])
p2.add_jobs(
[Job(print_job, ("p2 3.1", )),
Job(print_job, ("p2 3.2", ))],
run_in_parallel=True,
)
p2.add_jobs([Job(print_job, ("p2 4", ))])
# Create pipeline p1
p1 = Pipe("p1")
p1.add_jobs([Job(print_job, ("p1 1", ))])
p1.add_jobs([Job(print_job, ("p1 2.1", )), p2], run_in_parallel=True)
# Create pipeline p
# When created as Pipe("p") or Pipe("p", Gate.FAIL_FAST) the pipeline exits the
# execution in the first stage where it encounters exception.
from pypette import Job, Pipe
def print_job(message):
"""Sample method which takes a message to print."""
print(threading.currentThread().getName(), 'Starting')
sleep(1)
print('From within print_job : ' + str(message))
print(threading.currentThread().getName(), 'Ending')
# Create pipeline p2
p2 = Pipe('p2')
p2.add_jobs([
Job(print_job, ('p2 1', )),
Job(print_job, ('p2 2', )),
])
p2.add_jobs([
Job(print_job, ('p2 3.1', )),
Job(print_job, ('p2 3.2', )),
],
run_in_parallel=True)
p2.add_jobs([
Job(print_job, ('p2 4', )),
])
# Create pipeline p1
p1 = Pipe('p1')
p1.add_jobs([
Job(print_job, ('p1 1', )),
def test_equality(self):
"""Validates equality of jobs."""
def dummy(msg):
pass
def dummy1(msg):
pass
self.assertEqual(Job(dummy, args=("a", )), Job(dummy, args=("a", )))
self.assertNotEqual(Job(dummy, args=("a", )), Job(dummy1,
args=("a", )))
self.assertNotEqual(Job(dummy, args=("a", )), Job(dummy, args=("b", )))
self.assertEqual(Job(dummy, kwargs={"msg": "a"}),
Job(dummy, kwargs={"msg": "a"}))
self.assertNotEqual(Job(dummy, kwargs={"msg": "a"}),
Job(dummy1, kwargs={"msg": "a"}))
self.assertNotEqual(Job(dummy, kwargs={"msg": "a"}),
Job(dummy, kwargs={"msg": "b"}))