def func_for_mock_tmgr_test(mq_hostname, port, pending_queue, completed_queue):
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=mq_hostname, port=port))
mq_channel = mq_connection.channel()
tasks = list()
for _ in range(16):
t = Task()
t.state = states.SCHEDULING
t.executable = '/bin/echo'
tasks.append(t.to_dict())
tasks_as_json = json.dumps(tasks)
mq_channel.basic_publish(exchange='',
routing_key=pending_queue,
body=tasks_as_json)
cnt = 0
while cnt < 15:
method_frame, props, body = mq_channel.basic_get(queue=completed_queue)
if body:
task = Task()
task.from_dict(json.loads(body))
if task.state == states.DONE:
cnt += 1
mq_channel.basic_ack(delivery_tag=method_frame.delivery_tag)
mq_connection.close()
def func_for_mock_tmgr_test(mq_hostname, port, pending_queue, completed_queue):
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(
host=mq_hostname, port=port))
mq_channel = mq_connection.channel()
tasks = list()
for _ in range(16):
task = Task()
task.state = states.SCHEDULING
task.executable = '/bin/echo'
tasks.append(task.to_dict())
tasks_as_json = json.dumps(tasks)
mq_channel.basic_publish(exchange='',
routing_key=pending_queue,
body=tasks_as_json)
cnt = 0
while cnt < 15:
method_frame, props, body = mq_channel.basic_get(queue=completed_queue)
if not body:
continue
task = Task()
task.from_dict(json.loads(body))
if task.state == states.DONE:
cnt += 1
mq_channel.basic_ack(delivery_tag=method_frame.delivery_tag)
mq_connection.close()
def test_wfp_workflow_incomplete():
p = Pipeline()
s = Stage()
t = Task()
t.executable = ['/bin/date']
s.add_tasks(t)
p.add_stages(s)
amgr = Amgr(hostname=hostname, port=port)
amgr._setup_mqs()
wfp = WFprocessor(sid=amgr._sid,
workflow=[p],
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=False)
wfp._initialize_workflow()
assert wfp.workflow_incomplete()
amgr.workflow = [p]
profiler = ru.Profiler(name='radical.entk.temp')
p.stages[0].state == states.SCHEDULING
p.state == states.SCHEDULED
for t in p.stages[0].tasks:
t.state = states.COMPLETED
import json
import pika
task_as_dict = json.dumps(t.to_dict())
mq_connection = pika.BlockingConnection(
pika.ConnectionParameters(host=amgr._mq_hostname, port=amgr._port))
mq_channel = mq_connection.channel()
mq_channel.basic_publish(exchange='',
routing_key='%s-completedq-1' % amgr._sid,
body=task_as_dict)
amgr._terminate_sync = Event()
sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread')
sync_thread.start()
proc = Process(target=func_for_dequeue_test,
name='temp-proc',
args=(wfp, ))
proc.start()
proc.join()
amgr._terminate_sync.set()
sync_thread.join()
assert not wfp.workflow_incomplete()
def test_wfp_dequeue():
p = Pipeline()
s = Stage()
t = Task()
t.executable = '/bin/date'
s.add_tasks(t)
p.add_stages(s)
amgr = Amgr(hostname=hostname, port=port)
amgr._setup_mqs()
wfp = WFprocessor(sid=amgr._sid,
workflow=[p],
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._hostname,
port=amgr._port,
resubmit_failed=False)
wfp.initialize_workflow()
assert p.state == states.INITIAL
assert p.stages[0].state == states.INITIAL
for t in p.stages[0].tasks:
assert t.state == states.INITIAL
p.state == states.SCHEDULED
p.stages[0].state == states.SCHEDULING
for t in p.stages[0].tasks:
t.state = states.COMPLETED
task_as_dict = json.dumps(t.to_dict())
mq_connection = pika.BlockingConnection(
pika.ConnectionParameters(host=amgr._hostname, port=amgr._port))
mq_channel = mq_connection.channel()
mq_channel.basic_publish(exchange='',
routing_key='%s' % amgr._completed_queue[0],
body=task_as_dict)
wfp.start_processor()
th = mt.Thread(target=func_for_dequeue_test, name='temp-proc', args=(p, ))
th.start()
th.join()
wfp.terminate_processor()
assert p.state == states.DONE
assert p.stages[0].state == states.DONE
for t in p.stages[0].tasks:
assert t.state == states.DONE
def test_wfp_workflow_incomplete():
p = Pipeline()
s = Stage()
t = Task()
t.executable = ['/bin/date']
s.add_tasks(t)
p.add_stages(s)
amgr = Amgr(hostname=hostname, port=port)
amgr._setup_mqs()
wfp = WFprocessor(sid=amgr._sid,
workflow=[p],
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=False)
wfp._initialize_workflow()
assert wfp.workflow_incomplete()
amgr.workflow = [p]
profiler = ru.Profiler(name='radical.entk.temp')
p.stages[0].state == states.SCHEDULING
p.state == states.SCHEDULED
for t in p.stages[0].tasks:
t.state = states.COMPLETED
import json
import pika
task_as_dict = json.dumps(t.to_dict())
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=amgr._mq_hostname, port=amgr._port))
mq_channel = mq_connection.channel()
mq_channel.basic_publish(exchange='',
routing_key='%s-completedq-1' % amgr._sid,
body=task_as_dict)
amgr._terminate_sync = Event()
sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread')
sync_thread.start()
proc = Process(target=func_for_dequeue_test, name='temp-proc', args=(wfp,))
proc.start()
proc.join()
amgr._terminate_sync.set()
sync_thread.join()
assert not wfp.workflow_incomplete()
def test_wfp_workflow_incomplete():
p = Pipeline()
s = Stage()
t = Task()
t.executable = '/bin/date'
s.add_tasks(t)
p.add_stages(s)
amgr = Amgr(hostname=hostname, port=port, username=username,
password=password)
amgr._setup_mqs()
wfp = WFprocessor(sid=amgr._sid,
workflow=[p],
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
rmq_conn_params=amgr._rmq_conn_params,
resubmit_failed=False)
for t in p.stages[0].tasks:
t.state = states.COMPLETED
task_as_dict = json.dumps(t.to_dict())
credentials = pika.PlainCredentials(amgr._username, amgr._password)
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(
host=amgr._hostname, port=amgr._port,
credentials=credentials))
mq_channel = mq_connection.channel()
mq_channel.basic_publish(exchange = '',
routing_key = '%s' % amgr._completed_queue[0],
body = task_as_dict)
wfp.start_processor()
th = mt.Thread(target=func_for_dequeue_test, name='temp-proc', args=(p,))
th.start()
th.join()
wfp.terminate_processor()
assert not wfp.workflow_incomplete()
def helper(self):
# This function extracts currently tasks from the pending_queue
# and pushes it to the executed_queue. Thus mimicking an execution plugin
try:
self._logger.info('Helper process started')
# Thread should run till terminate condtion is encountered
mq_connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self._mq_hostname))
mq_channel = mq_connection.channel()
while not self._helper_terminate.is_set():
try:
method_frame, header_frame, body = mq_channel.basic_get(
queue=self._pending_queue[0])
if body:
try:
task = Task()
task.load_from_dict(json.loads(body))
task.state = states.DONE
task_as_dict = json.dumps(task.to_dict())
self._logger.debug(
'Got task %s from pending_queue %s' %
(task.uid, self._pending_queue[0]))
mq_channel.basic_publish(
exchange='fork',
routing_key='',
body=task_as_dict
#properties=pika.BasicProperties(
# make message persistent
# delivery_mode = 2,
#)
)
self._logger.debug(
'Pushed task %s with state %s to completed queue %s and synchronizerq'
% (task.uid, task.state,
self._completed_queue[0]))
mq_channel.basic_ack(
delivery_tag=method_frame.delivery_tag)
except Exception, ex:
# Rolling back queue and task status
self._logger.error(
'Error while pushing task to completed queue, rolling back: %s'
% ex)
raise UnknownError(text=ex)
if slow_run:
time.sleep(1)
except Exception, ex:
self._logger.error(
'Error getting messages from pending queue: %s' % ex)
raise UnknownError(text=ex)
except KeyboardInterrupt:
self._logger.error(
'Execution interrupted by user (you probably hit Ctrl+C), ' +
'trying to cancel enqueuer thread gracefully...')
raise KeyboardInterrupt
except Exception, ex:
self._logger.error('Unknown error in helper process: %s' % ex)
print traceback.format_exc()
raise UnknownError(text=ex)
