def test_issue_26():
appman = AppManager(hostname=hostname, port=port, username=username,
password=password, autoterminate=False)
appman.resource_desc = {'resource': 'local.localhost',
'walltime': 10,
'cpus' : 1,
'project' : ''}
p1 = create_pipeline()
p2 = create_pipeline()
appman.workflow = [p1]
appman.run()
appman.workflow = [p2]
appman.run()
appman.resource_terminate()
lhs = int(p1.stages[0].uid.split('.')[-1]) + 1
rhs = int(p2.stages[0].uid.split('.')[-1])
assert lhs == rhs
for t in p1.stages[0].tasks:
for tt in p2.stages[0].tasks:
lhs = int(t.uid.split('.')[-1]) + 1
rhs = int(tt.uid.split('.')[-1])
assert lhs == rhs
def test_issue_26():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/echo']
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port, autoterminate=False)
appman.resource_desc = res_dict
p1 = create_pipeline()
appman.workflow = [p1]
appman.run()
print p1.uid, p1.stages[0].uid
p2 = create_pipeline()
appman.workflow = [p2]
appman.run()
print p2.uid, p2.stages[0].uid
appman.resource_terminate()
lhs = int(p1.stages[0].uid.split('.')[-1]) + 1
rhs = int(p2.stages[0].uid.split('.')[-1])
assert lhs == rhs
for t in p1.stages[0].tasks:
for tt in p2.stages[0].tasks:
lhs = int(t.uid.split('.')[-1]) + 1
rhs = int(tt.uid.split('.')[-1])
assert lhs == rhs
def test_issue_26():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/echo']
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port, autoterminate=False)
appman.resource_desc = res_dict
p1 = create_pipeline()
appman.workflow = [p1]
appman.run()
print p1.uid, p1.stages[0].uid
p2 = create_pipeline()
appman.workflow = [p2]
appman.run()
print p2.uid, p2.stages[0].uid
appman.resource_terminate()
lhs = int(p1.stages[0].uid.split('.')[-1]) + 1
rhs = int(p2.stages[0].uid.split('.')[-1])
assert lhs == rhs
for t in p1.stages[0].tasks:
for tt in p2.stages[0].tasks:
lhs = int(t.uid.split('.')[-1]) + 1
rhs = int(tt.uid.split('.')[-1])
assert lhs == rhs
def test_shared_data():
for f in glob('%s/file*.txt' % cur_dir):
os.remove(f)
os.system('echo "Hello" > %s/file1.txt' % cur_dir)
os.system('echo "World" > %s/file2.txt' % cur_dir)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {'resource': 'local.localhost', 'walltime': 1, 'cpus': 1}
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
appman.shared_data = ['%s/file1.txt' % cur_dir, '%s/file2.txt' % cur_dir]
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
with open('%s/output.txt' % cur_dir, 'r') as fp:
assert [d.strip() for d in fp.readlines()] == ['Hello', 'World']
os.remove('%s/file1.txt' % cur_dir)
os.remove('%s/file2.txt' % cur_dir)
os.remove('%s/output.txt' % cur_dir)
def test_stage_post_exec():
global p1
p1.name = 'p1'
s = Stage()
s.name = 's1'
for t in range(NUM_TASKS):
s.add_tasks(create_single_task())
s.post_exec = condition
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 30,
'cpus': 1,
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(rts='radical.pilot', hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
def test_issue_271():
# Create Application Manager
appman = AppManager(hostname=hostname,
port=port,
username=username,
password=password)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {'resource': 'local.localhost', 'walltime': 10, 'cpus': 1}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
p = generate_pipeline()
appman.workflow = set([p])
# Run the Application Manager
appman.run()
# assert
for t in p.stages[0].tasks:
assert t.state == states.FAILED
def test_issue_214():
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': int(sleep) + 5,
'cpus': 1
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
def test_stage_post_exec():
global p1
p1.name = 'p1'
s = Stage()
s.name = 's1'
for t in range(NUM_TASKS):
s.add_tasks(create_single_task())
s.post_exec = {
'condition': condition,
'on_true': on_true,
'on_false': on_false
}
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 30,
'cpus': 1,
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(rts='radical.pilot', hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
def test_write_session_description():
amgr = AppManager(hostname=hostname, port=port)
amgr.resource_desc = {'resource' : 'xsede.stampede',
'walltime' : 59,
'cpus' : 128,
'gpus' : 64,
'project' : 'xyz',
'queue' : 'high'}
workflow = [generate_pipeline(1), generate_pipeline(2)]
amgr.workflow = workflow
amgr._wfp = WFprocessor(sid=amgr.sid,
workflow=amgr._workflow,
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
resubmit_failed=amgr._resubmit_failed,
rmq_conn_params=amgr._rmq_conn_params)
amgr._workflow = amgr._wfp.workflow
amgr._task_manager = TaskManager(sid=amgr._sid,
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
rmgr=amgr._rmgr,
rmq_conn_params=amgr._rmq_conn_params
)
write_session_description(amgr)
desc = ru.read_json('%s/radical.entk.%s.json' % (amgr._sid, amgr._sid))
src = '%s/sample_data' % pwd
assert desc == ru.read_json('%s/expected_desc_write_session.json' % src)
def test_amgr_assign_workflow():
amgr = Amgr()
with pytest.raises(TypeError):
amgr.workflow = [1, 2, 3]
with pytest.raises(TypeError):
amgr.workflow = set([1, 2, 3])
p1 = Pipeline()
p2 = Pipeline()
p3 = Pipeline()
amgr._workflow = [p1, p2, p3]
amgr._workflow = set([p1, p2, p3])
def test_amgr_assign_workflow():
amgr = Amgr()
with pytest.raises(TypeError):
amgr.workflow = [1, 2, 3]
with pytest.raises(TypeError):
amgr.workflow = set([1, 2, 3])
p1 = Pipeline()
p2 = Pipeline()
p3 = Pipeline()
amgr._workflow = [p1, p2, p3]
amgr._workflow = set([p1, p2, p3])
def test_issue_199():
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
def test_issue_214():
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': (int(sleep) // 60) + 5,
'cpus': 1
}
# Create Application Manager
appman = AppManager(hostname=hostname,
port=port,
username=username,
password=password)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
def test_amgr_synchronizer():
amgr = Amgr(hostname=host, port=port, username=username, password=password)
amgr._setup_mqs()
p = Pipeline()
s = Stage()
# Create and add 10 tasks to the stage
for cnt in range(10):
t = Task()
t.executable = 'some-executable-%s' % cnt
s.add_tasks(t)
p.add_stages(s)
p._validate()
amgr.workflow = [p]
sid = 'test.0016'
rmgr = BaseRmgr({}, sid, None, {})
tmgr = BaseTmgr(sid=sid,
pending_queue=['pending-1'],
completed_queue=['completed-1'],
rmgr=rmgr,
rmq_conn_params=amgr._rmq_conn_params,
rts=None)
amgr._rmgr = rmgr
rmgr._task_manager = tmgr
for t in p.stages[0].tasks:
assert t.state == states.INITIAL
assert p.stages[0].state == states.INITIAL
assert p.state == states.INITIAL
# Start the synchronizer method in a thread
amgr._terminate_sync = mt.Event()
sync_thread = mt.Thread(target=amgr._synchronizer,
name='synchronizer-thread')
sync_thread.start()
# Start the synchronizer method in a thread
proc = mp.Process(target=func_for_synchronizer_test, name='temp-proc',
args=(amgr._sid, p, tmgr))
proc.start()
proc.join()
# Wait for AppManager to finish the message exchange
# no need to set *)terminate_sync* but a timeout instead
# amgr._terminate_sync.set()
sync_thread.join(15)
for t in p.stages[0].tasks:
assert t.state == states.COMPLETED
def test_issue_259():
# Create Application Manager
appman = AppManager(hostname=hostname, port=port, username=username,
password=password)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
appman.workflow = set([generate_pipeline()])
# Run the Application Manager
appman.run()
# assert
with open('output.txt','r') as f:
assert '"Hello World"' == f.readlines()[0].strip()
def test_issue_271():
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
p = generate_pipeline()
appman.workflow = set([p])
# Run the Application Manager
appman.run()
# assert
for t in p.stages[0].tasks:
assert t.state == states.FAILED
def test_diff_rmq():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = '/bin/echo'
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
}
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
p1 = create_pipeline()
print(p1.uid, p1.stages[0].uid)
appman.workflow = [p1]
appman.run()
def test_amgr_assign_workflow():
amgr = Amgr(hostname=host, port=port, username=username,
password=password)
with pytest.raises(TypeError):
amgr.workflow = [1, 2, 3]
with pytest.raises(TypeError):
amgr.workflow = set([1, 2, 3])
p1 = Pipeline()
p2 = Pipeline()
p3 = Pipeline()
amgr._workflow = [p1, p2, p3]
amgr._workflow = set([p1, p2, p3])
def main():
cmd = "{0} 'ls {1}'".format(ssh, dir_)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
out, _ = p.communicate()
out = out.decode('utf-8').strip().split(linesep)
fullpaths = [op.join(dir_, p) for p in out]
print(fullpaths)
# Start radical entk pipeline
p = Pipeline()
for i in range(iterations):
s = Stage()
for fp in fullpaths:
t = Task()
t.name = 'Incrementation {}'.format(i)
t.pre_exec = [
'source /home/vhayot/miniconda3/etc/profile.d/conda.sh',
'conda activate radenv'
]
t.executable = 'python /home/vhayot/inc.py'
if i == 0:
t.arguments = [fp, out_dir, i]
else:
# Note: assuming all data is accessible through shared dir
# radical entk functions without sharedfs, however
t.arguments = [
op.join(out_dir,
"it-{0}-{1}".format(i - 1, op.basename(fp))),
out_dir, i
]
s.add_tasks(t)
# Create a new stage everytime there's a dependency
p.add_stages(s)
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = {
'resource': 'xsede.bridges',
'walltime': 20,
'cpus': 5,
'project': 'mc3bggp',
'schema': 'gsissh'
}
appman.workflow = set([p])
appman.run()
def test_wfp_workflow_incomplete():
p = Pipeline()
s = Stage()
t = Task()
t.executable = ['/bin/date']
s.add_tasks(t)
p.add_stages(s)
amgr = Amgr(hostname=hostname, port=port)
amgr._setup_mqs()
wfp = WFprocessor(sid=amgr._sid,
workflow=[p],
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=False)
wfp._initialize_workflow()
assert wfp.workflow_incomplete()
amgr.workflow = [p]
profiler = ru.Profiler(name='radical.entk.temp')
p.stages[0].state == states.SCHEDULING
p.state == states.SCHEDULED
for t in p.stages[0].tasks:
t.state = states.COMPLETED
import json
import pika
task_as_dict = json.dumps(t.to_dict())
mq_connection = pika.BlockingConnection(
pika.ConnectionParameters(host=amgr._mq_hostname, port=amgr._port))
mq_channel = mq_connection.channel()
mq_channel.basic_publish(exchange='',
routing_key='%s-completedq-1' % amgr._sid,
body=task_as_dict)
amgr._terminate_sync = Event()
sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread')
sync_thread.start()
proc = Process(target=func_for_dequeue_test,
name='temp-proc',
args=(wfp, ))
proc.start()
proc.join()
amgr._terminate_sync.set()
sync_thread.join()
assert not wfp.workflow_incomplete()
def test_state_order():
"""
**Purpose**: Test if the Pipeline, Stage and Task are assigned their states in the correct order
"""
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/date']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
os.environ['RP_ENABLE_OLD_DEFINES'] = 'True'
appman = Amgr(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
p_state_hist = p1.state_history
assert p_state_hist == ['DESCRIBED', 'SCHEDULING', 'DONE']
s_state_hist = p1.stages[0].state_history
assert s_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'DONE']
tasks = p1.stages[0].tasks
for t in tasks:
t_state_hist = t.state_history
assert t_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'SUBMITTING', 'SUBMITTED',
'EXECUTED', 'DEQUEUEING', 'DEQUEUED', 'DONE']
def test_state_order():
"""
**Purpose**: Test if the Pipeline, Stage and Task are assigned their states in the correct order
"""
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/date']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
os.environ['RP_ENABLE_OLD_DEFINES'] = 'True'
appman = Amgr(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
p_state_hist = p1.state_history
assert p_state_hist == ['DESCRIBED', 'SCHEDULING', 'DONE']
s_state_hist = p1.stages[0].state_history
assert s_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'DONE']
tasks = p1.stages[0].tasks
for t in tasks:
t_state_hist = t.state_history
assert t_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'SUBMITTING', 'SUBMITTED',
'EXECUTED', 'DEQUEUEING', 'DEQUEUED', 'DONE']
def test_amgr_synchronizer():
logger = ru.get_logger('radical.entk.temp_logger')
profiler = ru.Profiler(name='radical.entk.temp')
amgr = Amgr(hostname=hostname, port=port)
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=hostname, port=port))
mq_channel = mq_connection.channel()
amgr._setup_mqs()
p = Pipeline()
s = Stage()
# Create and add 100 tasks to the stage
for cnt in range(100):
t = Task()
t.executable = ['some-executable-%s' % cnt]
s.add_tasks(t)
p.add_stages(s)
p._assign_uid(amgr._sid)
p._validate()
amgr.workflow = [p]
for t in p.stages[0].tasks:
assert t.state == states.INITIAL
assert p.stages[0].state == states.INITIAL
assert p.state == states.INITIAL
# Start the synchronizer method in a thread
amgr._terminate_sync = Event()
sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread')
sync_thread.start()
# Start the synchronizer method in a thread
proc = Process(target=func_for_synchronizer_test, name='temp-proc',
args=(amgr._sid, p, logger, profiler))
proc.start()
proc.join()
for t in p.stages[0].tasks:
assert t.state == states.SCHEDULING
assert p.stages[0].state == states.SCHEDULING
assert p.state == states.SCHEDULING
amgr._terminate_sync.set()
sync_thread.join()
def test_wfp_workflow_incomplete():
p = Pipeline()
s = Stage()
t = Task()
t.executable = ['/bin/date']
s.add_tasks(t)
p.add_stages(s)
amgr = Amgr(hostname=hostname, port=port)
amgr._setup_mqs()
wfp = WFprocessor(sid=amgr._sid,
workflow=[p],
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=False)
wfp._initialize_workflow()
assert wfp.workflow_incomplete()
amgr.workflow = [p]
profiler = ru.Profiler(name='radical.entk.temp')
p.stages[0].state == states.SCHEDULING
p.state == states.SCHEDULED
for t in p.stages[0].tasks:
t.state = states.COMPLETED
import json
import pika
task_as_dict = json.dumps(t.to_dict())
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=amgr._mq_hostname, port=amgr._port))
mq_channel = mq_connection.channel()
mq_channel.basic_publish(exchange='',
routing_key='%s-completedq-1' % amgr._sid,
body=task_as_dict)
amgr._terminate_sync = Event()
sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread')
sync_thread.start()
proc = Process(target=func_for_dequeue_test, name='temp-proc', args=(wfp,))
proc.start()
proc.join()
amgr._terminate_sync.set()
sync_thread.join()
assert not wfp.workflow_incomplete()
def test_issue_236():
'''
Create folder temp to transfer as input to task:
.
./temp
./temp/dir1
./temp/dir1/file2.txt
./temp/file1.txt
'''
os.makedirs('%s/temp' % cur_dir)
os.makedirs('%s/temp/dir1' % cur_dir)
os.system('echo "Hello world" > %s/temp/file1.txt' % cur_dir)
os.system('echo "Hello world" > %s/temp/dir1/file2.txt' % cur_dir)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Create Application Manager
appman = AppManager(hostname=hostname, port=port, username=username,
password=password)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
# Assert folder movement
assert len(glob('/tmp/temp*')) >= 1
assert len(glob('/tmp/temp/dir*')) == 1
assert len(glob('/tmp/temp/*.txt')) == 1
assert len(glob('/tmp/temp/dir1/*.txt')) == 1
# Cleanup
shutil.rmtree('%s/temp' % cur_dir)
shutil.rmtree('/tmp/temp')
def test_issue_236():
'''
Create folder temp to transfer as input to task:
.
./temp
./temp/dir1
./temp/dir1/file2.txt
./temp/file1.txt
'''
os.makedirs('%s/temp' %cur_dir)
os.makedirs('%s/temp/dir1' %cur_dir)
os.system('echo "Hello world" > %s/temp/file1.txt' %cur_dir)
os.system('echo "Hello world" > %s/temp/dir1/file2.txt' %cur_dir)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
# Assert folder movement
assert len(glob('/tmp/temp*')) >=1
assert len(glob('/tmp/temp/dir*')) ==1
assert len(glob('/tmp/temp/*.txt')) ==1
assert len(glob('/tmp/temp/dir1/*.txt')) ==1
# Cleanup
shutil.rmtree('%s/temp' %cur_dir)
shutil.rmtree('/tmp/temp')
def test_amgr_run():
amgr = Amgr(hostname=host, port=port)
with pytest.raises(MissingError):
amgr.run()
p1 = Pipeline()
p2 = Pipeline()
p3 = Pipeline()
with pytest.raises(MissingError):
amgr.workflow = [p1, p2, p3]
def test_amgr_synchronizer():
logger = ru.Logger('radical.entk.temp_logger')
profiler = ru.Profiler(name='radical.entk.temp')
amgr = Amgr(hostname=hostname, port=port)
amgr._setup_mqs()
p = Pipeline()
s = Stage()
# Create and add 100 tasks to the stage
for cnt in range(100):
t = Task()
t.executable = ['some-executable-%s' % cnt]
s.add_tasks(t)
p.add_stages(s)
p._assign_uid(amgr._sid)
p._validate()
amgr.workflow = [p]
for t in p.stages[0].tasks:
assert t.state == states.INITIAL
assert p.stages[0].state == states.INITIAL
assert p.state == states.INITIAL
# Start the synchronizer method in a thread
amgr._terminate_sync = Event()
sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread')
sync_thread.start()
# Start the synchronizer method in a thread
proc = Process(target=func_for_synchronizer_test, name='temp-proc',
args=(amgr._sid, p, logger, profiler))
proc.start()
proc.join()
for t in p.stages[0].tasks:
assert t.state == states.SCHEDULING
assert p.stages[0].state == states.SCHEDULING
assert p.state == states.SCHEDULING
amgr._terminate_sync.set()
sync_thread.join()
def test_wfp_enqueue():
p = Pipeline()
s = Stage()
t = Task()
t.executable = ['/bin/date']
s.add_tasks(t)
p.add_stages(s)
amgr = Amgr(hostname=hostname, port=port)
amgr._setup_mqs()
wfp = WFprocessor(sid=amgr._sid,
workflow=[p],
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=False)
wfp._initialize_workflow()
amgr.workflow = [p]
profiler = ru.Profiler(name='radical.entk.temp')
for t in p.stages[0].tasks:
assert t.state == states.INITIAL
assert p.stages[0].state == states.INITIAL
assert p.state == states.INITIAL
amgr._terminate_sync = Event()
sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread')
sync_thread.start()
proc = Process(target=func_for_enqueue_test,
name='temp-proc',
args=(wfp, ))
proc.start()
proc.join()
amgr._terminate_sync.set()
sync_thread.join()
for t in p.stages[0].tasks:
assert t.state == states.SCHEDULED
assert p.stages[0].state == states.SCHEDULED
assert p.state == states.SCHEDULING
def test_issue_255():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = ['sleep']
t1.arguments = ['10']
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port, autoterminate=False)
appman.resource_desc = res_dict
p1 = create_pipeline()
appman.workflow = [p1]
appman.run()
# p1 = create_pipeline()
# appman.workflow = [p1]
# appman.run()
# appman.resource_terminate()
tmgr = appman._task_manager
tmgr.terminate_manager()
# tmgr.terminate_heartbeat()
tmgr.start_manager()
def test_write_workflow():
try:
wf = list()
wf.append(generate_pipeline(1))
wf.append(generate_pipeline(2))
amgr = AppManager(hostname=hostname, port=port)
amgr.workflow = wf
amgr._wfp = WFprocessor(sid=amgr._sid,
workflow=amgr._workflow,
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=amgr._resubmit_failed)
amgr._wfp._initialize_workflow()
wf = amgr._wfp.workflow
write_workflow(wf, 'test')
data = ru.read_json('test/entk_workflow.json')
assert len(data) == len(wf) + 1
stack = data.pop(0)
assert stack.keys() == ['stack']
assert stack['stack'].keys() == ['sys','radical']
assert stack['stack']['sys'].keys() == ["python","pythonpath","virtualenv"]
assert stack['stack']['radical'].keys() == ['saga', 'radical.pilot', 'radical.utils', 'radical.entk']
p_cnt = 0
for p in data:
assert p['uid'] == wf[p_cnt].uid
assert p['name'] == wf[p_cnt].name
assert p['state_history'] == wf[p_cnt].state_history
s_cnt = 0
for s in p['stages']:
assert s['uid'] == wf[p_cnt].stages[s_cnt].uid
assert s['name'] == wf[p_cnt].stages[s_cnt].name
assert s['state_history'] == wf[p_cnt].stages[s_cnt].state_history
for t in wf[p_cnt].stages[s_cnt].tasks:
assert t.to_dict() in s['tasks']
s_cnt += 1
p_cnt += 1
except Exception as ex:
shutil.rmtree('test')
raise
def test_issue_255():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = 'sleep'
t1.arguments = ['10']
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
appman = AppManager(hostname=hostname,
port=port,
username=username,
password=password,
autoterminate=False)
appman.resource_desc = res_dict
p1 = create_pipeline()
appman.workflow = [p1]
appman.run()
# p1 = create_pipeline()
# appman.workflow = [p1]
# appman.run()
# appman.resource_terminate()
tmgr = appman._task_manager
tmgr.terminate_manager()
# tmgr.terminate_heartbeat()
tmgr.start_manager()
def test_wfp_enqueue():
p = Pipeline()
s = Stage()
t = Task()
t.executable = ['/bin/date']
s.add_tasks(t)
p.add_stages(s)
amgr = Amgr(hostname=hostname, port=port)
amgr._setup_mqs()
wfp = WFprocessor(sid=amgr._sid,
workflow=[p],
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=False)
wfp._initialize_workflow()
amgr.workflow = [p]
profiler = ru.Profiler(name='radical.entk.temp')
for t in p.stages[0].tasks:
assert t.state == states.INITIAL
assert p.stages[0].state == states.INITIAL
assert p.state == states.INITIAL
amgr._terminate_sync = Event()
sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread')
sync_thread.start()
proc = Process(target=func_for_enqueue_test, name='temp-proc', args=(wfp,))
proc.start()
proc.join()
amgr._terminate_sync.set()
sync_thread.join()
for t in p.stages[0].tasks:
assert t.state == states.SCHEDULED
assert p.stages[0].state == states.SCHEDULED
assert p.state == states.SCHEDULING
def test_write_session_description():
amgr = AppManager(hostname=hostname,
port=port,
username=username,
password=password)
amgr.resource_desc = {
'resource': 'xsede.stampede',
'walltime': 59,
'cpus': 128,
'gpus': 64,
'project': 'xyz',
'queue': 'high'
}
workflow = [generate_pipeline(1), generate_pipeline(2)]
amgr.workflow = workflow
amgr._wfp = WFprocessor(sid=amgr.sid,
workflow=amgr._workflow,
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
resubmit_failed=amgr._resubmit_failed,
rmq_conn_params=amgr._rmq_conn_params)
amgr._workflow = amgr._wfp.workflow
amgr._task_manager = TaskManager(sid=amgr._sid,
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
rmgr=amgr._rmgr,
rmq_conn_params=amgr._rmq_conn_params)
write_session_description(amgr)
desc = ru.read_json('%s/radical.entk.%s.json' % (amgr._sid, amgr._sid))
# tasks are originally set but saved as a list in json
# uses sorting for convenient comparison, this doesn't change validity
for k, v in (desc['tree'].items()):
if k.startswith("stage"):
desc['tree'][k]['children'] = sorted(v['children'])
src = '%s/sample_data' % pwd
assert desc == ru.read_json('%s/expected_desc_write_session.json' % src)
def test_write_session_description():
hostname = os.environ.get('RMQ_HOSTNAME', 'localhost')
port = int(os.environ.get('RMQ_PORT', 5672))
amgr = AppManager(hostname=hostname, port=port)
amgr.resource_desc = {
'resource': 'xsede.stampede',
'walltime': 60,
'cpus': 128,
'gpus': 64,
'project': 'xyz',
'queue': 'high'
}
workflow = [generate_pipeline(1), generate_pipeline(2)]
amgr.workflow = workflow
amgr._wfp = WFprocessor(sid=amgr._sid,
workflow=amgr._workflow,
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=amgr._resubmit_failed)
amgr._wfp._initialize_workflow()
amgr._workflow = amgr._wfp.workflow
amgr._task_manager = TaskManager(sid=amgr._sid,
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
rmgr=amgr._resource_manager,
port=amgr._port
)
# os.mkdir(amgr._sid)
write_session_description(amgr)
desc = ru.read_json('%s/radical.entk.%s.json' % (amgr._sid, amgr._sid))
curdir = os.path.dirname(os.path.abspath(__file__))
src = '%s/sample_data' % curdir
assert desc == ru.read_json('%s/expected_desc_write_session.json' % src)
def test_shared_data():
for f in glob('%s/file*.txt' %cur_dir):
os.remove(f)
os.system('echo "Hello" > %s/file1.txt' %cur_dir)
os.system('echo "World" > %s/file2.txt' %cur_dir)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 1,
'cpus': 1
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
appman.shared_data = ['%s/file1.txt' %cur_dir, '%s/file2.txt' %cur_dir]
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
with open('%s/output.txt' %cur_dir, 'r') as fp:
assert [d.strip() for d in fp.readlines()] == ['Hello', 'World']
os.remove('%s/file1.txt' %cur_dir)
os.remove('%s/file2.txt' %cur_dir)
os.remove('%s/output.txt' %cur_dir)
def test_integration_local():
"""
**Purpose**: Run an EnTK application on localhost
"""
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = '/bin/echo'
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
def test_integration_local():
"""
**Purpose**: Run an EnTK application on localhost
"""
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/echo']
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
def test_amgr_run_mock():
p = Pipeline()
s = Stage()
t = Task()
t.name = 'simulation'
t.executable = '/bin/date'
s.tasks = t
p.add_stages(s)
res_dict = {'resource': 'local.localhost',
'walltime': 5,
'cpus' : 1,
'project' : ''}
appman = Amgr(hostname=host, port=port, rts="mock")
appman.resource_desc = res_dict
appman.workflow = [p]
appman.run()
def test_write_workflow():
wf = list()
wf.append(generate_pipeline(1))
wf.append(generate_pipeline(2))
amgr = AppManager(hostname=hostname, port=port)
amgr.workflow = wf
amgr._wfp = WFprocessor(sid=amgr._sid,
workflow=amgr._workflow,
pending_queue=amgr._pending_queue,
completed_queue=amgr._completed_queue,
mq_hostname=amgr._mq_hostname,
port=amgr._port,
resubmit_failed=amgr._resubmit_failed)
amgr._wfp._initialize_workflow()
wf = amgr._wfp.workflow
write_workflow(wf, 'test')
data = ru.read_json('test/entk_workflow.json')
assert len(data) == len(wf)
p_cnt = 0
for p in data:
assert p['uid'] == wf[p_cnt].uid
assert p['name'] == wf[p_cnt].name
assert p['state_history'] == wf[p_cnt].state_history
s_cnt = 0
for s in p['stages']:
assert s['uid'] == wf[p_cnt].stages[s_cnt].uid
assert s['name'] == wf[p_cnt].stages[s_cnt].name
assert s['state_history'] == wf[p_cnt].stages[s_cnt].state_history
for t in wf[p_cnt].stages[s_cnt].tasks:
assert t.to_dict() in s['tasks']
s_cnt += 1
p_cnt += 1
shutil.rmtree('test')
def test_diff_rmq():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/echo']
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
p1 = create_pipeline()
print p1.uid, p1.stages[0].uid
appman.workflow = [p1]
appman.run()
def test_suspend_pipeline():
# Create a dictionary describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost_anaconda',
'walltime': 15,
'cpus': 2,
}
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
def test_amgr_run_mock():
p = Pipeline()
s = Stage()
t = Task()
t.name = 'simulation'
t.executable = ['/bin/date']
s.tasks = t
p.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
appman = Amgr(hostname=hostname, port=port, rts="mock")
appman.resource_desc = res_dict
appman.workflow = [p]
appman.run()
if __name__ == '__main__':
pipelines = []
for cnt in range(10):
pipelines.append(generate_pipeline())
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
appman.workflow = set(pipelines)
# Run the Application Manager
appman.run()
# Create a dictionary to describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.summit',
'queue': 'batch',
'schema': 'local',
'walltime': 15,
'cpus': 48,
'gpus': 4
}
# Create Application Manager
appman = AppManager()
appman.resource_desc = res_dict
p1 = generate_MD_pipeline()
p2 = generate_ML_pipeline()
pipelines = []
pipelines.append(p1)
pipelines.append(p2)
# Assign the workflow as a list of Pipelines to the Application Manager. In
# this way, all the pipelines in the list will execute concurrently.
appman.workflow = pipelines
# Run the Application Manager
appman.run()
def test_rp_da_scheduler_bw():
"""
**Purpose**: Run an EnTK application on localhost
"""
p1 = Pipeline()
p1.name = 'p1'
n = 10
s1 = Stage()
s1.name = 's1'
for x in range(n):
t = Task()
t.name = 't%s'%x
t.executable = ['/bin/hostname']
t.arguments = ['>','hostname.txt']
t.cpu_reqs['processes'] = 1
t.cpu_reqs['threads_per_process'] = 16
t.cpu_reqs['thread_type'] = ''
t.cpu_reqs['process_type'] = ''
t.lfs_per_process = 10
t.download_output_data = ['hostname.txt > s1_t%s_hostname.txt'%(x)]
s1.add_tasks(t)
p1.add_stages(s1)
s2 = Stage()
s2.name = 's2'
for x in range(n):
t = Task()
t.executable = ['/bin/hostname']
t.arguments = ['>','hostname.txt']
t.cpu_reqs['processes'] = 1
t.cpu_reqs['threads_per_process'] = 16
t.cpu_reqs['thread_type'] = ''
t.cpu_reqs['process_type'] = ''
t.download_output_data = ['hostname.txt > s2_t%s_hostname.txt'%(x)]
t.tag = 't%s'%x
s2.add_tasks(t)
p1.add_stages(s2)
res_dict = {
'resource' : 'ncsa.bw_aprun',
'walltime' : 10,
'cpus' : 128,
'project' : 'gk4',
'queue' : 'high'
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
for i in range(n):
assert open('s1_t%s_hostname.txt'%i,'r').readline().strip() == open('s2_t%s_hostname.txt'%i,'r').readline().strip()
txts = glob('%s/*.txt' % os.getcwd())
for f in txts:
os.remove(f)
t.arguments = ['Hello World'] # Assign arguments for the task executable
# Add Task to the Stage
s.add_tasks(t)
# Add Stage to the Pipeline
p.add_stages(s)
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
appman.workflow = set([p])
# Run the Application Manager
appman.run()
}
# Add Stage to the Pipeline
p.add_stages(s1)
return p
if __name__ == '__main__':
# Create a dictionary describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 15,
'cpus': 2,
}
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
}
# Add Stage to the Pipeline
p.add_stages(s1)
return p
if __name__ == '__main__':
# Create a dictionary describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 15,
'cpus': 2,
}
# Create Application Manager
appman = AppManager()
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
return p
if __name__ == '__main__':
p1 = generate_pipeline(name='Pipeline 1', stages=1)
p2 = generate_pipeline(name='Pipeline 2', stages=2)
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
appman.workflow = set([p1, p2])
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Run the Application Manager
if __name__ == '__main__':
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
# 'resource': 'ncsa.bw_aprun',
# 'walltime': 10,
# 'cpus': 32,
#'project': 'bamm',
#'queue': 'high'
'resource': 'local.localhost',
'walltime': 10,
'cpus':2
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
appman.workflow = set([generate_pipeline()])
# Run the Application Manager
appman.run()
