def test_shared_data():
for f in glob('%s/file*.txt' % cur_dir):
os.remove(f)
os.system('echo "Hello" > %s/file1.txt' % cur_dir)
os.system('echo "World" > %s/file2.txt' % cur_dir)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {'resource': 'local.localhost', 'walltime': 1, 'cpus': 1}
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
appman.shared_data = ['%s/file1.txt' % cur_dir, '%s/file2.txt' % cur_dir]
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
with open('%s/output.txt' % cur_dir, 'r') as fp:
assert [d.strip() for d in fp.readlines()] == ['Hello', 'World']
os.remove('%s/file1.txt' % cur_dir)
os.remove('%s/file2.txt' % cur_dir)
os.remove('%s/output.txt' % cur_dir)
def test_stage_post_exec():
global p1
p1.name = 'p1'
s = Stage()
s.name = 's1'
for t in range(NUM_TASKS):
s.add_tasks(create_single_task())
s.post_exec = condition
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 30,
'cpus': 1,
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(rts='radical.pilot', hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
def test_issue_271():
# Create Application Manager
appman = AppManager(hostname=hostname,
port=port,
username=username,
password=password)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {'resource': 'local.localhost', 'walltime': 10, 'cpus': 1}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
p = generate_pipeline()
appman.workflow = set([p])
# Run the Application Manager
appman.run()
# assert
for t in p.stages[0].tasks:
assert t.state == states.FAILED
def test_issue_214():
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': int(sleep) + 5,
'cpus': 1
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
def test_diff_rmq():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = '/bin/echo'
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
}
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
p1 = create_pipeline()
print(p1.uid, p1.stages[0].uid)
appman.workflow = [p1]
appman.run()
def test_issue_259():
# Create Application Manager
appman = AppManager(hostname=hostname, port=port, username=username,
password=password)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
appman.workflow = set([generate_pipeline()])
# Run the Application Manager
appman.run()
# assert
with open('output.txt','r') as f:
assert '"Hello World"' == f.readlines()[0].strip()
def test_issue_26():
appman = AppManager(hostname=hostname, port=port, username=username,
password=password, autoterminate=False)
appman.resource_desc = {'resource': 'local.localhost',
'walltime': 10,
'cpus' : 1,
'project' : ''}
p1 = create_pipeline()
p2 = create_pipeline()
appman.workflow = [p1]
appman.run()
appman.workflow = [p2]
appman.run()
appman.resource_terminate()
lhs = int(p1.stages[0].uid.split('.')[-1]) + 1
rhs = int(p2.stages[0].uid.split('.')[-1])
assert lhs == rhs
for t in p1.stages[0].tasks:
for tt in p2.stages[0].tasks:
lhs = int(t.uid.split('.')[-1]) + 1
rhs = int(tt.uid.split('.')[-1])
assert lhs == rhs
def test_issue_214():
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': (int(sleep) // 60) + 5,
'cpus': 1
}
# Create Application Manager
appman = AppManager(hostname=hostname,
port=port,
username=username,
password=password)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
def test_issue_271():
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
p = generate_pipeline()
appman.workflow = set([p])
# Run the Application Manager
appman.run()
# assert
for t in p.stages[0].tasks:
assert t.state == states.FAILED
def test_issue_199():
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
def test_stage_post_exec():
global p1
p1.name = 'p1'
s = Stage()
s.name = 's1'
for t in range(NUM_TASKS):
s.add_tasks(create_single_task())
s.post_exec = {
'condition': condition,
'on_true': on_true,
'on_false': on_false
}
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 30,
'cpus': 1,
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(rts='radical.pilot', hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
def test_issue_26():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/echo']
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port, autoterminate=False)
appman.resource_desc = res_dict
p1 = create_pipeline()
appman.workflow = [p1]
appman.run()
print p1.uid, p1.stages[0].uid
p2 = create_pipeline()
appman.workflow = [p2]
appman.run()
print p2.uid, p2.stages[0].uid
appman.resource_terminate()
lhs = int(p1.stages[0].uid.split('.')[-1]) + 1
rhs = int(p2.stages[0].uid.split('.')[-1])
assert lhs == rhs
for t in p1.stages[0].tasks:
for tt in p2.stages[0].tasks:
lhs = int(t.uid.split('.')[-1]) + 1
rhs = int(tt.uid.split('.')[-1])
assert lhs == rhs
def main():
cmd = "{0} 'ls {1}'".format(ssh, dir_)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
out, _ = p.communicate()
out = out.decode('utf-8').strip().split(linesep)
fullpaths = [op.join(dir_, p) for p in out]
print(fullpaths)
# Start radical entk pipeline
p = Pipeline()
for i in range(iterations):
s = Stage()
for fp in fullpaths:
t = Task()
t.name = 'Incrementation {}'.format(i)
t.pre_exec = [
'source /home/vhayot/miniconda3/etc/profile.d/conda.sh',
'conda activate radenv'
]
t.executable = 'python /home/vhayot/inc.py'
if i == 0:
t.arguments = [fp, out_dir, i]
else:
# Note: assuming all data is accessible through shared dir
# radical entk functions without sharedfs, however
t.arguments = [
op.join(out_dir,
"it-{0}-{1}".format(i - 1, op.basename(fp))),
out_dir, i
]
s.add_tasks(t)
# Create a new stage everytime there's a dependency
p.add_stages(s)
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = {
'resource': 'xsede.bridges',
'walltime': 20,
'cpus': 5,
'project': 'mc3bggp',
'schema': 'gsissh'
}
appman.workflow = set([p])
appman.run()
def test_state_order():
"""
**Purpose**: Test if the Pipeline, Stage and Task are assigned their states in the correct order
"""
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/date']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
os.environ['RP_ENABLE_OLD_DEFINES'] = 'True'
appman = Amgr(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
p_state_hist = p1.state_history
assert p_state_hist == ['DESCRIBED', 'SCHEDULING', 'DONE']
s_state_hist = p1.stages[0].state_history
assert s_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'DONE']
tasks = p1.stages[0].tasks
for t in tasks:
t_state_hist = t.state_history
assert t_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'SUBMITTING', 'SUBMITTED',
'EXECUTED', 'DEQUEUEING', 'DEQUEUED', 'DONE']
def test_state_order():
"""
**Purpose**: Test if the Pipeline, Stage and Task are assigned their states in the correct order
"""
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/date']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
os.environ['RP_ENABLE_OLD_DEFINES'] = 'True'
appman = Amgr(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
p_state_hist = p1.state_history
assert p_state_hist == ['DESCRIBED', 'SCHEDULING', 'DONE']
s_state_hist = p1.stages[0].state_history
assert s_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'DONE']
tasks = p1.stages[0].tasks
for t in tasks:
t_state_hist = t.state_history
assert t_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'SUBMITTING', 'SUBMITTED',
'EXECUTED', 'DEQUEUEING', 'DEQUEUED', 'DONE']
def test_issue_26():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/echo']
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port, autoterminate=False)
appman.resource_desc = res_dict
p1 = create_pipeline()
appman.workflow = [p1]
appman.run()
print p1.uid, p1.stages[0].uid
p2 = create_pipeline()
appman.workflow = [p2]
appman.run()
print p2.uid, p2.stages[0].uid
appman.resource_terminate()
lhs = int(p1.stages[0].uid.split('.')[-1]) + 1
rhs = int(p2.stages[0].uid.split('.')[-1])
assert lhs == rhs
for t in p1.stages[0].tasks:
for tt in p2.stages[0].tasks:
lhs = int(t.uid.split('.')[-1]) + 1
rhs = int(tt.uid.split('.')[-1])
assert lhs == rhs
def test_issue_236():
'''
Create folder temp to transfer as input to task:
.
./temp
./temp/dir1
./temp/dir1/file2.txt
./temp/file1.txt
'''
os.makedirs('%s/temp' %cur_dir)
os.makedirs('%s/temp/dir1' %cur_dir)
os.system('echo "Hello world" > %s/temp/file1.txt' %cur_dir)
os.system('echo "Hello world" > %s/temp/dir1/file2.txt' %cur_dir)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
# Assert folder movement
assert len(glob('/tmp/temp*')) >=1
assert len(glob('/tmp/temp/dir*')) ==1
assert len(glob('/tmp/temp/*.txt')) ==1
assert len(glob('/tmp/temp/dir1/*.txt')) ==1
# Cleanup
shutil.rmtree('%s/temp' %cur_dir)
shutil.rmtree('/tmp/temp')
def test_issue_236():
'''
Create folder temp to transfer as input to task:
.
./temp
./temp/dir1
./temp/dir1/file2.txt
./temp/file1.txt
'''
os.makedirs('%s/temp' % cur_dir)
os.makedirs('%s/temp/dir1' % cur_dir)
os.system('echo "Hello world" > %s/temp/file1.txt' % cur_dir)
os.system('echo "Hello world" > %s/temp/dir1/file2.txt' % cur_dir)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 1
}
# Create Application Manager
appman = AppManager(hostname=hostname, port=port, username=username,
password=password)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
# Assert folder movement
assert len(glob('/tmp/temp*')) >= 1
assert len(glob('/tmp/temp/dir*')) == 1
assert len(glob('/tmp/temp/*.txt')) == 1
assert len(glob('/tmp/temp/dir1/*.txt')) == 1
# Cleanup
shutil.rmtree('%s/temp' % cur_dir)
shutil.rmtree('/tmp/temp')
def test_amgr_run():
amgr = Amgr(hostname=host, port=port)
with pytest.raises(MissingError):
amgr.run()
p1 = Pipeline()
p2 = Pipeline()
p3 = Pipeline()
with pytest.raises(MissingError):
amgr.workflow = [p1, p2, p3]
def test_issue_255():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = ['sleep']
t1.arguments = ['10']
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port, autoterminate=False)
appman.resource_desc = res_dict
p1 = create_pipeline()
appman.workflow = [p1]
appman.run()
# p1 = create_pipeline()
# appman.workflow = [p1]
# appman.run()
# appman.resource_terminate()
tmgr = appman._task_manager
tmgr.terminate_manager()
# tmgr.terminate_heartbeat()
tmgr.start_manager()
def test_amgr_run():
amgr = Amgr()
with pytest.raises(MissingError):
amgr.run()
p1 = Pipeline()
p2 = Pipeline()
p3 = Pipeline()
amgr._workflow = [p1, p2, p3]
with pytest.raises(MissingError):
amgr.run()
def test_amgr_run():
amgr = Amgr()
with pytest.raises(MissingError):
amgr.run()
p1 = Pipeline()
p2 = Pipeline()
p3 = Pipeline()
amgr._workflow = [p1, p2, p3]
with pytest.raises(MissingError):
amgr.run()
def test_issue_255():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = 'sleep'
t1.arguments = ['10']
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
appman = AppManager(hostname=hostname,
port=port,
username=username,
password=password,
autoterminate=False)
appman.resource_desc = res_dict
p1 = create_pipeline()
appman.workflow = [p1]
appman.run()
# p1 = create_pipeline()
# appman.workflow = [p1]
# appman.run()
# appman.resource_terminate()
tmgr = appman._task_manager
tmgr.terminate_manager()
# tmgr.terminate_heartbeat()
tmgr.start_manager()
def test_integration_local():
"""
**Purpose**: Run an EnTK application on localhost
"""
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/echo']
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
def test_shared_data():
for f in glob('%s/file*.txt' %cur_dir):
os.remove(f)
os.system('echo "Hello" > %s/file1.txt' %cur_dir)
os.system('echo "World" > %s/file2.txt' %cur_dir)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 1,
'cpus': 1
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Assign resource manager to the Application Manager
appman.resource_desc = res_dict
appman.shared_data = ['%s/file1.txt' %cur_dir, '%s/file2.txt' %cur_dir]
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
with open('%s/output.txt' %cur_dir, 'r') as fp:
assert [d.strip() for d in fp.readlines()] == ['Hello', 'World']
os.remove('%s/file1.txt' %cur_dir)
os.remove('%s/file2.txt' %cur_dir)
os.remove('%s/output.txt' %cur_dir)
def test_integration_local():
"""
**Purpose**: Run an EnTK application on localhost
"""
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = '/bin/echo'
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
def test_amgr_run_mock():
p = Pipeline()
s = Stage()
t = Task()
t.name = 'simulation'
t.executable = '/bin/date'
s.tasks = t
p.add_stages(s)
res_dict = {'resource': 'local.localhost',
'walltime': 5,
'cpus' : 1,
'project' : ''}
appman = Amgr(hostname=host, port=port, rts="mock")
appman.resource_desc = res_dict
appman.workflow = [p]
appman.run()
def test_diff_rmq():
def create_pipeline():
p = Pipeline()
s = Stage()
t1 = Task()
t1.name = 'simulation'
t1.executable = ['/bin/echo']
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
s.add_tasks(t1)
p.add_stages(s)
return p
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
p1 = create_pipeline()
print p1.uid, p1.stages[0].uid
appman.workflow = [p1]
appman.run()
def test_suspend_pipeline():
# Create a dictionary describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost_anaconda',
'walltime': 15,
'cpus': 2,
}
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
def test_amgr_run_mock():
p = Pipeline()
s = Stage()
t = Task()
t.name = 'simulation'
t.executable = ['/bin/date']
s.tasks = t
p.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 5,
'cpus': 1,
'project': ''
}
appman = Amgr(hostname=hostname, port=port, rts="mock")
appman.resource_desc = res_dict
appman.workflow = [p]
appman.run()
# Create a dictionary to describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.summit',
'queue': 'batch',
'schema': 'local',
'walltime': 15,
'cpus': 48,
'gpus': 4
}
# Create Application Manager
appman = AppManager()
appman.resource_desc = res_dict
p1 = generate_MD_pipeline()
p2 = generate_ML_pipeline()
pipelines = []
pipelines.append(p1)
pipelines.append(p2)
# Assign the workflow as a list of Pipelines to the Application Manager. In
# this way, all the pipelines in the list will execute concurrently.
appman.workflow = pipelines
# Run the Application Manager
appman.run()
def main(cmt_filename):
'''This tiny function runs shit
Args:
cmt_filename: str containing the path to the cmt solution that is
supposed to be inverted for
Usage:
From the commandline:
python pipeline <path/to/cmtsolution>
'''
# Path to pipeline file
pipelinepath = os.path.abspath(__file__)
pipelinedir = os.path.dirname(pipelinepath)
# Define parameter directory
param_path = os.path.join(os.path.dirname(pipelinedir), "params")
databaseparam_path = os.path.join(param_path,
"Database/DatabaseParameters.yml")
DB_params = read_yaml_file(databaseparam_path)
print(DB_params)
# Earthquake specific database parameters
# Dir and eq_id
eq_dir, eq_id = get_eq_entry_path(DB_params["databasedir"], cmt_filename)
# Earthquake file in the database
cmt_file_db = os.path.join(eq_dir, "eq_" + eq_id + ".cmt")
# Create a Pipeline object
p = Pipeline()
# ---- DATABASE ENTRY TASK ---------------------------------------------- #
# Path to function
create_database_func = os.path.join(pipelinedir,
"01_Create_Database_Entry.py")
# Create a Stage object
database_entry = Stage()
t1 = Task()
t1.name = 'database-entry'
t1.pre_exec = [ # Conda activate
DB_params["conda-activate"]
]
t1.executable = [DB_params['bin-python']] # Assign executable to the task
t1.arguments = [create_database_func, os.path.abspath(cmt_filename)]
# In the future maybe to database dir as a total log?
t1.stdout = os.path.join(pipelinedir,
"database-entry." + eq_id + ".stdout")
t1.stderr = os.path.join(pipelinedir,
"database-entry." + eq_id + ".stderr")
# Add Task to the Stage
database_entry.add_tasks(t1)
# Add Stage to the Pipeline
p.add_stages(database_entry)
# # ---- REQUEST DATA ----------------------------------------------------- #
#
# # Path to function
# request_data_func = os.path.join(pipelinedir, "02_Request_Data.py")
#
# # Create a Stage object
# datarequest = Stage()
#
# datarequest_t = Task()
# datarequest_t.name = 'data-request'
# datarequest_t.pre_exec = [ # Conda activate
# DB_params["conda-activate"]]
# datarequest_t.executable = [DB_params['bin-python']] # Assign executable
# # to the task
# datarequest_t.arguments = [request_data_func, cmt_file_db]
#
# # In the future maybe to database dir as a total log?
# datarequest_t.stdout = os.path.join(pipelinedir,
# "datarequest." + eq_id + ".stdout")
# datarequest_t.stderr = os.path.join(pipelinedir,
# "datarequest." + eq_id + ".stderr")
#
# # Add Task to the Stage
# datarequest.add_tasks(datarequest_t)
#
# # Add Stage to the Pipeline
# p.add_stages(datarequest)
# ---- Write Sources ---------------------------------------------------- #
# Path to function
write_source_func = os.path.join(pipelinedir, "03_Write_Sources.py")
# Create a Stage object
w_sources = Stage()
w_sources.name = 'Write-Sources'
# Create Task for stage
w_sources_t = Task()
w_sources_t.name = 'Write-Sources'
w_sources_t.pre_exec = [ # Conda activate
DB_params["conda-activate"]
]
w_sources_t.executable = [DB_params['bin-python']] # Assign executable
# to the task
w_sources_t.arguments = [write_source_func, cmt_file_db]
# In the future maybe to database dir as a total log?
w_sources_t.stdout = os.path.join(pipelinedir,
"write_sources." + eq_id + ".stdout")
w_sources_t.stderr = os.path.join(pipelinedir,
"write_sources." + eq_id + ".stderr")
# Add Task to the Stage
w_sources.add_tasks(w_sources_t)
# Add Stage to the Pipeline
p.add_stages(w_sources)
# ---- Run Specfem ----------------------------------------------------- #
specfemspec_path = os.path.join(param_path,
"SpecfemParams/SpecfemParams.yml")
comp_and_modules_path = os.path.join(
param_path, "SpecfemParams/"
"CompilersAndModules.yml")
# Load Parameters
specfemspecs = read_yaml_file(specfemspec_path)
cm_dict = read_yaml_file(comp_and_modules_path)
attr = [
"CMT", "CMT_rr", "CMT_tt", "CMT_pp", "CMT_rt", "CMT_rp", "CMT_tp",
"CMT_depth", "CMT_lat", "CMT_lon"
]
simdir = os.path.join(eq_dir, "CMT_SIMs")
# Create a Stage object
runSF3d = Stage()
runSF3d.name = 'Simulation'
for at in attr[0]:
sf_t = Task()
sf_t.name = 'run-' + at
# Module Loading
sf_t.pre_exec = [ # Get rid of existing modules
'module purge'
]
for module in cm_dict["modulelist"]:
sf_t.pre_exec.append("module load %s" % module)
sf_t.pre_exec.append("module load %s" % cm_dict["gpu_module"])
# Change directory to specfem directories
sf_t.pre_exec.append( # Change directory
"cd %s" % os.path.join(simdir, at))
sf_t.executable = ['./bin/xspecfem3D'] # Assign executable
# In the future maybe to database dir as a total log?
sf_t.stdout = os.path.join(pipelinedir,
"run_specfem." + eq_id + ".stdout")
sf_t.stderr = os.path.join(pipelinedir,
"run_specfem." + eq_id + ".stderr")
sf_t.gpu_reqs = {
'processes': 6,
'process_type': 'MPI',
'threads_per_process': 1,
'thread_type': 'OpenMP'
}
# Add Task to the Stage
runSF3d.add_tasks(sf_t)
# Add Simulation stage to the Pipeline
p.add_stages(runSF3d)
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Create a dictionary describe four mandatory keys:
# resource, walltime, and cpus
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'princeton.tiger_gpu',
'project': 'geo',
'queue': 'gpu',
'schema': 'local',
'walltime': 300,
'cpus': 2,
'gpus': 6
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
appman.workflow = set([p])
# Run the Application Manager
appman.run()
}
# Add Stage to the Pipeline
p.add_stages(s1)
return p
if __name__ == '__main__':
# Create a dictionary describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'local.localhost',
'walltime': 15,
'cpus': 2,
}
# Create Application Manager
appman = AppManager()
appman.resource_desc = res_dict
p = generate_pipeline()
# Assign the workflow as a set of Pipelines to the Application Manager
appman.workflow = [p]
# Run the Application Manager
appman.run()
t = Task()
t.name = 't%s' % (cnt + 1)
t.pre_exec = ['export PATH=/home/karahbit/stress-ng-0.10.16:$PATH']
t.executable = ['stress-ng']
t.arguments = ['-c', '1', '-t', '100']
t.cpu_reqs = {
'processes': 1,
'thread_type': None,
'threads_per_process': 1,
'process_type': None
}
s.add_tasks(t)
p.add_stages(s)
pipelines.add(p)
# Resource and AppManager
amgr = AppManager(hostname=hostname, port=port)
amgr.workflow = pipelines
amgr.shared_data = []
amgr.resource_desc = {
'resource': 'local.localhost',
'walltime': 10,
'cpus': 8
}
amgr.run()
print("--- %s seconds ---" % (time.time() - start_time))
def test_rp_da_scheduler_bw():
"""
**Purpose**: Run an EnTK application on localhost
"""
p1 = Pipeline()
p1.name = 'p1'
n = 10
s1 = Stage()
s1.name = 's1'
for x in range(n):
t = Task()
t.name = 't%s'%x
t.executable = ['/bin/hostname']
t.arguments = ['>','hostname.txt']
t.cpu_reqs['processes'] = 1
t.cpu_reqs['threads_per_process'] = 16
t.cpu_reqs['thread_type'] = ''
t.cpu_reqs['process_type'] = ''
t.lfs_per_process = 10
t.download_output_data = ['hostname.txt > s1_t%s_hostname.txt'%(x)]
s1.add_tasks(t)
p1.add_stages(s1)
s2 = Stage()
s2.name = 's2'
for x in range(n):
t = Task()
t.executable = ['/bin/hostname']
t.arguments = ['>','hostname.txt']
t.cpu_reqs['processes'] = 1
t.cpu_reqs['threads_per_process'] = 16
t.cpu_reqs['thread_type'] = ''
t.cpu_reqs['process_type'] = ''
t.download_output_data = ['hostname.txt > s2_t%s_hostname.txt'%(x)]
t.tag = 't%s'%x
s2.add_tasks(t)
p1.add_stages(s2)
res_dict = {
'resource' : 'ncsa.bw_aprun',
'walltime' : 10,
'cpus' : 128,
'project' : 'gk4',
'queue' : 'high'
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
for i in range(n):
assert open('s1_t%s_hostname.txt'%i,'r').readline().strip() == open('s2_t%s_hostname.txt'%i,'r').readline().strip()
txts = glob('%s/*.txt' % os.getcwd())
for f in txts:
os.remove(f)
def generate_pipeline(name, stages): #Generate Pipeline for Stream Extraction/Morphological Thinning
# Create a Pipeline object
p = Pipeline()
p.name = 'p1'
for s_cnt in range(stages):
# Create a Stage object, Stream Extraction
s1 = Stage()
s1.name = ‘Stage 1’
# Create a Stage object, Morphological Thinning
s2 = Stage()
s2.name = ‘Stage 2’
# Create Task 1, Stream Extraction
t1 = Task()
t1.name = 'Task 1'
t1.executable = ['sbatch'] # Assign executable to the task
t1.arguments = ['$SCRATCH/ashk96/StreamExtraction.bat'] # Assign arguments for the StreamExtraction
# Add Task 1 to Stage 1
s1.add_tasks(t1)
# Add Stage 1 to the Pipeline
p.add_stages(s1)
# Create Task 2, Morphological Thinning
t2 = Task()
t2.name = 'Task 2'
t2.executable = ['sbatch'] # Assign executable to the task
t2.arguments = ['$SCRATCH/ashk96/Thin_Multi.bat'] # Assign arguments for the task executable
# Add Task 2 to Stage 2
s2.add_tasks(t2)
# Add Stage 2 to the Pipeline
p.add_stages(s2)
return p
if __name__ == '__main__':
p1 = generate_pipeline(name='Stream Extraction', stages=2)
res_dict = {
'resource': 'xsede.bridges',
'walltime': 02:00:00,
'cores': 2,
'project': 'TG-MCB090174',
'queue': '',
'schema': 'ssh'
}
# Create Resource Manager object with the above resource description
rman = ResourceManager(res_dict)
# Create Application Manager
appman = AppManager()
# Assign resource manager to the Application Manager
appman.resource_manager = rman
# Execute pipeline
appman.assign_workflow(p1)
# Run the Application Manager
appman.run()
def run(self):
# Get experiment time to save outputs to specific folder
# To remove eventually and find a better solution
fldr_name = os.path.join(self.output, str(int(time())))
ps_file = os.path.join(fldr_name, os.path.basename(self.param_space))
select_name = "selection.csv"
select_file = os.path.join(fldr_name, select_name)
i = 0
# check if stop criteria has been met
# otherwise run model with updated set of parameters
value = i < self.n_iterations if self.n_iterations is not None
cond = "convergence" if self.convergence is not None else "iterations"
while self.evaluate(cond, value):
# Setting up the pipeline
# Create a Pipeline object
p = Pipeline()
# Create a Stage object
s1 = Stage()
# generate tasks
tasks = self.generate(fldr_name)
# add tasks to stage
s1.add_tasks(tasks)
# create selection stage
s2 = Stage()
# create selection task(s)
selec_tasks = self.selection(ps_file, select_file)
s2.add_tasks(selec_tasks)
# Add Stage to the Pipeline
p.add_stages([s1, s2])
# Create Application Manager
appman = AppManager(
hostname=self.hostname,
port=self.port,
username=self.username,
password=self.password,
)
# Assign the workflow as a set or list of Pipelines to the Application Manager
appman.workflow = set([p])
# Assign resource request description to the Application Manager
appman.resource_desc = self.resource_dict
# Run the Application Manager
appman.run()
# TODO find a better solution
self.select_file = select_file
i += 1
value = i < self.n_iterations if self.n_iterations is not None
appman = AppManager(autoterminate=False,
port=33215) # Create Application Manager
appman.resource_desc = res_dict # Assign resource manager to the Application Manager
Exchange = synchronousExchange.InitCycle(Replicas, Replica_Cores,
MD_Executable, ExchangeMethod,
timesteps)
appman.workflow = set([
Exchange
]) # Assign the workflow as a set of Pipelines to the Application Manager
prof.prof('Run_Cycle_0', uid=uid1)
appman.run() # Run the Application Manager
prof.prof('End_Cycle_0', uid=uid1)
for Cycle in range(Cycles):
prof.prof('Create_Workflow_{0}'.format(Cycle + 1), uid=uid1)
Exchange_gen = synchronousExchange.GeneralCycle(
Replicas, Replica_Cores, Cycle, MD_Executable, ExchangeMethod)
appman.workflow = set(
[Exchange_gen]
) # Assign the workflow as a set of Pipelines to the Application Manager
prof.prof('Run_Cycle_{0}'.format(Cycle + 1), uid=uid1)
class Runner(object):
def __init__(self):
self._cores = 0
self._protocols = list()
self._hostname = None
self._port = None
self.ids = None
self.app_manager = None
self.total_replicas = 0
# Profiler for Runner
self._uid = ru.generate_id('radical.yank.workflow_runner')
self._logger = ru.get_logger('radical.yank.workflow_runner')
self._prof = ru.Profiler(name=self._uid)
self._prof.prof('create workflow_runner obj', uid=self._uid)
self._root_directories = list()
self.ids = dict()
def add_protocol(self, protocol):
self._protocols.append(protocol)
@property
def cores(self):
return self._cores
@cores.setter
def cores(self, val):
if isinstance(val, int):
self._cores = val
else:
raise TypeError()
def rabbitmq_config(self, hostname='localhost', port=5672):
self._hostname = hostname
self._port = port
def run(self, strong_scaled=1, autoterminate=True, queue='high', walltime=1440):
pipelines = set()
input_data = list()
for protocol in self._protocols:
gen_pipeline = protocol.generate_pipeline()
pipelines.add(gen_pipeline)
input_data.extend(protocol.input_data)
self.ids[protocol.id()] = gen_pipeline
# protocol.id is the uuid, gen_pipeline.uid is the pipeline
self.total_replicas += protocol.replicas
self._cores = self._cores * self.total_replicas
print 'Running on', self._cores, 'cores.'
res_dict = {'resource': 'ncsa.bw_aprun',
'walltime': walltime,
'cores': int(self._cores*strong_scaled),
'project': 'bamm',
'queue': queue,
'access_schema': 'gsissh'}
# Create Resource Manager object with the above resource description
resource_manager = ResourceManager(res_dict)
resource_manager.shared_data = input_data
# Create Application Manager
self.app_manager = AppManager(hostname=self._hostname, port=self._port, autoterminate=autoterminate)
self.app_manager.resource_manager = resource_manager
self.app_manager.assign_workflow(pipelines)
self._prof.prof('execution_run')
print 'Running...'
self.app_manager.run() # this method is blocking until all pipelines show state = completed
def rerun(self, protocol=None, terminate=True, previous_pipeline=None):
if self.ids.get(previous_pipeline.id(), None) is not None:
pipelines = set()
gen_pipeline = protocol.generate_pipeline(previous_pipeline=self.ids[previous_pipeline.id()])
pipelines.add(gen_pipeline)
self.ids[protocol.id()] = gen_pipeline
self.app_manager.assign_workflow(pipelines)
self.app_manager.run()
if terminate:
self.app_manager.resource_terminate()
else:
print "ERROR: previous protocol instance is not found"
def test_rp_da_scheduler_bw():
"""
**Purpose**: Run an EnTK application on localhost
"""
p1 = Pipeline()
p1.name = 'p1'
n = 10
s1 = Stage()
s1.name = 's1'
for x in range(n):
t = Task()
t.name = 't%s' % x
t.executable = ['/bin/hostname']
t.arguments = ['>', 'hostname.txt']
t.cpu_reqs['processes'] = 1
t.cpu_reqs['threads_per_process'] = 16
t.cpu_reqs['thread_type'] = ''
t.cpu_reqs['process_type'] = ''
t.lfs_per_process = 10
t.download_output_data = ['hostname.txt > s1_t%s_hostname.txt' % (x)]
s1.add_tasks(t)
p1.add_stages(s1)
s2 = Stage()
s2.name = 's2'
for x in range(n):
t = Task()
t.executable = ['/bin/hostname']
t.arguments = ['>', 'hostname.txt']
t.cpu_reqs['processes'] = 1
t.cpu_reqs['threads_per_process'] = 16
t.cpu_reqs['thread_type'] = ''
t.cpu_reqs['process_type'] = ''
t.download_output_data = ['hostname.txt > s2_t%s_hostname.txt' % (x)]
t.tag = 't%s' % x
s2.add_tasks(t)
p1.add_stages(s2)
res_dict = {
'resource': 'ncsa.bw_aprun',
'walltime': 10,
'cpus': 128,
'project': 'gk4',
'queue': 'high'
}
os.environ['RADICAL_PILOT_DBURL'] = MLAB
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
for i in range(n):
assert open('s1_t%s_hostname.txt' % i,
'r').readline().strip() == open('s2_t%s_hostname.txt' % i,
'r').readline().strip()
txts = glob('%s/*.txt' % os.getcwd())
for f in txts:
os.remove(f)
def workflow(cmt_filename, param_path):
"""This function submits the complete workflow
:param cmt_filename: str containing the path to the cmt solution that is
supposed to be inverted for
Usage:
```bash
python 1pipeline <path/to/cmtsolution>
```
"""
# Get Database parameters
databaseparam_path = os.path.join(param_path,
"Database/DatabaseParameters.yml")
DB_params = read_yaml_file(databaseparam_path)
# Earthquake specific database parameters: Dir and Cid
Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_filename)
# Earthquake file in the database
cmt_file_db = os.path.join(Cdir, "C" + Cid + ".cmt")
# Create a counter for all tasks in one pipeline
task_counter = 0
# Create a Pipeline object
p = Pipeline()
if HEADNODE_AVAILABLE:
# ---- Create Database Entry --------------------------------------------- #
# Create Database entry stage:
database_entry_stage, task_counter = create_entry(
cmt_filename, param_path, task_counter)
# Add Stage to the Pipeline
p.add_stages(database_entry_stage)
# ---- REQUEST DATA ------------------------------------------------- #
# # Request data stage
# datarequest_stage, task_counter = data_request(cmt_file_db,
# param_path,
# task_counter)
#
# # Add Stage to the Pipeline
# p.add_stages(datarequest_stage)
else:
# Create the entry now before running the pipeline
task_counter = call_create_entry(cmt_filename, param_path,
task_counter)
# # # Download the data from the headnode before running the pipeline
# task_counter = call_download_data(cmt_file_db, param_path,
# task_counter)
# ---- Write Sources ---------------------------------------------------- #
# # Create Source modification stage
# w_sources_stage, task_counter = write_sources(cmt_file_db, param_path,
# task_counter)
#
# # Add Stage to the Pipeline
# p.add_stages(w_sources_stage)
# ---- Run Specfem ------------------------------------------------------ #
# # Create Specfem Stage
# runSF3D_stage, task_counter = run_specfem(cmt_file_db,
# param_path,
# task_counter)
#
# # Add Simulation stage to the Pipeline
# p.add_stages(runSF3D_stage)
#
# # ---- Clean Up Specfem ------------------------------------------------- #
#
# # Create clean_up stage
# clean_up_stage, task_counter = specfem_clean_up(cmt_file_db,
# param_path,
# task_counter)
#
# # Add Stage to the Pipeline
# p.add_stages(clean_up_stage)
# ---- Convert to ASDF -------------------------------------------------- #
# Create conversion stage
conversion_stage, task_counter = convert_traces(cmt_file_db, param_path,
task_counter)
# Add stage to pipeline
p.add_stages(conversion_stage)
# ---- Create Process Path files ---------------------------------------- #
# Create Process Stage Pipeline
process_path_stage, task_counter = create_process_path_files(
cmt_file_db, param_path, task_counter)
p.add_stages(process_path_stage)
# ---- Process Traces --------------------------------------------------- #
# Create processing stage
processing_stages, task_counter = create_processing_stage(
cmt_file_db, param_path, task_counter)
for stage in processing_stages:
p.add_stages(stage)
# ---- Window Traces ---------------------------------------------------- #
# Create processing stage
windowing_stages, task_counter = create_windowing_stage(
cmt_file_db, param_path, task_counter)
for windowing_stage in windowing_stages:
p.add_stages(windowing_stage)
# ---- Create Inversion Dictionaries------------------------------------- #
# Create processing stage
inv_dict_stage, task_counter = create_inversion_dict_stage(
cmt_file_db, param_path, task_counter)
p.add_stages(inv_dict_stage)
# ---- Inversion -------------------------------------------------------- #
# Create processing stage
inversion_stage = create_inversion_stage(cmt_file_db, param_path,
task_counter)
p.add_stages(inversion_stage)
# ============== RUNNING THE PIPELINE ==================================== #
# Create Application Manager
appman = AppManager(hostname=hostname, port=port)
# Compute the necessary walltime from walltime/per simulation
# Load parameters
specfem_specs = read_yaml_file(
os.path.join(param_path, "SpecfemParams/SpecfemParams.yml"))
# Get twalltime from walltime specification in the parameter file.
walltime_per_simulation = specfem_specs["walltime"].split(":")
hours_in_min = float(walltime_per_simulation[0]) * 60
min_in_min = float(walltime_per_simulation[1])
sec_in_min = float(walltime_per_simulation[2]) / 60
cpus = int(specfem_specs["cpus"])
tasks = int(specfem_specs["tasks"])
# Add times to get full simulation time. The 45 min are accounting for
# everything that is not simulation time
total_min = int(1/math.ceil(float(cpus)/40) \
* 10 * int(round(hours_in_min + min_in_min + sec_in_min)) + 45)
# Create a dictionary describe four mandatory keys:
# resource, walltime, cpus etc.
# resource is "local.localhost" to execute locally
# Define which resources to get depending on how specfem is run!
if specfem_specs["GPU_MODE"] is False:
# res_dict_cpu = {
# "resource": "princeton.tiger_cpu",
# "project": "geo",
# "queue": "cpu",
# "schema": "local",
# "walltime": total_min,
# "cpus": int(specfem_specs["cpus"]),
# }
res_dict_cpu = {
"resource": "princeton.tiger_cpu",
"project": "geo",
"queue": "cpu",
"schema": "local",
"walltime": 45,
"cpus": 20
}
else:
res_dict_gpu = {
"resource": "princeton.tiger_gpu",
"project": "geo",
"queue": "gpu",
"schema": "local",
"walltime": 300,
"cpus": int(specfem_specs["cpus"]),
"gpus": int(specfem_specs["gpus"])
}
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict_cpu
# Assign the workflow as a set or list of Pipelines to the Application Manager
# Note: The list order is not guaranteed to be preserved
appman.workflow = set([p])
# Run the Application Manager
appman.run()
def main():
# Get/Set radical configuration attributes
if os.environ.get("RADICAL_ENTK_VERBOSE") == None:
os.environ["RADICAL_ENTK_REPORT"] = "True"
hostname = os.environ.get("RMQ_HOSTNAME", "localhost")
port = os.environ.get("RMQ_PORT", 5672)
username = os.environ.get("RMQ_USERNAME")
password = os.environ.get("RMQ_PASSWORD")
# Input argument parsing
parser = argparse.ArgumentParser("Launch icemodel ensemble")
parser.add_argument("executable",
type=str,
help="location of the model executable")
parser.add_argument("clifvmax_step",
type=int,
help="CLIFVMAX paramater range step value")
parser.add_argument("crevliq_step",
type=int,
help="CREVLIQ paramater range step value")
parser.add_argument(
"param_space",
type=int,
help=
"The size of the 2d parameter space (e.g. <param_space>X<param_space))",
)
parser.add_argument(
"resource_reqs",
type=argparse.FileType("r"),
help="Resource requirements json",
)
args = parser.parse_args()
# Create iterable objects for the range of values possible for CLIFFMAX and CREVLIQ
clifvmax_range = range(0, args.clifvmax_step * args.param_space,
args.clifvmax_step)
crevliq_range = range(0, args.crevliq_step * args.param_space,
args.crevliq_step)
# Setting up the pipeline
# Create a Pipeline object
p = Pipeline()
# Create a Stage object
s1 = Stage()
# Add tasks to stage
add_tasks(crevliq_range, clifvmax_range, s1, args.executable)
# Add Stage to the Pipeline
p.add_stages(s1)
# Create Application Manager
appman = AppManager(hostname=hostname,
port=port,
username=username,
password=password)
# Assign the workflow as a set or list of Pipelines to the Application Manager
appman.workflow = set([p])
# Load resource requirements file for running on Bridges
res_dict = json.load(args.resource_reqs)
# Assign resource request description to the Application Manager
appman.resource_desc = res_dict
# Run the Application Manager
appman.run()
def test_stage_post_exec():
p1 = Pipeline()
p1.name = 'p1'
s = Stage()
s.name = 's1'
def create_single_task():
t1 = Task()
t1.name = 'simulation'
t1.executable = '/bin/echo'
t1.arguments = ['hello']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
NUM_TASKS = 2
MAX_STAGES = 5
CUR_STAGE = 1
def condition():
nonlocal CUR_STAGE, MAX_STAGES
if CUR_STAGE < MAX_STAGES:
CUR_STAGE += 1
on_true()
on_false()
def on_true():
nonlocal NUM_TASKS, CUR_STAGE, p1
NUM_TASKS *= 2
s = Stage()
s.name = 's%s' % CUR_STAGE
for _ in range(NUM_TASKS):
s.add_tasks(create_single_task())
s.post_exec = condition
p1.add_stages(s)
def on_false():
pass
for _ in range(NUM_TASKS):
s.add_tasks(create_single_task())
s.post_exec = condition
p1.add_stages(s)
res_dict = {
'resource': 'local.localhost',
'walltime': 30,
'cpus': 1,
}
appman = AppManager(rts='radical.pilot',
hostname=hostname,
port=port,
username=username,
password=password)
appman.resource_desc = res_dict
appman.workflow = [p1]
appman.run()
class DeepDriveMD:
"""
Implements an interface for the DeepDriveMD computational
motif presented in: https://arxiv.org/abs/1909.07817
"""
def __init__(self,
md_sims,
preprocs,
ml_algs,
outlier_algs,
resources,
max_iter=1,
pipeline_name='MD_ML',
md_stage_name='MD',
pre_stage_name='Preprocess',
ml_stage_name='ML',
outlier_stage_name='Outlier'):
"""
Parameters
----------
md_sims : list
list of DeepDriveMD.taskmanager.TaskManager objects
which manage simulations
preprocs : list
list of DeepDriveMD.taskmanager.TaskManager objects
which manage data preprocessing
ml_algs : list
list of DeepDriveMD.taskmanager.TaskManager objects
which manage representation learning
outlier_algs : list
list of DeepDriveMD.taskmanager.TaskManager objects
which manage outlier detection
resources : dict
Configuration settings for running on Summit
max_iter : int
Max number of iterations through the pipeline
pipeline_name : str
Name of computational pipeline
md_stage_name : str
Name of MD stage
pre_stage_name : str
Name of preprocessing stage
ml_stage_name : str
Name of ML stage
outlier_stage_name : str
Name of outlier detection stage
"""
# Checks environment variables are set
self._validate_environment()
# Number of iterations through the pipeline
self.current_iter = 0
self.max_iter = max_iter
# Neatly stores stage name and taskmanagers
StageData = namedtuple('StageData', ['name', 'taskmanagers'])
# Dictionary storing name and taskmanagers for each stage
self.stages = {
'md': StageData(md_stage_name, md_sims),
'preprocess': StageData(pre_stage_name, preprocs),
'ml': StageData(ml_stage_name, ml_algs),
'outlier': StageData(outlier_stage_name, outlier_algs)
}
# Initialize pipeline
self.__pipeline = Pipeline()
self.__pipeline.name = pipeline_name
# Sets pipeline stages
self._pipeline()
# Create Application Manager
self.appman = AppManager(hostname=os.environ.get('RMQ_HOSTNAME'),
port=int(os.environ.get('RMQ_PORT')))
# Assign hardware resources to application
self.appman.resource_desc = resources
# Assign the workflow as a set of Pipelines to the Application Manager. In
# this way, all the pipelines in the set will execute concurrently.
self.appman.workflow = {self.__pipeline}
def run(self):
"""
Effects
-------
Runs the Application Manager.
"""
self.appman.run()
def _validate_environment(self):
if not os.environ.get('RADICAL_ENTK_VERBOSE'):
os.environ['RADICAL_ENTK_REPORT'] = 'True'
# All envars must be set prior
envars = [
'RMQ_HOSTNAME', 'RMQ_PORT', 'RADICAL_PILOT_DBURL',
'RADICAL_PILOT_PROFILE', 'RADICAL_ENTK_PROFILE'
]
for envar in envars:
if not os.environ.get(envar):
raise Exception(f'{envar} environment variable not set')
def _generate_stage(self, stage_type):
"""
Parameters
----------
stage_type : str
key into self.stages dictionary to retrieve stage name and taskmanagers.
"""
stage = Stage()
stage.name = self.stages[stage_type].name
for taskman in self.stages[stage_type].taskmanagers:
stage.add_tasks(set(taskman.tasks(self.current_iter)))
return stage
def _condition(self):
if self.current_iter < self.max_iter:
self._pipeline()
print('Done')
def _pipeline(self):
"""
Effects
-------
Adds stages to pipeline.
"""
if self.current_iter:
print(
f'Finished pipeline iteration {self.current_iter} of {self.max_iter}'
)
# Add the first three stages to the pipeline
for stage_type in ['md', 'preprocess', 'ml']:
self.__pipeline.add_stages(self._generate_stage(stage_type))
# Generate last stage seperate to add post execution step
last_stage = self._generate_stage('outlier')
# Set post execution for last stage
last_stage.post_exec = self._condition
self.current_iter += 1
self.__pipeline.add_stages(last_stage)
