print p.stages
print p.stages[1].tasks
# Create a dictionary describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'xsede.supermic',
'walltime': 240,
'cores': 96,
'access_schema': 'gsissh',
'queue': 'workq',
'project': 'TG-MCB090174',
}
# Create Resource Manager object with the above resource description
rman = ResourceManager(res_dict)
# Create Application Manager
appman = AppManager()
# Assign resource manager to the Application Manager
appman.resource_manager = rman
# Assign the workflow as a set of Pipelines to the Application Manager
appman.assign_workflow(set([p]))
# Run the Application Manager
## appman.run()
pipes_set.add(get_pipeline(num_tasks))
# Create a dictionary describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': res_name,
'walltime': res_coll[res_name]['walltime'],
'cores': num_tasks,
'project': res_coll[res_name]['project'],
'queue': res_coll[res_name]['queue'],
'access_schema': res_coll[res_name]['schema']
}
# Create Resource Manager object with the above resource description
rman = ResourceManager(res_dict)
# Create Application Manager
appman = AppManager()
# Assign resource manager to the Application Manager
appman.resource_manager = rman
# Assign the workflow as a set of Pipelines to the Application Manager
appman.assign_workflow(pipes_set)
# Run the Application Manager
appman.run()
def generate_pipeline(name, stages): #Generate Pipeline for Stream Extraction/Morphological Thinning
# Create a Pipeline object
p = Pipeline()
p.name = 'p1'
for s_cnt in range(stages):
# Create a Stage object, Stream Extraction
s1 = Stage()
s1.name = ‘Stage 1’
# Create a Stage object, Morphological Thinning
s2 = Stage()
s2.name = ‘Stage 2’
# Create Task 1, Stream Extraction
t1 = Task()
t1.name = 'Task 1'
t1.executable = ['sbatch'] # Assign executable to the task
t1.arguments = ['$SCRATCH/ashk96/StreamExtraction.bat'] # Assign arguments for the StreamExtraction
# Add Task 1 to Stage 1
s1.add_tasks(t1)
# Add Stage 1 to the Pipeline
p.add_stages(s1)
# Create Task 2, Morphological Thinning
t2 = Task()
t2.name = 'Task 2'
t2.executable = ['sbatch'] # Assign executable to the task
t2.arguments = ['$SCRATCH/ashk96/Thin_Multi.bat'] # Assign arguments for the task executable
# Add Task 2 to Stage 2
s2.add_tasks(t2)
# Add Stage 2 to the Pipeline
p.add_stages(s2)
return p
if __name__ == '__main__':
p1 = generate_pipeline(name='Stream Extraction', stages=2)
res_dict = {
'resource': 'xsede.bridges',
'walltime': 02:00:00,
'cores': 2,
'project': 'TG-MCB090174',
'queue': '',
'schema': 'ssh'
}
# Create Resource Manager object with the above resource description
rman = ResourceManager(res_dict)
# Create Application Manager
appman = AppManager()
# Assign resource manager to the Application Manager
appman.resource_manager = rman
# Execute pipeline
appman.assign_workflow(p1)
# Run the Application Manager
appman.run()