def test_assign_workload_exception():
appman = AppManager()
data_type = [1, 'a', True, [1], set([1])]
for data in data_type:
with pytest.raises(TypeError):
appman.assign_workflow(data)
print p.stages
print p.stages[1].tasks
# Create a dictionary describe four mandatory keys:
# resource, walltime, cores and project
# resource is 'local.localhost' to execute locally
res_dict = {
'resource': 'xsede.supermic',
'walltime': 240,
'cores': 96,
'access_schema': 'gsissh',
'queue': 'workq',
'project': 'TG-MCB090174',
}
# Create Resource Manager object with the above resource description
rman = ResourceManager(res_dict)
# Create Application Manager
appman = AppManager()
# Assign resource manager to the Application Manager
appman.resource_manager = rman
# Assign the workflow as a set of Pipelines to the Application Manager
appman.assign_workflow(set([p]))
# Run the Application Manager
## appman.run()
def generate_pipeline(name, stages): #Generate Pipeline for Stream Extraction/Morphological Thinning
# Create a Pipeline object
p = Pipeline()
p.name = 'p1'
for s_cnt in range(stages):
# Create a Stage object, Stream Extraction
s1 = Stage()
s1.name = ‘Stage 1’
# Create a Stage object, Morphological Thinning
s2 = Stage()
s2.name = ‘Stage 2’
# Create Task 1, Stream Extraction
t1 = Task()
t1.name = 'Task 1'
t1.executable = ['sbatch'] # Assign executable to the task
t1.arguments = ['$SCRATCH/ashk96/StreamExtraction.bat'] # Assign arguments for the StreamExtraction
# Add Task 1 to Stage 1
s1.add_tasks(t1)
# Add Stage 1 to the Pipeline
p.add_stages(s1)
# Create Task 2, Morphological Thinning
t2 = Task()
t2.name = 'Task 2'
t2.executable = ['sbatch'] # Assign executable to the task
t2.arguments = ['$SCRATCH/ashk96/Thin_Multi.bat'] # Assign arguments for the task executable
# Add Task 2 to Stage 2
s2.add_tasks(t2)
# Add Stage 2 to the Pipeline
p.add_stages(s2)
return p
if __name__ == '__main__':
p1 = generate_pipeline(name='Stream Extraction', stages=2)
res_dict = {
'resource': 'xsede.bridges',
'walltime': 02:00:00,
'cores': 2,
'project': 'TG-MCB090174',
'queue': '',
'schema': 'ssh'
}
# Create Resource Manager object with the above resource description
rman = ResourceManager(res_dict)
# Create Application Manager
appman = AppManager()
# Assign resource manager to the Application Manager
appman.resource_manager = rman
# Execute pipeline
appman.assign_workflow(p1)
# Run the Application Manager
appman.run()
class Runner(object):
def __init__(self):
self._cores = 0
self._protocols = list()
self._hostname = None
self._port = None
self.ids = None
self.app_manager = None
self.total_replicas = 0
# Profiler for Runner
self._uid = ru.generate_id('radical.yank.workflow_runner')
self._logger = ru.get_logger('radical.yank.workflow_runner')
self._prof = ru.Profiler(name=self._uid)
self._prof.prof('create workflow_runner obj', uid=self._uid)
self._root_directories = list()
self.ids = dict()
def add_protocol(self, protocol):
self._protocols.append(protocol)
@property
def cores(self):
return self._cores
@cores.setter
def cores(self, val):
if isinstance(val, int):
self._cores = val
else:
raise TypeError()
def rabbitmq_config(self, hostname='localhost', port=5672):
self._hostname = hostname
self._port = port
def run(self, strong_scaled=1, autoterminate=True, queue='high', walltime=1440):
pipelines = set()
input_data = list()
for protocol in self._protocols:
gen_pipeline = protocol.generate_pipeline()
pipelines.add(gen_pipeline)
input_data.extend(protocol.input_data)
self.ids[protocol.id()] = gen_pipeline
# protocol.id is the uuid, gen_pipeline.uid is the pipeline
self.total_replicas += protocol.replicas
self._cores = self._cores * self.total_replicas
print 'Running on', self._cores, 'cores.'
res_dict = {'resource': 'ncsa.bw_aprun',
'walltime': walltime,
'cores': int(self._cores*strong_scaled),
'project': 'bamm',
'queue': queue,
'access_schema': 'gsissh'}
# Create Resource Manager object with the above resource description
resource_manager = ResourceManager(res_dict)
resource_manager.shared_data = input_data
# Create Application Manager
self.app_manager = AppManager(hostname=self._hostname, port=self._port, autoterminate=autoterminate)
self.app_manager.resource_manager = resource_manager
self.app_manager.assign_workflow(pipelines)
self._prof.prof('execution_run')
print 'Running...'
self.app_manager.run() # this method is blocking until all pipelines show state = completed
def rerun(self, protocol=None, terminate=True, previous_pipeline=None):
if self.ids.get(previous_pipeline.id(), None) is not None:
pipelines = set()
gen_pipeline = protocol.generate_pipeline(previous_pipeline=self.ids[previous_pipeline.id()])
pipelines.add(gen_pipeline)
self.ids[protocol.id()] = gen_pipeline
self.app_manager.assign_workflow(pipelines)
self.app_manager.run()
if terminate:
self.app_manager.resource_terminate()
else:
print "ERROR: previous protocol instance is not found"
if __name__ == '__main__':
p1 = generate_pipeline(name='Pipeline 1', stages=2)
res_dict = {
'resource': 'xsede.bridges',
'username': '******' #username is not currently included in ensemble toolkit, so an error arises
'walltime': 10,
'cores': 1,
'project': '',
}
# Create Resource Manager object with the above resource description
rman = ResourceManager(res_dict)
# Create Application Manager
appman = AppManager()
# Assign resource manager to the Application Manager
appman.resource_manager = rman
# Execute pipeline
appman.assign_workflow(p1)
# Run the Application Manager
appman.run()
class Runner(object):
def __init__(self, resource='local', comm_server=None):
"""The workhorse of high throughput binding affinity calculations.
Manages arbitrary number of protocols on any resource (including supercomputers).
Parameters
----------
resource: str
The name of the resource where the protocols will be run. This is usually then name of the supercomputer
or 'local' if the job will be executed locally. (the default is to try to run locally).
comm_server: tuple(str, int)
The communication server used by the execution system. Specify a hostname and port number as a tuple. If
None, then the dedicated server might be used from the resource description if present.
"""
self.resource = yaml.load(resource_stream(__name__, 'resources.yaml'))[resource]
if comm_server is None:
comm_server = self.resource.get('dedicated_rabbitmq_server')
self._protocols = list()
self._app_manager = AppManager(*comm_server)
# Profiler for Runner
self._uid = ru.generate_id('radical.htbac.workflow_runner')
self._logger = ru.get_logger('radical.htbac.workflow_runner')
self._prof = ru.Profiler(name=self._uid)
self._prof.prof('create workflow_runner obj', uid=self._uid)
self._root_directories = list()
def add_protocol(self, protocol):
"""Add a new protocol to the list of protocols to be executed.
Parameters
----------
protocol: Simulatable
"""
protocol.configure_engine_for_resource(self.resource)
self._protocols.append(protocol)
def run(self, resource=None, walltime=None, strong_scaled=1, queue=None, access_schema=None, dry_run=False):
"""Run protocols.
Parameters
----------
resource: str
The specific resource and sub-resource you want to use.
walltime: int
Wall time in minutes.
strong_scaled: float
For testing strong scaling. Number of cores will be multiplied by this number before execution.
queue: str
Name of the queue. If there is a default for your resource that will be used.
access_schema: str
One of ssh, gsissh, local
dry_run: bool
Whether to execute the `.run` command or not.
"""
pipelines = set()
shared_data = set()
cores = 0
max_cu_count = self.resource.get('max_cu_count', 0)
for protocol in self._protocols:
gen_pipeline = protocol.generate_pipeline()
cu_count = len(gen_pipeline.stages[0].tasks)
if max_cu_count and cu_count > max_cu_count:
raise ValueError('Resource allows up to {} concurrent CUs. You have {}.'.format(max_cu_count, cu_count))
pipelines.add(gen_pipeline)
shared_data.update(protocol.shared_data)
cores += protocol.cores
cores *= strong_scaled
cores += self.resource.get('agent_cores', 0)
self.resource['resource_dictionary']['cores'] = cores
if resource:
self.resource['resource_dictionary']['resource'] = resource
if walltime:
self.resource['resource_dictionary']['walltime'] = walltime
if queue:
self.resource['resource_dictionary']['queue'] = queue
if access_schema:
self.resource['resource_dictionary']['access_schema'] = access_schema
logger.info('Using total number of cores: {}.'.format(cores))
logger.info('Resource dictionary:\n{}'.format(pprint.pformat(self.resource['resource_dictionary'])))
# Create Resource Manager object with the above resource description
resource_manager = ResourceManager(self.resource['resource_dictionary'])
resource_manager.shared_data = list(shared_data)
# Create Application Manager
self._app_manager.resource_manager = resource_manager
self._app_manager.assign_workflow(pipelines)
logger.info("\n".join("Stage {}: {}*{} cores.".
format(i, len(s.tasks), next(iter(s.tasks)).cores)
for i, s in enumerate(next(iter(pipelines)).stages)))
self._prof.prof('execution_run')
logger.info('Running workflow.')
if not dry_run:
self._app_manager.run() # this method is blocking until all pipelines show state = completed
def rerun(self, protocol=None, terminate=True, previous_pipeline=None):
if self.ids.get(previous_pipeline.id(), None) is not None:
pipelines = set()
gen_pipeline = protocol.generate_pipeline(previous_pipeline=self.ids[previous_pipeline.id()])
pipelines.add(gen_pipeline)
self.ids[protocol.id] = gen_pipeline
self.app_manager.assign_workflow(pipelines)
self.app_manager.run()
if terminate:
self.app_manager.resource_terminate()
else:
print "ERROR: previous protocol instance is not found"
if __name__ == '__main__':
p1 = Pipeline()
p1.name = 'p1'
p2 = Pipeline()
p2.name = 'p2'
stages = 3
for cnt in range(stages):
s = Stage()
s.name = 's_%s' % cnt
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p1.add_stages(s)
for cnt in range(stages - 1):
s = Stage()
s.name = 's-%s' % cnt
s.tasks = create_single_task()
s.add_tasks(create_single_task())
p2.add_stages(s)
appman = AppManager()
appman.assign_workflow(set([p1, p2]))
appman.run()
t1.name = 'dummy_task'
t1.executable = ['placeholder']
t1.arguments = ['a', 'b', 'c']
t1.copy_input_data = []
t1.copy_output_data = []
return t1
if __name__ == '__main__':
pipes = []
for i in range(4):
p = Pipeline()
s1 = Stage()
s1.name = 'prediction'
s1.tasks = create_single_task()
p.add_stages(s1)
s2 = Stage()
s2.name = 'quality-check'
s2.tasks = create_single_task()
p.add_stages(s2)
pipes.append(p)
appman = AppManager()
appman.assign_workflow(set(pipes))
appman.run()
