def manage_workload (workload, config) :
"""
Parse and execute a given workload (see `execute_workload()`)
"""
session = troy.Session (config)
planner = troy.Planner (session)
overlay_mgr = troy.OverlayManager (session)
workload_mgr = troy.WorkloadManager (session)
strategy = troy.AUTOMATIC
if 'plugin_strategy' in session.cfg :
strategy = session.cfg['plugin_strategy']
if strategy == troy.AUTOMATIC :
strategy = 'basic_late_binding'
# FIXME: we should also accept workload instances or json strings -- bext
# put that flexibility into parse_workload
parsed_workload = workload_mgr.parse_workload (workload)
return troy.execute_workload (parsed_workload, planner,
overlay_mgr, workload_mgr,
strategy)
task_descr.executable = "%(mdrun)s"
task_descr.cardinality = "%(bag_size)s"
task_descr.inputs = ["input/topol.tpr > topol.tpr"]
task_descr.outputs = ["output/%(session_id)s_state.cpt.%(cardinal)s < state.cpt",
"output/%(session_id)s_confout.gro.%(cardinal)s < confout.gro",
"output/%(session_id)s_ener.edr.%(cardinal)s < ener.edr",
"output/%(session_id)s_traj.trr.%(cardinal)s < traj.trr",
"output/%(session_id)s_md.log.%(cardinal)s < md.log"]
task_descr.working_directory = "%(home)s/troy_tutorial/troy_tutorial_02_%(cardinal)s/"
workload = troy.Workload (session, task_descr)
# ------------------------------------------------------------------------------
#
# create managers within session (and its configs)
#
planner = troy.Planner (session)
overlay_mgr = troy.OverlayManager (session)
workload_mgr = troy.WorkloadManager (session)
# ------------------------------------------------------------------------------
#
# execute workload
#
troy.execute_workload (workload, planner, overlay_mgr, workload_mgr,
strategy=strategy)
# ------------------------------------------------------------------------------
def main(args):
# AIMES workload variables.
working_directory = None
aimes_workloads = []
# TROY variables.
planner = None
workload_manager = None
overlay_manager = None
task_descriptions = []
# Check whether the requested application generator, pilot system and
# skeleton modes are implemented.
if not args.application_generator in ['skeleton', 'file']:
raise Exception("Application generator \'%s\' not supported." %
args.application_generator)
if args.skeleton_mode != 'Shell':
raise Exception("%s is not supported." % args.skeleton_mode)
if not args.pilot_system in ['bigjob', 'sinon']:
raise Exception("Pilot system \'%s\' is not supported." %
args.pilot_system)
# Generate the workload(s) to be executed by means of an overlay.
for counter in range(args.workload_count):
w = Workload(counter,
args.tag,
args.workload_pattern,
args.data_staging,
args.local_working_directory,
args.remote_working_directory,
args.workload_directory,
args.task_duration,
args.task_count,
args.task_input_file_size,
args.task_output_file_size)
w.create_tasks()
aimes_workloads.append(w)
# Translate the workload(s) into TROY internal workload description.
# NOTE: This is missing in TROY at the moment. We need a plugin for each
# workload.
for w in aimes_workloads:
for t in w.tasks:
cu_description = troy.TaskDescription()
if args.remote_working_directory:
cu_description.working_directory = args.remote_working_directory
print "OWMS DEBUG: task_description.working_directory: %s" % args.remote_working_directory
else:
cu_description.working_directory = args.local_working_directory
print "OWMS DEBUG: task_description.working_directory: %s" % args.local_working_directory
cu_description.tag = t.tag
cu_description.executable = '/bin/sh'
cu_description.arguments = [t.executable_name]
# Comment out until data staging will not be supported by TROY.
# Type None for .input_file and output_file are not yet managed.
# if args.data_staging:
# cu_description.inputs = [t.input_file, t.executable_name]
# cu_description.outputs = [t.output_file]
task_descriptions.append(cu_description)
# print cu_description.as_dict ()
# Create a session for TROY.
session = troy.Session(
{
'planner_concurrent': {
'concurrency': args.concurrency
},
'workload_dispatcher_bigjob_pilot': {
'coordination_url ': args.bigjob_coordination_endpoint
},
'overlay_provisioner_bigjob_pilot': {
'coordination_url': args.bigjob_coordination_endpoint,
'queue' : 'batch'
},
'workload_dispatcher_sinon': {
'coordination_url': args.sinon_coordination_endpoint
},
'overlay_provisioner_sinon': {
'coordination_url': args.sinon_coordination_endpoint,
'queue' : 'batch'
}
})
# Add resources to the session.
if args.pilot_system == 'bigjob':
session.user_cfg['overlay_scheduler_round_robin'] = {
# 'resources': 'pbs+ssh://india.futuregrid.org/,pbs+ssh://sierra.futuregrid.org/'
'resources': 'pbs+ssh://india.futuregrid.org/'
}
elif args.pilot_system == 'sinon':
session.user_cfg['overlay_scheduler_round_robin'] = {
# 'resources': 'futuregrid.INDIA,futuregrid.SIERRA'
'resources': 'futuregrid.INDIA'
}
# Manage credentials.
# TODO: set it to args.protocol (default ssh).
c1 = troy.Context ('ssh')
c1.user_id = args.ssh_user_name
session.add_context (c1)
# Instantiate TROY planner and managers.
planner = troy.Planner(planner = args.troy_planner, session = session)
# if args.data_staging:
# workload_manager = troy.WorkloadManager(dispatcher = args.troy_workload_dispatcher,
# session = session)
# else:
# workload_manager = troy.WorkloadManager(dispatcher = args.troy_workload_dispatcher,
# session = session)
workload_manager = troy.WorkloadManager(dispatcher = args.troy_workload_dispatcher,
scheduler = args.troy_workload_scheduler,
session = session)
overlay_manager = troy.OverlayManager (scheduler = args.troy_overlay_scheduler,
provisioner = args.troy_overlay_provisioner,
session = session)
# Questions:
# - How do we use bundles?
workload_id = workload_manager.create_workload(task_descriptions)
troy.execute_workload(workload_id, planner, overlay_manager,
workload_manager, strategy=args.troy_strategy)
planner = troy.Planner (planner = plugin_planner,
session = session)
# the troy.OverlayManager translates an overlay transcription into an
# overlay, then schedules and provisions it.
overlay_mgr = troy.OverlayManager (translator = plugin_overlay_translator,
scheduler = plugin_overlay_scheduler,
provisioner = plugin_overlay_provisioner,
session = session)
# the troy.WorkloadManager transforms a workload, schedules it over an
# overlay, and dispatches it to the pilots.
workload_mgr = troy.WorkloadManager (translator = plugin_workload_translator,
scheduler = plugin_workload_scheduler,
dispatcher = plugin_workload_dispatcher,
session = session)
# The order of actions on the planner, overlay manager and workload manager
# is orchestrated by a troy execution strategy (which represents a specific
# trace in the original troy design).
troy.execute_workload (workload = workload,
planner = planner,
overlay_mgr = overlay_mgr,
workload_mgr = workload_mgr,
strategy = plugin_strategy)
# Woohooo! Magic has happened!
# Planning makes initial mapping of workload to overlay
planner = troy.Planner (session, derive = 'concurrent')
# Create a task for every radicalist
task_descriptions = list()
for r in radicalists:
fin = fnames[r] + '.in'
fout = fnames[r] + '.out'
task_descr = troy.TaskDescription()
task_descr.tag = "%s" % r
task_descr.executable = '/bin/cp'
task_descr.arguments = ['input', 'output']
task_descr.inputs = ["%s > %s" % (fin, 'input')]
task_descr.outputs = ["%s < %s" % (fout, 'output')]
task_descr.working_directory = "%%(home)s/troy_demo/tasks/%s/" % fnames[r]
print task_descr
task_descriptions.append (task_descr)
workload = troy.Workload (session, task_descriptions)
# execute the workload with the given execution strategy
troy.execute_workload (workload.id, planner, overlay_mgr, workload_mgr, strategy='basic')
# Wohooo! Magic has happened!