def process(self, run_settings):
try:
self.id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.id = 0
try:
self.created_nodes = ast.literal_eval(getval(
run_settings, '%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.created_nodes = []
try:
self.scheduled_nodes = ast.literal_eval(getval(
run_settings, '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.scheduled_nodes = []
try:
self.bootstrapped_nodes = ast.literal_eval(getval(
run_settings, '%s/stages/bootstrap/bootstrapped_nodes' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.bootstrapped_nodes = []
#messages.info(run_settings, "%d: destroy" % self.id)
comp_pltf_settings = self.get_platform_settings(
run_settings, '%s/platform/computation' % django_settings.SCHEMA_PREFIX)
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
def process(self, run_settings):
try:
self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.id = 0
try:
self.created_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/create/created_nodes' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.created_nodes = []
try:
self.scheduled_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.scheduled_nodes = []
try:
self.bootstrapped_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/bootstrap/bootstrapped_nodes' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.bootstrapped_nodes = []
messages.info(run_settings, "%d: destroy" % self.id)
comp_pltf_settings = self.get_platform_settings(
run_settings, 'http://rmit.edu.au/schemas/platform/computation')
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
def process(self, run_settings):
#messages.info(run_settings, "0: bootstrapping nodes")
comp_pltf_settings = self.get_platform_settings(
run_settings, '%s/platform/computation' % django_settings.SCHEMA_PREFIX)
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
def process(self, run_settings):
messages.info(run_settings, "1: create")
comp_pltf_settings = self.get_platform_settings(
run_settings, 'http://rmit.edu.au/schemas/platform/computation')
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
def process(self, run_settings):
messages.info(run_settings, "bootstrapping nodes")
comp_pltf_settings = self.get_platform_settings(
run_settings, 'http://rmit.edu.au/schemas/platform/computation')
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
def process(self, run_settings):
#messages.info(run_settings, "0: bootstrapping nodes")
comp_pltf_settings = self.get_platform_settings(
run_settings,
'%s/platform/computation' % django_settings.SCHEMA_PREFIX)
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
def make_runcontext_for_directive(platform_name,
directive_name,
directive_args,
initial_settings,
username,
parent=None):
"""
Create a new runcontext with the commmand equivalent to the directive
on the platform.
"""
logger.debug("directive_args=%s" % directive_args)
logger.debug("Platform Name %s" % platform_name)
user = User.objects.get(username=username) # FIXME: pass in username
logger.debug("user=%s" % user)
profile = models.UserProfile.objects.get(user=user)
logger.debug("profile=%s" % profile)
#platform = models.Platform.objects.get(name=platform_name)
#logger.debug("platform=%s" % platform)
run_settings = dict(initial_settings) # we may share initial_settings
directive = models.Directive.objects.get(name=directive_name)
logger.debug("directive=%s" % directive)
# Have removed need to search on platform as should need to check platform
# in this situation.
# command_for_directive = models.Command.objects.get(directive=directive,
# platform=platform)
# command_for_directive = models.Command.objects.get(directive=directive)
# logger.debug("command_for_directive=%s" % command_for_directive)
user_settings = retrieve_settings(profile)
logger.debug("user_settings=%s" % pformat(user_settings))
# turn the user's arguments into real command arguments.
command_args = _get_command_actual_args(directive_args, user_settings)
logger.debug("command_args=%s" % pformat(command_args))
run_settings = _make_run_settings_for_command(command_args, run_settings)
logger.debug("updated run_settings=%s" % run_settings)
try:
settings_valid, problem = check_settings_valid(run_settings,
user_settings,
directive.stage)
except Exception, e:
tc = traceback.format_exc()
logger.error(e)
logger.error(tc)
messages.error(
run_settings,
"0: internal error (%s stage):%s" % (directive.stage.name, e))
raise InvalidInputError("0: internal error (%s stage):%s" %
(directive.stage.name, e))
def process(self, run_settings):
#messages.info(run_settings, "1: create")
comp_pltf_settings = self.get_platform_settings(
run_settings, '%s/platform/computation' % django_settings.SCHEMA_PREFIX)
logger.debug("comp_pltf_settings=%s" % comp_pltf_settings)
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
def process(self, run_settings):
logger.debug("schedule processing")
comp_pltf_settings = self.get_platform_settings(
run_settings,
'%s/platform/computation' % django_settings.SCHEMA_PREFIX)
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
def make_runcontext_for_directive(platform_name, directive_name,
directive_args, initial_settings, username, parent=None):
"""
Create a new runcontext with the commmand equivalent to the directive
on the platform.
"""
logger.debug("directive_args=%s" % directive_args)
logger.debug("Platform Name %s" % platform_name)
user = User.objects.get(username=username) # FIXME: pass in username
logger.debug("user=%s" % user)
profile = models.UserProfile.objects.get(user=user)
logger.debug("profile=%s" % profile)
#platform = models.Platform.objects.get(name=platform_name)
#logger.debug("platform=%s" % platform)
run_settings = dict(initial_settings) # we may share initial_settings
directive = models.Directive.objects.get(name=directive_name)
logger.debug("directive=%s" % directive)
# Have removed need to search on platform as should need to check platform
# in this situation.
# command_for_directive = models.Command.objects.get(directive=directive,
# platform=platform)
# command_for_directive = models.Command.objects.get(directive=directive)
# logger.debug("command_for_directive=%s" % command_for_directive)
user_settings = retrieve_settings(profile)
logger.debug("user_settings=%s" % pformat(user_settings))
# turn the user's arguments into real command arguments.
command_args = _get_command_actual_args(
directive_args, user_settings)
logger.debug("command_args=%s" % pformat(command_args))
run_settings = _make_run_settings_for_command(command_args, run_settings)
logger.debug("updated run_settings=%s" % run_settings)
try:
settings_valid, problem = check_settings_valid(run_settings,
user_settings,
directive.stage)
except Exception, e:
logger.error(e)
messages.error(run_settings, "0: internal error (%s stage):%s" %
(directive.stage.name, e))
raise InvalidInputError("0: internal error (%s stage):%s" %
(directive.stage.name, e))
class Create(Stage):
def __init__(self, user_settings=None):
# self.group_id = ''
self.platform_type = None
logger.debug("Create stage initialized")
def is_triggered(self, run_settings):
"""
Return True if configure done and no nodes are created
"""
try:
configure_done = int(
getval(run_settings,
'%s/stages/configure/configure_done' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
return False
try:
create_done = int(
getval(run_settings,
'%s/stages/create/create_done' % RMIT_SCHEMA))
if create_done:
return False
except (SettingNotFoundException, ValueError):
pass
try:
self.created_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/create/created_nodes' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.created_nodes = []
return True
return False
def process(self, run_settings):
messages.info(run_settings, "1: create")
comp_pltf_settings = self.get_platform_settings(
run_settings, 'http://rmit.edu.au/schemas/platform/computation')
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
if platform_type == 'nectar' or platform_type == 'csrack':
self.strategy = strategies.CloudStrategy()
elif platform_type == 'nci':
self.strategy = strategies.ClusterStrategy()
local_settings = {}
try:
self.strategy.set_create_settings(run_settings, local_settings)
local_settings.update(comp_pltf_settings)
logger.debug('local_settings=%s' % local_settings)
except SettingNotFoundException, e:
logger.error(e)
messages.error(run_settings, e)
return
class Bootstrap(Stage):
"""
Schedules processes on a cloud infrastructure
"""
def __init__(self, user_settings=None):
logger.debug('Bootstrap stage initialised')
def is_triggered(self, run_settings):
try:
self.created_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/create/created_nodes' % RMIT_SCHEMA))
running_created_nodes = [
x for x in self.created_nodes if str(x[3]) == 'running'
]
logger.debug('running_created_nodes=%s' % running_created_nodes)
if len(running_created_nodes) == 0:
return False
except (SettingNotFoundException, ValueError):
return False
try:
self.bootstrapped_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/bootstrap/bootstrapped_nodes' % RMIT_SCHEMA))
logger.debug(
'bootstrapped nodes=%d, running created nodes = %d' %
(len(self.bootstrapped_nodes), len(running_created_nodes)))
return len(self.bootstrapped_nodes) < len(running_created_nodes)
except (SettingNotFoundException, ValueError):
self.bootstrapped_nodes = []
return True
return False
def process(self, run_settings):
messages.info(run_settings, "bootstrapping nodes")
comp_pltf_settings = self.get_platform_settings(
run_settings, 'http://rmit.edu.au/schemas/platform/computation')
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
if platform_type == 'nectar' or platform_type == 'csrack':
self.strategy = strategies.CloudStrategy()
elif platform_type == 'nci':
self.strategy = strategies.ClusterStrategy()
local_settings = {}
try:
self.strategy.set_bootstrap_settings(run_settings, local_settings)
local_settings.update(comp_pltf_settings)
except SettingNotFoundException, e:
logger.error(e)
messages.error(run_settings, e)
return
class Destroy(stage.Stage):
def __init__(self, user_settings=None):
logger.debug('Destroy stage initialised')
def is_triggered(self, run_settings):
try:
converged = int(
getval(run_settings,
'%s/stages/converge/converged' % RMIT_SCHEMA))
logger.debug("converged=%s" % converged)
except (ValueError, SettingNotFoundException) as e:
return False
if converged:
try:
run_finished = int(
getval(run_settings,
'%s/stages/destroy/run_finished' % RMIT_SCHEMA))
except (ValueError, SettingNotFoundException) as e:
return True
return not run_finished
return False
def process(self, run_settings):
try:
self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.id = 0
try:
self.created_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/create/created_nodes' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.created_nodes = []
try:
self.scheduled_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.scheduled_nodes = []
try:
self.bootstrapped_nodes = ast.literal_eval(
getval(run_settings,
'%s/stages/bootstrap/bootstrapped_nodes' % RMIT_SCHEMA))
except (SettingNotFoundException, ValueError):
self.bootstrapped_nodes = []
messages.info(run_settings, "%d: destroy" % self.id)
comp_pltf_settings = self.get_platform_settings(
run_settings, 'http://rmit.edu.au/schemas/platform/computation')
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
if platform_type == 'nectar' or platform_type == 'csrack':
self.strategy = strategies.CloudStrategy()
elif platform_type == 'nci':
self.strategy = strategies.ClusterStrategy()
local_settings = {}
try:
self.strategy.set_destroy_settings(run_settings, local_settings)
local_settings.update(comp_pltf_settings)
logger.debug('local_settings=%s' % local_settings)
except SettingNotFoundException, e:
logger.error(e)
messages.error(run_settings, e)
return
class Bootstrap(Stage):
"""
Schedules processes on a cloud infrastructure
"""
def __init__(self, user_settings=None):
logger.debug('Bootstrap stage initialised')
def is_triggered(self, run_settings):
try:
self.created_nodes = ast.literal_eval(
getval(
run_settings, '%s/stages/create/created_nodes' %
django_settings.SCHEMA_PREFIX))
running_created_nodes = [
x for x in self.created_nodes if str(x[3]) == 'running'
]
logger.debug('running_created_nodes=%s' % running_created_nodes)
if len(running_created_nodes) == 0:
return False
except (SettingNotFoundException, ValueError):
return False
try:
self.bootstrapped_nodes = ast.literal_eval(
getval(
run_settings, '%s/stages/bootstrap/bootstrapped_nodes' %
django_settings.SCHEMA_PREFIX))
logger.debug(
'bootstrapped nodes=%d, running created nodes = %d' %
(len(self.bootstrapped_nodes), len(running_created_nodes)))
return len(self.bootstrapped_nodes) < len(running_created_nodes)
except (SettingNotFoundException, ValueError):
self.bootstrapped_nodes = []
return True
return False
def process(self, run_settings):
#messages.info(run_settings, "0: bootstrapping nodes")
comp_pltf_settings = self.get_platform_settings(
run_settings,
'%s/platform/computation' % django_settings.SCHEMA_PREFIX)
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
# TODO: cache is as unlikely to change during execution
for platform_hook in django_settings.PLATFORM_CLASSES:
try:
hook = jobs.safe_import(platform_hook, [], {})
except ImproperlyConfigured as e:
logger.error("Cannot load platform hook %s" % e)
continue
logger.debug("hook=%s" % hook)
logger.debug("hook.get_platform_types=%s" %
hook.get_platform_types())
logger.debug("platform_type=%s" % platform_type)
if platform_type in hook.get_platform_types():
self.strategy = hook.get_strategy(platform_type)
logger.debug("self.strategy=%s" % self.strategy)
break
# if platform_type in ['nectar', 'csrack', 'amazon']:
# self.strategy = strategies.CloudStrategy()
# elif platform_type in ['nci']:
# self.strategy = strategies.ClusterStrategy()
local_settings = {}
try:
self.strategy.set_bootstrap_settings(run_settings, local_settings)
local_settings.update(comp_pltf_settings)
except SettingNotFoundException, e:
logger.error(e)
messages.error(run_settings, e)
return
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
if platform_type == 'nectar' or platform_type == 'csrack':
self.strategy = strategies.CloudStrategy()
elif platform_type == 'nci':
self.strategy = strategies.ClusterStrategy()
local_settings = {}
try:
self.strategy.set_schedule_settings(run_settings, local_settings)
local_settings.update(comp_pltf_settings)
except SettingNotFoundException, e:
logger.error(e)
messages.error(run_settings, e)
return
try:
self.started = int(getval(run_settings, '%s/stages/schedule/schedule_started' % RMIT_SCHEMA))
except SettingNotFoundException:
self.started = 0
except ValueError, e:
logger.error(e)
logger.debug("started=%s" % self.started)
logger.debug('Schedule there')
if not self.started:
logger.debug("initial run")
try:
self.schedule_index = int(getval(run_settings, '%s/stages/schedule/schedule_index' % RMIT_SCHEMA))
except SettingNotFoundException:
class Destroy(stage.Stage):
def __init__(self, user_settings=None):
logger.debug('Destroy stage initialised')
def is_triggered(self, run_settings):
try:
converged = int(getval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX))
logger.debug("converged=%s" % converged)
except (ValueError, SettingNotFoundException) as e:
return False
if converged:
try:
run_finished = int(getval(run_settings,
'%s/stages/destroy/run_finished'
% django_settings.SCHEMA_PREFIX))
except (ValueError, SettingNotFoundException) as e:
return True
return not run_finished
return False
def process(self, run_settings):
try:
self.id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.id = 0
try:
self.created_nodes = ast.literal_eval(getval(
run_settings, '%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.created_nodes = []
try:
self.scheduled_nodes = ast.literal_eval(getval(
run_settings, '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.scheduled_nodes = []
try:
self.bootstrapped_nodes = ast.literal_eval(getval(
run_settings, '%s/stages/bootstrap/bootstrapped_nodes' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.bootstrapped_nodes = []
#messages.info(run_settings, "%d: destroy" % self.id)
comp_pltf_settings = self.get_platform_settings(
run_settings, '%s/platform/computation' % django_settings.SCHEMA_PREFIX)
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
# TODO: cache is as unlikely to change during execution
for platform_hook in django_settings.PLATFORM_CLASSES:
try:
hook = jobs.safe_import(platform_hook, [], {})
except ImproperlyConfigured as e:
logger.error("Cannot load platform hook %s" % e)
continue
logger.debug("hook=%s" % hook)
logger.debug("hook.get_platform_types=%s" % hook.get_platform_types())
logger.debug("platform_type=%s" % platform_type)
if platform_type in hook.get_platform_types():
self.strategy = hook.get_strategy(platform_type)
logger.debug("self.strategy=%s" % self.strategy)
break
# if platform_type in ['nectar', 'csrack', 'amazon']:
# self.strategy = strategies.CloudStrategy()
# elif platform_type in ['nci']:
# self.strategy = strategies.ClusterStrategy()
local_settings = {}
try:
self.strategy.set_destroy_settings(run_settings, local_settings)
local_settings.update(comp_pltf_settings)
logger.debug('local_settings=%s' % local_settings)
except SettingNotFoundException, e:
logger.error(e)
messages.error(run_settings, e)
return
continue
logger.debug("hook=%s" % hook)
logger.debug("hook.get_platform_types=%s" %
hook.get_platform_types())
logger.debug("platform_type=%s" % platform_type)
if platform_type in hook.get_platform_types():
self.strategy = hook.get_strategy(platform_type)
logger.debug("self.strategy=%s" % self.strategy)
break
local_settings = {}
try:
self.strategy.set_schedule_settings(run_settings, local_settings)
local_settings.update(comp_pltf_settings)
except SettingNotFoundException, e:
logger.error(e)
messages.error(run_settings, e)
return
try:
self.started = int(
getval(
run_settings, '%s/stages/schedule/schedule_started' %
django_settings.SCHEMA_PREFIX))
except SettingNotFoundException:
self.started = 0
except ValueError, e:
logger.error(e)
logger.debug("started=%s" % self.started)
logger.debug('Schedule there')
if not self.started:
logger.debug("initial run")
# context creation.
#user_settings = run_settings[models.UserProfile.PROFILE_SCHEMA_NS]
# PROFILE_SCHEMA is now deprecated, so
user_settings = {}
triggered = 0
for current_stage in stageset:
logger.debug("checking stage %s for trigger" % current_stage.name)
# get the actual stage object
try:
stage = jobs.safe_import(current_stage.package, [],
{'user_settings': deepcopy(user_settings)}) # obviously need to cache this
except ImproperlyConfigured, e:
logger.error(e)
messages.error(run_settings, "0: internal error (%s stage):%s" % (str(current_stage.name), e))
raise
logger.debug("process stage=%s", stage)
task_run_settings = deepcopy(run_settings)
logger.debug("starting task settings = %s" % pformat(task_run_settings))
# stage_settings are read only as transfered into context here
stage_settings = current_stage.get_settings()
logger.debug("stage_settings=%s" % stage_settings)
# This is nasty
task_run_settings = jobs.transfer(task_run_settings, stage_settings)
#task_run_settings.update(stage_settings)
logger.debug("task run_settings=%s" % task_run_settings)
class Create(Stage):
def __init__(self, user_settings=None):
# self.group_id = ''
self.platform_type = None
logger.debug("Create stage initialized")
def is_triggered(self, run_settings):
"""
Return True if configure done and no nodes are created
"""
try:
configure_done = int(getval(run_settings,
'%s/stages/configure/configure_done' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
return False
try:
create_done = int(getval(run_settings,
'%s/stages/create/create_done' % django_settings.SCHEMA_PREFIX))
if create_done:
return False
except (SettingNotFoundException, ValueError):
pass
try:
self.created_nodes = ast.literal_eval(getval(
run_settings, '%s/stages/create/created_nodes' % django_settings.SCHEMA_PREFIX))
except (SettingNotFoundException, ValueError):
self.created_nodes = []
return True
return False
def process(self, run_settings):
#messages.info(run_settings, "1: create")
comp_pltf_settings = self.get_platform_settings(
run_settings, '%s/platform/computation' % django_settings.SCHEMA_PREFIX)
logger.debug("comp_pltf_settings=%s" % comp_pltf_settings)
try:
platform_type = comp_pltf_settings['platform_type']
except KeyError, e:
logger.error(e)
messages.error(run_settings, e)
return
logger.debug("platform_type=%s" % platform_type)
# TODO: cache is as unlikely to change during execution
for platform_hook in django_settings.PLATFORM_CLASSES:
try:
hook = jobs.safe_import(platform_hook, [], {})
except ImproperlyConfigured as e:
logger.error("Cannot load platform hook %s" % e)
continue
logger.debug("hook=%s" % hook)
logger.debug("hook.get_platform_types=%s" % hook.get_platform_types())
logger.debug("platform_type=%s" % platform_type)
if platform_type in hook.get_platform_types():
self.strategy = hook.get_strategy(platform_type)
logger.debug("self.strategy=%s" % self.strategy)
break
# if platform_type in ['nectar', 'csrack', 'amazon']:
# self.strategy = strategies.CloudStrategy()
# elif platform_type in ['nci']:
# self.strategy = strategies.ClusterStrategy()
local_settings = {}
try:
self.strategy.set_create_settings(run_settings, local_settings)
local_settings.update(comp_pltf_settings)
logger.debug('local_settings=%s' % local_settings)
except SettingNotFoundException, e:
logger.error(e)
messages.error(run_settings, e)
return
