def output(self, run_settings):
setval(run_settings,
'%s/stages/configure/configure_done' % RMIT_SCHEMA,
1)
setval(run_settings,
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA,
str(self.experiment_id))
if self.output_exists(run_settings):
if self.output_platform_name:
run_settings.setdefault(
RMIT_SCHEMA + '/platform/storage/output',
{})[u'platform_url'] = self.output_platform_name
#if self.output_platform_offset:
# run_settings[RMIT_SCHEMA + '/platform/storage/output']['offset'] = self.output_platform_offset
#else:
run_settings['http://rmit.edu.au/schemas/platform/storage/output']['offset'] = self.output_loc_offset
if self.input_exists(run_settings):
if self.input_platform_name:
run_settings.setdefault(
RMIT_SCHEMA + '/platform/storage/input',
{})[u'platform_url'] = self.input_platform_name
if self.input_platform_offset:
run_settings[RMIT_SCHEMA + '/platform/storage/input']['offset'] = self.input_platform_offset
logger.debug('self.compute_platform_name=%s' % self.compute_platform_name)
if self.compute_platform_name:
run_settings.setdefault('http://rmit.edu.au/schemas/platform/computation',
{})[u'platform_url'] = self.compute_platform_name
if self.compute_platform_offset:
run_settings.setdefault('http://rmit.edu.au/schemas/platform/computation',
{})[u'offset'] = self.compute_platform_offset
return run_settings
def output(self, run_settings):
"""
Output new runs_left value (including zero value)
"""
logger.debug("finished stage output")
executing_procs = [x for x in self.current_processes if x['status'] != 'ready']
#nodes_working = len(self.executed_procs) - (len(self.finished_nodes) + self.failed_processes)
nodes_working = len(executing_procs) - (len(self.finished_nodes) + self.failed_processes)
#if len(self.finished_nodes) == 0 and self.failed_processes == 0:
# nodes_working = 0
logger.debug("%d %d " %(len(self.finished_nodes), self.failed_processes))
#logger.debug("self.executed_procs=%s" % self.executed_procs)
logger.debug("self.executed_procs=%s" % executing_procs)
logger.debug("self.finished_nodes=%s" % self.finished_nodes)
#FIXME: nodes_working can be negative
# FIXME: possible race condition?
# if not self._exists(run_settings, '%s/stages/run' % django_settings.SCHEMA_PREFIX):
# run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX] = {}
# #run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]['runs_left'] = nodes_working
#run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]['error_nodes'] = nodes_working
setvals(run_settings, {
'%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX: nodes_working,
'%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX: nodes_working,
})
if not nodes_working:
self.finished_nodes = []
#messages.success(run_settings, '%d: All processes in current iteration completed' % (self.id + 1))
setvals(run_settings, {
'%s/stages/run/finished_nodes' % django_settings.SCHEMA_PREFIX: str(self.finished_nodes),
'%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX: str(self.all_processes),
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: str(self.current_processes),
'%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: str(self.executed_procs)
})
if self.failed_nodes:
setval(run_settings, '%s/stages/create/failed_nodes' % django_settings.SCHEMA_PREFIX, self.failed_nodes)
# run_settings.setdefault(
completed_procs = [x for x in self.executed_procs if x['status'] == 'completed']
# messages.info(run_settings, "%s: waiting (%s of %s processes done)"
# % (self.id + 1, len(completed_procs), len(self.current_processes) - (self.failed_processes)))
if self.procs_2b_rescheduled:
setvals(run_settings, {
'%s/stages/schedule/scheduled_started' % django_settings.SCHEMA_PREFIX: 0,
'%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX: self.procs_2b_rescheduled
})
return run_settings
def output(self, run_settings):
setval(run_settings, '%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX, str(self.experiment_id))
if not self.done_iterating:
# trigger first of iteration corestages
logger.debug("nonconvergence")
setvals(run_settings, {
'%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX: '[]',
'%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX: '[]',
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX: '[]',
'%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX: 0,
'%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX: 0,
'%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX: 0
})
logger.debug('scheduled_nodes=%s' % getval(run_settings, '%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX))
try:
delkey(run_settings, '%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
try:
delkey(run_settings, '%s/stages/run/error_nodes' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
# run = run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]
# del run['error_nodes']
#update_key('converged', False, context)
setval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 0)
# run_settings['%s/stages/converge' % django_settings.SCHEMA_PREFIX][u'converged'] = 0
# delete_key('runs_left', context)
# delete_key('error_nodes', context)
# update_key('converged', False, context)
else:
logger.debug("convergence")
# we are done, so trigger next stage outside of converge
#update_key('converged', True, context)
setval(run_settings, '%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX, 1)
# we are done, so don't trigger iteration stages
#update_key('criterion', self.criterion, context)
setval(run_settings, '%s/stages/converge/criterion' % django_settings.SCHEMA_PREFIX, unicode(self.criterion))
# delete_key('error_nodes', context)
#delete_key('transformed', context)
try:
delkey(run_settings, '%s/stages/transform/transformed' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
self.id += 1
# update_key('id', self.id, context)
setval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX, self.id)
return run_settings
def output(self, run_settings):
"""
Inserting a new group if into run settings.
"""
logger.debug('output')
setval(run_settings, "%s/stages/create/created_nodes" % RMIT_SCHEMA,
self.created_nodes)
setval(run_settings, "%s/stages/create/create_done" % RMIT_SCHEMA, 1)
logger.debug("Updated run settings %s" % run_settings)
return run_settings
def output(self, run_settings):
"""
Inserting a new group if into run settings.
"""
logger.debug('output')
setval(run_settings,
"%s/stages/create/created_nodes" % RMIT_SCHEMA,
self.created_nodes)
setval(run_settings,
"%s/stages/create/create_done" % RMIT_SCHEMA,
1)
logger.debug("Updated run settings %s" % run_settings)
return run_settings
def output(self, run_settings):
self.writeout_output(run_settings)
self.writeout_input(run_settings)
self.writeout_computation(run_settings)
setval(run_settings,
'%s/stages/configure/configure_done' % django_settings.SCHEMA_PREFIX,
1)
setval(run_settings,
'%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX,
str(self.experiment_id))
return run_settings
def output(self, run_settings):
logger.debug("sweep output")
setval(run_settings,
'%s/stages/sweep/sweep_done' % django_settings.SCHEMA_PREFIX, 1)
logger.debug('interesting run_settings=%s' % run_settings)
with ignored(SettingNotFoundException):
if getvals(run_settings,
'%s/input/mytardis' % django_settings.SCHEMA_PREFIX):
setval(
run_settings, '%s/input/mytardis/experiment_id' %
django_settings.SCHEMA_PREFIX, str(self.experiment_id))
if not self.error_detected:
messages.success(run_settings, "0: sweep completed")
return run_settings
def output(self, run_settings):
logger.debug("sweep output")
setval(run_settings, '%s/stages/sweep/sweep_done' % RMIT_SCHEMA, 1)
logger.debug('interesting run_settings=%s' % run_settings)
try:
if getvals(run_settings, '%s/input/mytardis' % RMIT_SCHEMA):
setval(run_settings,
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA,
str(self.experiment_id))
except SettingNotFoundException:
pass
if not self.error_detected:
messages.success(run_settings, "0: completed")
return run_settings
def output(self, run_settings):
setvals(
run_settings, {
'%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX:
str(self.scheduled_nodes),
'%s/stages/schedule/rescheduled_nodes' % django_settings.SCHEMA_PREFIX:
str(self.rescheduled_nodes),
'%s/stages/schedule/all_processes' % django_settings.SCHEMA_PREFIX:
str(self.all_processes),
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX:
str(self.current_processes)
})
if not self.started:
setvals(
run_settings, {
'%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX:
1,
'%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX:
self.procs_2b_rescheduled
})
if not self.procs_2b_rescheduled:
setvals(
run_settings, {
'%s/stages/schedule/total_processes' % django_settings.SCHEMA_PREFIX:
str(self.total_processes),
'%s/stages/schedule/schedule_index' % django_settings.SCHEMA_PREFIX:
self.schedule_index
})
else:
if self.procs_2b_rescheduled:
setval(
run_settings,
'%s/stages/schedule/total_rescheduled_procs' %
django_settings.SCHEMA_PREFIX,
self.total_rescheduled_procs)
if self.total_rescheduled_procs == len(
self.procs_2b_rescheduled):
setvals(
run_settings, {
'%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX:
1,
'%s/stages/schedule/procs_2b_rescheduled' % django_settings.SCHEMA_PREFIX:
[],
'%s/stages/schedule/total_rescheduled_procs' % django_settings.SCHEMA_PREFIX:
0,
'%s/stages/schedule/rescheduled_nodes' % django_settings.SCHEMA_PREFIX:
[],
})
else:
setval(
run_settings, '%s/stages/schedule/total_scheduled_procs' %
django_settings.SCHEMA_PREFIX, self.total_scheduled_procs)
if self.total_scheduled_procs == len(self.current_processes):
setval(
run_settings, '%s/stages/schedule/schedule_completed' %
django_settings.SCHEMA_PREFIX, 1)
return run_settings
def output(self, run_settings):
setvals(run_settings, {
'%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: str(self.scheduled_nodes),
'%s/stages/schedule/rescheduled_nodes' % RMIT_SCHEMA: str(self.rescheduled_nodes),
'%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes),
'%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.current_processes)
})
if not self.started:
setvals(run_settings, {
'%s/stages/schedule/schedule_started' % RMIT_SCHEMA: 1,
'%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: self.procs_2b_rescheduled
})
if not self.procs_2b_rescheduled:
setvals(run_settings, {
'%s/stages/schedule/total_processes' % RMIT_SCHEMA: str(self.total_processes),
'%s/stages/schedule/schedule_index' % RMIT_SCHEMA: self.schedule_index
})
else:
if self.procs_2b_rescheduled:
setval(run_settings,
'%s/stages/schedule/total_rescheduled_procs' % RMIT_SCHEMA,
self.total_rescheduled_procs)
if self.total_rescheduled_procs == len(self.procs_2b_rescheduled):
setvals(run_settings, {
'%s/stages/schedule/schedule_completed' % RMIT_SCHEMA: 1,
'%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: [],
'%s/stages/schedule/total_rescheduled_procs' % RMIT_SCHEMA: 0,
'%s/stages/schedule/rescheduled_nodes' % RMIT_SCHEMA: [],
})
else:
setval(run_settings,
'%s/stages/schedule/total_scheduled_procs' % RMIT_SCHEMA,
self.total_scheduled_procs)
if self.total_scheduled_procs == len(self.current_processes):
setval(run_settings,
'%s/stages/schedule/schedule_completed' % RMIT_SCHEMA,
1)
return run_settings
class Sweep(Stage):
def __init__(self, user_settings=None):
self.numbfile = 0
logger.debug("Sweep stage initialized")
def is_triggered(self, run_settings):
logger.debug('run_settings=%s' % run_settings)
try:
configure_done = int(getval(run_settings,
'%s/stages/sweep/sweep_done' % RMIT_SCHEMA))
except (ValueError, SettingNotFoundException):
return True
return not configure_done
def _get_sweep_name(self, run_settings):
try:
sweep_name = getval(run_settings, '%s/directive_profile/sweep_name' % RMIT_SCHEMA)
except SettingNotFoundException:
sweep_name = 'unknown_sweep'
return sweep_name
def process(self, run_settings):
logger.debug('run_settings=%s' % run_settings)
# Need to make copy because we pass on run_settings to sub connector
# so any changes we make here to run_settings WILL be inherited
def make_local_settings(run_settings):
from copy import deepcopy
local_settings = deepcopy(getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS))
update(local_settings, run_settings,
RMIT_SCHEMA + '/system/platform',
# RMIT_SCHEMA + '/input/mytardis/experiment_id',
# RMIT_SCHEMA + '/system/random_numbers',
)
local_settings['bdp_username'] = getval(
run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA)
return local_settings
local_settings = make_local_settings(run_settings)
logger.debug('local_settings=%s' % local_settings)
setval(run_settings,
'%s/platform/computation/platform_url' % RMIT_SCHEMA,
getval(run_settings,
'%s/input/system/compplatform/computation_platform'
% RMIT_SCHEMA))
def _parse_output_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return name, offset
contextid = int(getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA))
logger.debug("contextid=%s" % contextid)
sweep_name = self._get_sweep_name(run_settings)
logger.debug("sweep_name=%s" % sweep_name)
output_loc = self.output_exists(run_settings)
location = ""
if output_loc:
location = getval(run_settings, output_loc)
output_storage_name, output_storage_offset = \
_parse_output_location(run_settings, location)
setval(run_settings,
'%s/platform/storage/output/platform_url' % RMIT_SCHEMA,
output_storage_name)
setval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA,
os.path.join(output_storage_offset, '%s%s' % (sweep_name, contextid)))
def _parse_input_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return (name, offset)
input_loc = self.input_exists(run_settings)
if input_loc:
location = getval(run_settings, input_loc)
input_storage_name, input_storage_offset = \
_parse_input_location(run_settings, location)
setval(run_settings, '%s/platform/storage/input/platform_url' % RMIT_SCHEMA,
input_storage_name)
# store offsets
setval(run_settings,
'%s/platform/storage/input/offset' % RMIT_SCHEMA,
input_storage_offset)
# TODO: replace with scratch space computation platform space
self.scratch_platform = '%s%s%s' % (
manage.get_scratch_platform(), sweep_name,
contextid)
# mytardis
if output_loc:
try:
self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA))
except KeyError:
self.experiment_id = 0
except ValueError:
self.experiment_id = 0
try:
curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA)
except SettingNotFoundException:
curate_data = False
if curate_data:
self.experiment_id = self.curate_data(run_settings, location, self.experiment_id)
setval(run_settings,
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA,
str(self.experiment_id))
# generate all variations
map_text = getval(run_settings, '%s/input/sweep/sweep_map' % RMIT_SCHEMA)
# map_text = run_settings[RMIT_SCHEMA + '/input/sweep']['sweep_map']
sweep_map = json.loads(map_text)
logger.debug("sweep_map=%s" % pformat(sweep_map))
runs = _expand_variations(maps=[sweep_map], values={})
logger.debug("runs=%s" % runs)
# Create random numbers if needed
# TODO: move iseed out of hrmc into separate generic schema
# to use on any sweepable connector and make this function
# completely hrmc independent.
rands = []
try:
self.rand_index = getval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA)
logger.debug("rand_index=%s" % self.rand_index)
except SettingNotFoundException:
pass
else:
# prep random seeds for each run based off original iseed
# FIXME: inefficient for large random file
# TODO, FIXME: this is potentially problematic if different
# runs end up overlapping in the random numbers they utilise.
# solution is to have separate random files per run or partition
# big file up.
try:
num_url = getval(run_settings, "%s/system/random_numbers" % RMIT_SCHEMA)
logger.debug('num_url=%s' % num_url)
except SettingNotFoundException:
pass
else:
try:
local_settings['random_numbers'] = num_url
rands = generate_rands(settings=local_settings,
start_range=0,
end_range=-1,
num_required=len(runs),
start_index=self.rand_index)
logger.debug("rands=%s" % rands)
except Exception, e:
logger.debug('error')
logger.error(e)
raise
# load initial values map in the input directory which
# contains variable to use for all subdirectives
starting_map = {}
if input_loc:
input_storage_settings = self.get_platform_settings(
run_settings, 'http://rmit.edu.au/schemas/platform/storage/input')
try:
input_prefix = '%s://%s@' % (input_storage_settings['scheme'],
input_storage_settings['type'])
values_url = get_url_with_credentials(
input_storage_settings,
input_prefix + os.path.join(input_storage_settings['ip_address'],
input_storage_offset, "initial", VALUES_MAP_FILE),
is_relative_path=False)
logger.debug("values_url=%s" % values_url)
values_e_url = get_url_with_credentials(
local_settings,
values_url,
is_relative_path=False)
logger.debug("values_url=%s" % values_e_url)
values_content = get_file(values_e_url)
logger.debug("values_content=%s" % values_content)
starting_map = dict(json.loads(values_content))
except IOError:
logger.warn("no starting values file found")
except ValueError:
logger.error("problem parsing contents of %s" % VALUES_MAP_FILE)
pass
logger.debug("starting_map after initial values=%s"
% pformat(starting_map))
# Copy form input values info starting map
# FIXME: could have name collisions between form inputs and
# starting values.
for ns in run_settings:
if ns.startswith(RMIT_SCHEMA + "/input"):
# for k, v in run_settings[ns].items():
for k, v in getvals(run_settings, ns).items():
starting_map[k] = v
logger.debug("starting_map after form=%s" % pformat(starting_map))
# FIXME: we assume we will always have input directory
# Get input_url directory
input_url = ""
if input_loc:
input_prefix = '%s://%s@' % (input_storage_settings['scheme'],
input_storage_settings['type'])
input_url = get_url_with_credentials(input_storage_settings,
input_prefix + os.path.join(input_storage_settings['ip_address'],
input_storage_offset),
is_relative_path=False)
logger.debug("input_url=%s" % input_url)
current_context = models.Context.objects.get(id=contextid)
user = current_context.owner.user.username
# For each of the generated runs, copy across initial input
# to individual input directories with variation values,
# and then schedule subrun of sub directive
logger.debug("run_settings=%s" % run_settings)
for i, context in enumerate(runs):
run_counter = int(context['run_counter'])
logger.debug("run_counter=%s" % run_counter)
run_inputdir = os.path.join(self.scratch_platform,
SUBDIRECTIVE_DIR % {'run_counter': str(run_counter)},
FIRST_ITERATION_DIR,)
logger.debug("run_inputdir=%s" % run_inputdir)
run_iter_url = get_url_with_credentials(local_settings,
run_inputdir, is_relative_path=False)
logger.debug("run_iter_url=%s" % run_iter_url)
# Duplicate any input_directory into runX duplicates
if input_loc:
logger.debug("context=%s" % context)
logger.debug("systemsettings=%s"
% pformat(getvals(run_settings, RMIT_SCHEMA + '/input/system')))
copy_directories(input_url, run_iter_url)
# Need to load up existing values, because original input_dir could
# have contained values for the whole run
# This code is deprecated in favour of single values file.
self.error_detected = False
try:
template_name = getval(run_settings,
'%s/stages/sweep/template_name'
% RMIT_SCHEMA)
except SettingNotFoundException:
pass
else:
logger.debug("template_name=%s" % template_name)
v_map = {}
try:
values_url = get_url_with_credentials(
local_settings,
os.path.join(run_inputdir, "initial",
VALUES_MAP_TEMPLATE_FILE % {'template_name': template_name}),
is_relative_path=False)
logger.debug("values_url=%s" % values_url)
values_content = get_file(values_url)
logger.debug("values_content=%s" % values_content)
v_map = dict(json.loads(values_content), indent=4)
except IOError:
logger.warn("no values file found")
except ValueError:
logger.error("problem parsing contents of %s" % VALUES_MAP_FILE)
pass
v_map.update(starting_map)
v_map.update(context)
logger.debug("new v_map=%s" % v_map)
put_file(values_url, json.dumps(v_map, indent=4))
v_map = {}
try:
values_url = get_url_with_credentials(
local_settings,
os.path.join(run_inputdir, "initial",
VALUES_MAP_FILE),
is_relative_path=False)
logger.debug("values_url=%s" % values_url)
values_content = get_file(values_url)
logger.debug("values_content=%s" % values_content)
v_map = dict(json.loads(values_content), )
except IOError:
logger.warn("no values file found")
except ValueError:
logger.error("problem parsing contents of %s" % VALUES_MAP_FILE)
pass
v_map.update(starting_map)
v_map.update(context)
logger.debug("new v_map=%s" % v_map)
put_file(values_url, json.dumps(v_map, indent=4))
# Set random numbers for subdirective
logger.debug("run_settings=%s" % pformat(run_settings))
if rands:
setval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA, rands[i])
if input_loc:
# Set revised input_location for subdirective
setval(run_settings, input_loc,
"%s/%s/%s" % (self.scratch_platform,
SUBDIRECTIVE_DIR
% {'run_counter': str(run_counter)},
FIRST_ITERATION_DIR))
# Redirect input
run_input_storage_name, run_input_storage_offset = \
_parse_input_location(run_settings,
"local/sweep%s/run%s/input_0" % (contextid, run_counter))
# setval(run_settings,
# '%s/platform/storage/input/platform_url' % RMIT_SCHEMA,
# run_input_storage_name)
# setval(run_settings,
# '%s/platform/storage/input/offset' % RMIT_SCHEMA,
# run_input_storage_offset)
logger.debug("run_settings=%s" % pformat(run_settings))
try:
_submit_subdirective("nectar", run_settings, user, current_context)
except Exception, e:
logger.error(e)
raise e
def output(self, run_settings):
# if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/converge'):
# run_settings['http://rmit.edu.au/schemas/stages/converge'] = {}
setval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA, str(self.experiment_id))
# run_settings['http://rmit.edu.au/schemas/input/mytardis']['experiment_id'] = str(self.experiment_id)
if not self.done_iterating:
# trigger first of iteration corestages
logger.debug("nonconvergence")
setvals(run_settings, {
'%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA: '[]',
'%s/stages/execute/executed_procs' % RMIT_SCHEMA: '[]',
'%s/stages/schedule/current_processes' % RMIT_SCHEMA: '[]',
'%s/stages/schedule/total_scheduled_procs' % RMIT_SCHEMA: 0,
'%s/stages/schedule/schedule_completed' % RMIT_SCHEMA: 0,
'%s/stages/schedule/schedule_started' % RMIT_SCHEMA: 0
})
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'scheduled_nodes'] = '[]'
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/execute',
# {})[u'executed_procs'] = '[]'
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'current_processes'] = '[]'
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'total_scheduled_procs'] = 0
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'schedule_completed'] = 0
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'schedule_started'] = 0
logger.debug('scheduled_nodes=%s' % getval(run_settings, '%s/stages/schedule/scheduled_nodes' % RMIT_SCHEMA))
try:
delkey(run_settings, '%s/stages/run/runs_left' % RMIT_SCHEMA)
except SettingNotFoundException:
pass
# run = run_settings['http://rmit.edu.au/schemas/stages/run']
# del run['runs_left']
try:
delkey(run_settings, '%s/stages/run/error_nodes' % RMIT_SCHEMA)
except SettingNotFoundException:
pass
# run = run_settings['%s/stages/run' % RMIT_SCHEMA]
# del run['error_nodes']
#update_key('converged', False, context)
setval(run_settings, '%s/stages/converge/converged' % RMIT_SCHEMA, 0)
# run_settings['%s/stages/converge' % RMIT_SCHEMA][u'converged'] = 0
# delete_key('runs_left', context)
# delete_key('error_nodes', context)
# update_key('converged', False, context)
else:
logger.debug("convergence")
# we are done, so trigger next stage outside of converge
#update_key('converged', True, context)
setval(run_settings, '%s/stages/converge/converged' % RMIT_SCHEMA, 1)
# run_settings['http://rmit.edu.au/schemas/stages/converge'][u'converged'] = 1
# we are done, so don't trigger iteration stages
#update_key('criterion', self.criterion, context)
setval(run_settings, '%s/stages/converge/criterion' % RMIT_SCHEMA, unicode(self.criterion))
# run_settings['http://rmit.edu.au/schemas/stages/converge'][u'criterion'] = unicode(self.criterion)
# delete_key('error_nodes', context)
#delete_key('transformed', context)
try:
delkey(run_settings, '%s/stages/transform/transformed' % RMIT_SCHEMA)
except SettingNotFoundException:
pass
# run = run_settings['http://rmit.edu.au/schemas/stages/transform']
# del run['transformed']
self.id += 1
# update_key('id', self.id, context)
setval(run_settings, '%s/system/id' % RMIT_SCHEMA, self.id)
# run_settings['http://rmit.edu.au/schemas/system'][u'id'] = self.id
return run_settings
def process(self, run_settings):
logger.debug('run_settings=%s' % run_settings)
messages.info(run_settings, "0: sweep started")
# Need to make copy because we pass on run_settings to sub connector
# so any changes we make here to run_settings WILL be inherited
def make_local_settings(run_settings):
from copy import deepcopy
local_settings = deepcopy(
getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS))
update(
local_settings,
run_settings,
django_settings.SCHEMA_PREFIX + '/system/platform',
# django_settings.SCHEMA_PREFIX + '/input/mytardis/experiment_id',
# django_settings.SCHEMA_PREFIX + '/system/random_numbers',
)
local_settings['bdp_username'] = getval(
run_settings,
'%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX)
return local_settings
local_settings = make_local_settings(run_settings)
logger.debug('local_settings=%s' % local_settings)
compplatform = [
k for k, v in run_settings.iteritems()
if k.startswith('%s/input/system/compplatform' %
django_settings.SCHEMA_PREFIX)
]
setval(
run_settings, '%s/platform/computation/platform_url' %
django_settings.SCHEMA_PREFIX,
getval(run_settings, '%s/computation_platform' % compplatform[0]))
def _parse_output_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return name, offset
contextid = int(
getval(run_settings,
'%s/system/contextid' % django_settings.SCHEMA_PREFIX))
logger.debug("contextid=%s" % contextid)
sweep_name = self._get_sweep_name(run_settings)
logger.debug("sweep_name=%s" % sweep_name)
output_loc = self.output_exists(run_settings)
location = ""
if output_loc:
location = getval(run_settings, output_loc)
output_storage_name, output_storage_offset = \
_parse_output_location(run_settings, location)
setval(
run_settings, '%s/platform/storage/output/platform_url' %
django_settings.SCHEMA_PREFIX, output_storage_name)
setval(
run_settings, '%s/platform/storage/output/offset' %
django_settings.SCHEMA_PREFIX,
os.path.join(output_storage_offset,
'%s%s' % (sweep_name, contextid)))
def _parse_input_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return (name, offset)
input_loc = self.input_exists(run_settings)
logger.debug('special_input_loc=%s' % input_loc)
if input_loc:
location = getval(run_settings, input_loc)
input_storage_name, input_storage_offset = \
_parse_input_location(run_settings, location)
setval(
run_settings, '%s/platform/storage/input/platform_url' %
django_settings.SCHEMA_PREFIX, input_storage_name)
# store offsets
setval(
run_settings, '%s/platform/storage/input/offset' %
django_settings.SCHEMA_PREFIX, input_storage_offset)
# TODO: replace with scratch space computation platform space
self.scratch_platform = '%s%s%s' % (manage.get_scratch_platform(),
sweep_name, contextid)
# mytardis
if output_loc:
try:
self.experiment_id = int(
getval(
run_settings, '%s/input/mytardis/experiment_id' %
django_settings.SCHEMA_PREFIX))
except KeyError, ValueError:
self.experiment_id = 0
try:
curate_data = getval(
run_settings, '%s/input/mytardis/curate_data' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
curate_data = False
curate_data = False #TODO remove
if curate_data:
logger.debug('location=%s' % location)
location = "%s%s" % (sweep_name, contextid)
self.experiment_id = self.curate_data(run_settings, location,
self.experiment_id)
setval(
run_settings, '%s/input/mytardis/experiment_id' %
django_settings.SCHEMA_PREFIX, str(self.experiment_id))
except ValueError:
logger.error("problem parsing contents of %s" %
VALUES_MAP_FILE)
pass
v_map.update(starting_map)
v_map.update(context)
v_map['run_counter'] = 1
logger.debug("new v_map=%s" % v_map)
put_file(values_url, json.dumps(v_map, indent=4))
# Set random numbers for subdirective
logger.debug("run_settings=%s" % pformat(run_settings))
if rands:
setval(run_settings,
'%s/input/hrmc/iseed' % django_settings.SCHEMA_PREFIX,
rands[i])
if input_loc:
# Set revised input_location for subdirective
setval(
run_settings, input_loc,
"%s/%s/%s" % (self.scratch_platform, SUBDIRECTIVE_DIR % {
'run_counter': str(run_counter)
}, FIRST_ITERATION_DIR))
# Redirect input
run_input_storage_name, run_input_storage_offset = \
_parse_input_location(run_settings,
"local/%s%s/run%s/input_0" % (sweep_name, contextid, run_counter))
def process(self, run_settings):
logger.debug('run_settings=%s' % run_settings)
# Need to make copy because we pass on run_settings to sub connector
# so any changes we make here to run_settings WILL be inherited
def make_local_settings(run_settings):
from copy import deepcopy
local_settings = deepcopy(getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS))
update(local_settings, run_settings,
RMIT_SCHEMA + '/system/platform',
# RMIT_SCHEMA + '/input/mytardis/experiment_id',
# RMIT_SCHEMA + '/system/random_numbers',
)
local_settings['bdp_username'] = getval(
run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA)
return local_settings
local_settings = make_local_settings(run_settings)
logger.debug('local_settings=%s' % local_settings)
setval(run_settings,
'%s/platform/computation/platform_url' % RMIT_SCHEMA,
getval(run_settings,
'%s/input/system/compplatform/computation_platform'
% RMIT_SCHEMA))
def _parse_output_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return name, offset
contextid = int(getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA))
logger.debug("contextid=%s" % contextid)
sweep_name = self._get_sweep_name(run_settings)
logger.debug("sweep_name=%s" % sweep_name)
output_loc = self.output_exists(run_settings)
location = ""
if output_loc:
location = getval(run_settings, output_loc)
output_storage_name, output_storage_offset = \
_parse_output_location(run_settings, location)
setval(run_settings,
'%s/platform/storage/output/platform_url' % RMIT_SCHEMA,
output_storage_name)
setval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA,
os.path.join(output_storage_offset, '%s%s' % (sweep_name, contextid)))
def _parse_input_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return (name, offset)
input_loc = self.input_exists(run_settings)
if input_loc:
location = getval(run_settings, input_loc)
input_storage_name, input_storage_offset = \
_parse_input_location(run_settings, location)
setval(run_settings, '%s/platform/storage/input/platform_url' % RMIT_SCHEMA,
input_storage_name)
# store offsets
setval(run_settings,
'%s/platform/storage/input/offset' % RMIT_SCHEMA,
input_storage_offset)
# TODO: replace with scratch space computation platform space
self.scratch_platform = '%s%s%s' % (
manage.get_scratch_platform(), sweep_name,
contextid)
# mytardis
if output_loc:
try:
self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA))
except KeyError:
self.experiment_id = 0
except ValueError:
self.experiment_id = 0
try:
curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA)
except SettingNotFoundException:
curate_data = False
if curate_data:
self.experiment_id = self.curate_data(run_settings, location, self.experiment_id)
setval(run_settings,
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA,
str(self.experiment_id))
# generate all variations
map_text = getval(run_settings, '%s/input/sweep/sweep_map' % RMIT_SCHEMA)
# map_text = run_settings[RMIT_SCHEMA + '/input/sweep']['sweep_map']
sweep_map = json.loads(map_text)
logger.debug("sweep_map=%s" % pformat(sweep_map))
runs = _expand_variations(maps=[sweep_map], values={})
logger.debug("runs=%s" % runs)
# Create random numbers if needed
# TODO: move iseed out of hrmc into separate generic schema
# to use on any sweepable connector and make this function
# completely hrmc independent.
rands = []
try:
self.rand_index = getval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA)
logger.debug("rand_index=%s" % self.rand_index)
except SettingNotFoundException:
pass
else:
# prep random seeds for each run based off original iseed
# FIXME: inefficient for large random file
# TODO, FIXME: this is potentially problematic if different
# runs end up overlapping in the random numbers they utilise.
# solution is to have separate random files per run or partition
# big file up.
try:
num_url = getval(run_settings, "%s/system/random_numbers" % RMIT_SCHEMA)
logger.debug('num_url=%s' % num_url)
except SettingNotFoundException:
pass
else:
try:
local_settings['random_numbers'] = num_url
rands = generate_rands(settings=local_settings,
start_range=0,
end_range=-1,
num_required=len(runs),
start_index=self.rand_index)
logger.debug("rands=%s" % rands)
except Exception, e:
logger.debug('error')
logger.error(e)
raise
def output(self, run_settings):
"""
Output new runs_left value (including zero value)
"""
logger.debug("finished stage output")
executing_procs = [x for x in self.current_processes if x['status'] != 'ready']
#nodes_working = len(self.executed_procs) - (len(self.finished_nodes) + self.failed_processes)
nodes_working = len(executing_procs) - (len(self.finished_nodes) + self.failed_processes)
#if len(self.finished_nodes) == 0 and self.failed_processes == 0:
# nodes_working = 0
logger.debug("%d %d " %(len(self.finished_nodes), self.failed_processes))
#logger.debug("self.executed_procs=%s" % self.executed_procs)
logger.debug("self.executed_procs=%s" % executing_procs)
logger.debug("self.finished_nodes=%s" % self.finished_nodes)
#FIXME: nodes_working can be negative
# FIXME: possible race condition?
# if not self._exists(run_settings, 'http://rmit.edu.au/schemas/stages/run'):
# run_settings['http://rmit.edu.au/schemas/stages/run'] = {}
# #run_settings['http://rmit.edu.au/schemas/stages/run']['runs_left'] = nodes_working
#run_settings['http://rmit.edu.au/schemas/stages/run']['error_nodes'] = nodes_working
setvals(run_settings, {
'%s/stages/run/runs_left' % RMIT_SCHEMA: nodes_working,
'%s/stages/run/error_nodes' % RMIT_SCHEMA: nodes_working,
})
if not nodes_working:
self.finished_nodes = []
#messages.success(run_settings, '%d: All processes in current iteration completed' % (self.id + 1))
#run_settings['http://rmit.edu.au/schemas/stages/run']['finished_nodes'] = str(self.finished_nodes)
setvals(run_settings, {
'%s/stages/run/finished_nodes' % RMIT_SCHEMA: str(self.finished_nodes),
'%s/stages/schedule/all_processes' % RMIT_SCHEMA: str(self.all_processes),
'%s/stages/schedule/current_processes' % RMIT_SCHEMA: str(self.current_processes),
'%s/stages/execute/executed_procs' % RMIT_SCHEMA: str(self.executed_procs)
})
if self.failed_nodes:
setval(run_settings, '%s/stages/create/failed_nodes' % RMIT_SCHEMA, self.failed_nodes)
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/create', {})[u'failed_nodes'] = self.failed_nodes
completed_procs = [x for x in self.executed_procs if x['status'] == 'completed']
# messages.info(run_settings, "%s: waiting (%s of %s processes done)"
# % (self.id + 1, len(completed_procs), len(self.current_processes) - (self.failed_processes)))
if self.procs_2b_rescheduled:
setvals(run_settings, {
'%s/stages/schedule/scheduled_started' % RMIT_SCHEMA: 0,
'%s/stages/schedule/procs_2b_rescheduled' % RMIT_SCHEMA: self.procs_2b_rescheduled
})
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'schedule_started'] = 0
# run_settings.setdefault(
# 'http://rmit.edu.au/schemas/stages/schedule',
# {})[u'procs_2b_rescheduled'] = self.procs_2b_rescheduled
return run_settings
def output(self, run_settings):
setval(
run_settings,
'%s/input/mytardis/experiment_id' % django_settings.SCHEMA_PREFIX,
str(self.experiment_id))
if not self.done_iterating:
# trigger first of iteration corestages
logger.debug("nonconvergence")
setvals(
run_settings, {
'%s/stages/schedule/scheduled_nodes' % django_settings.SCHEMA_PREFIX:
'[]',
'%s/stages/execute/executed_procs' % django_settings.SCHEMA_PREFIX:
'[]',
'%s/stages/schedule/current_processes' % django_settings.SCHEMA_PREFIX:
'[]',
'%s/stages/schedule/total_scheduled_procs' % django_settings.SCHEMA_PREFIX:
0,
'%s/stages/schedule/schedule_completed' % django_settings.SCHEMA_PREFIX:
0,
'%s/stages/schedule/schedule_started' % django_settings.SCHEMA_PREFIX:
0
})
logger.debug('scheduled_nodes=%s' % getval(
run_settings, '%s/stages/schedule/scheduled_nodes' %
django_settings.SCHEMA_PREFIX))
try:
delkey(
run_settings,
'%s/stages/run/runs_left' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
try:
delkey(
run_settings, '%s/stages/run/error_nodes' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
# run = run_settings['%s/stages/run' % django_settings.SCHEMA_PREFIX]
# del run['error_nodes']
#update_key('converged', False, context)
setval(
run_settings,
'%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX,
0)
# run_settings['%s/stages/converge' % django_settings.SCHEMA_PREFIX][u'converged'] = 0
# delete_key('runs_left', context)
# delete_key('error_nodes', context)
# update_key('converged', False, context)
else:
logger.debug("convergence")
# we are done, so trigger next stage outside of converge
#update_key('converged', True, context)
setval(
run_settings,
'%s/stages/converge/converged' % django_settings.SCHEMA_PREFIX,
1)
# we are done, so don't trigger iteration stages
#update_key('criterion', self.criterion, context)
setval(run_settings,
'%s/stages/converge/criterion' % django_settings.SCHEMA_PREFIX,
unicode(self.criterion))
# delete_key('error_nodes', context)
#delete_key('transformed', context)
try:
delkey(
run_settings, '%s/stages/transform/transformed' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
pass
self.id += 1
# update_key('id', self.id, context)
setval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX,
self.id)
return run_settings
