if 'pottype' in local_settings:
logger.debug("pottype=%s" % local_settings['pottype'])
try:
pottype = int(local_settings['pottype'])
except ValueError:
logger.error("cannot convert %s to pottype" %
local_settings['pottype'])
pottype = 0
else:
pottype = 0
optimisation_scheme = local_settings['optimisation_scheme']
if optimisation_scheme == 'MC':
N = local_settings['fanout_per_kept_result']
rand_nums = jobs.generate_rands(local_settings, 0,
local_settings['max_seed_int'], N,
rand_index)
rand_index += N
map = {
'temp': [300],
'iseed': rand_nums,
'istart': [1 if id > 0 else 2],
'pottype': [pottype]
}
elif optimisation_scheme == 'MCSA':
threshold = local_settings['threshold']
logger.debug("threshold=%s" % threshold)
N = int(ast.literal_eval(threshold)[0])
logger.debug("N=%s" % N)
if not id:
class Sweep(Stage):
def __init__(self, user_settings=None):
self.numbfile = 0
logger.debug("Sweep stage initialized")
def input_valid(self, settings_to_test):
#fixme: move to hrmc
with ignored(SettingNotFoundException, ValueError):
iseed = int(
getval(settings_to_test,
'%s/input/hrmc/iseed' % django_settings.SCHEMA_PREFIX))
NUMBER_SEEDS = 10000 #fixme: should be length of no lines on random_number file
if not iseed in range(0, NUMBER_SEEDS):
return (False,
'Random Number Seed should be in range (0, %d)' %
(NUMBER_SEEDS - 1))
return (True, 'valid input')
def is_triggered(self, run_settings):
logger.debug('run_settings=%s' % run_settings)
try:
configure_done = int(
getval(
run_settings, '%s/stages/sweep/sweep_done' %
django_settings.SCHEMA_PREFIX))
except (ValueError, SettingNotFoundException):
return True
return not configure_done
def _get_sweep_name(self, run_settings):
try:
return getval(
run_settings, '%s/directive_profile/sweep_name' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
return 'unknown_sweep'
def process(self, run_settings):
logger.debug('run_settings=%s' % run_settings)
messages.info(run_settings, "0: sweep started")
# Need to make copy because we pass on run_settings to sub connector
# so any changes we make here to run_settings WILL be inherited
def make_local_settings(run_settings):
from copy import deepcopy
local_settings = deepcopy(
getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS))
update(
local_settings,
run_settings,
django_settings.SCHEMA_PREFIX + '/system/platform',
# django_settings.SCHEMA_PREFIX + '/input/mytardis/experiment_id',
# django_settings.SCHEMA_PREFIX + '/system/random_numbers',
)
local_settings['bdp_username'] = getval(
run_settings,
'%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX)
return local_settings
local_settings = make_local_settings(run_settings)
logger.debug('local_settings=%s' % local_settings)
compplatform = [
k for k, v in run_settings.iteritems()
if k.startswith('%s/input/system/compplatform' %
django_settings.SCHEMA_PREFIX)
]
setval(
run_settings, '%s/platform/computation/platform_url' %
django_settings.SCHEMA_PREFIX,
getval(run_settings, '%s/computation_platform' % compplatform[0]))
def _parse_output_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return name, offset
contextid = int(
getval(run_settings,
'%s/system/contextid' % django_settings.SCHEMA_PREFIX))
logger.debug("contextid=%s" % contextid)
sweep_name = self._get_sweep_name(run_settings)
logger.debug("sweep_name=%s" % sweep_name)
output_loc = self.output_exists(run_settings)
location = ""
if output_loc:
location = getval(run_settings, output_loc)
output_storage_name, output_storage_offset = \
_parse_output_location(run_settings, location)
setval(
run_settings, '%s/platform/storage/output/platform_url' %
django_settings.SCHEMA_PREFIX, output_storage_name)
setval(
run_settings, '%s/platform/storage/output/offset' %
django_settings.SCHEMA_PREFIX,
os.path.join(output_storage_offset,
'%s%s' % (sweep_name, contextid)))
def _parse_input_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return (name, offset)
input_loc = self.input_exists(run_settings)
logger.debug('special_input_loc=%s' % input_loc)
if input_loc:
location = getval(run_settings, input_loc)
input_storage_name, input_storage_offset = \
_parse_input_location(run_settings, location)
setval(
run_settings, '%s/platform/storage/input/platform_url' %
django_settings.SCHEMA_PREFIX, input_storage_name)
# store offsets
setval(
run_settings, '%s/platform/storage/input/offset' %
django_settings.SCHEMA_PREFIX, input_storage_offset)
# TODO: replace with scratch space computation platform space
self.scratch_platform = '%s%s%s' % (manage.get_scratch_platform(),
sweep_name, contextid)
# mytardis
if output_loc:
try:
self.experiment_id = int(
getval(
run_settings, '%s/input/mytardis/experiment_id' %
django_settings.SCHEMA_PREFIX))
except KeyError, ValueError:
self.experiment_id = 0
try:
curate_data = getval(
run_settings, '%s/input/mytardis/curate_data' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
curate_data = False
curate_data = False #TODO remove
if curate_data:
logger.debug('location=%s' % location)
location = "%s%s" % (sweep_name, contextid)
self.experiment_id = self.curate_data(run_settings, location,
self.experiment_id)
setval(
run_settings, '%s/input/mytardis/experiment_id' %
django_settings.SCHEMA_PREFIX, str(self.experiment_id))
# generate all variations
map_text = getval(
run_settings,
'%s/input/sweep/sweep_map' % django_settings.SCHEMA_PREFIX)
if not map_text:
map_text = {}
# map_text = run_settings[django_settings.SCHEMA_PREFIX + '/input/sweep']['sweep_map']
sweep_map = json.loads(map_text)
logger.debug("sweep_map=%s" % pformat(sweep_map))
runs = _expand_variations(maps=[sweep_map], values={})
logger.debug("runs=%s" % runs)
# Create random numbers if needed
# TODO: move iseed out of hrmc into separate generic schema
# to use on any sweepable connector and make this function
# completely hrmc independent.
rands = []
try:
self.rand_index = getval(
run_settings,
'%s/input/hrmc/iseed' % django_settings.SCHEMA_PREFIX)
logger.debug("rand_index=%s" % self.rand_index)
except SettingNotFoundException:
pass
else:
# prep random seeds for each run based off original iseed
# FIXME: inefficient for large random file
# TODO, FIXME: this is potentially problematic if different
# runs end up overlapping in the random numbers they utilise.
# solution is to have separate random files per run or partition
# big file up.
try:
num_url = getval(
run_settings,
"%s/system/random_numbers" % django_settings.SCHEMA_PREFIX)
logger.debug('num_url=%s' % num_url)
except SettingNotFoundException:
pass
else:
try:
local_settings['random_numbers'] = num_url
rands = generate_rands(settings=local_settings,
start_range=0,
end_range=-1,
num_required=len(runs),
start_index=self.rand_index)
logger.debug("rands=%s" % rands)
except Exception, e:
logger.debug('error')
logger.error(e)
raise
# variations map spectification
if 'pottype' in local_settings:
logger.debug("pottype=%s" % local_settings['pottype'])
try:
pottype = int(local_settings['pottype'])
except ValueError:
logger.error("cannot convert %s to pottype" % local_settings['pottype'])
pottype = 0
else:
pottype = 0
optimisation_scheme = local_settings['optimisation_scheme']
if optimisation_scheme == 'MC':
N = local_settings['fanout_per_kept_result']
rand_nums = jobs.generate_rands(local_settings,
0, local_settings['max_seed_int'],
N, rand_index)
rand_index += N
map = {
'temp': [300],
'iseed': rand_nums,
'istart': [1 if id > 0 else 2],
'pottype': [pottype]
}
elif optimisation_scheme == 'MCSA':
threshold = local_settings['threshold']
logger.debug("threshold=%s" % threshold)
N = int(ast.literal_eval(threshold)[0])
logger.debug("N=%s" % N)
if not id:
def process(self, run_settings):
logger.debug('run_settings=%s' % run_settings)
# Need to make copy because we pass on run_settings to sub connector
# so any changes we make here to run_settings WILL be inherited
def make_local_settings(run_settings):
from copy import deepcopy
local_settings = deepcopy(getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS))
update(local_settings, run_settings,
RMIT_SCHEMA + '/system/platform',
# RMIT_SCHEMA + '/input/mytardis/experiment_id',
# RMIT_SCHEMA + '/system/random_numbers',
)
local_settings['bdp_username'] = getval(
run_settings, '%s/bdp_userprofile/username' % RMIT_SCHEMA)
return local_settings
local_settings = make_local_settings(run_settings)
logger.debug('local_settings=%s' % local_settings)
setval(run_settings,
'%s/platform/computation/platform_url' % RMIT_SCHEMA,
getval(run_settings,
'%s/input/system/compplatform/computation_platform'
% RMIT_SCHEMA))
def _parse_output_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return name, offset
contextid = int(getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA))
logger.debug("contextid=%s" % contextid)
sweep_name = self._get_sweep_name(run_settings)
logger.debug("sweep_name=%s" % sweep_name)
output_loc = self.output_exists(run_settings)
location = ""
if output_loc:
location = getval(run_settings, output_loc)
output_storage_name, output_storage_offset = \
_parse_output_location(run_settings, location)
setval(run_settings,
'%s/platform/storage/output/platform_url' % RMIT_SCHEMA,
output_storage_name)
setval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA,
os.path.join(output_storage_offset, '%s%s' % (sweep_name, contextid)))
def _parse_input_location(run_settings, location):
loc_list = location.split('/')
name = loc_list[0]
offset = ''
if len(loc_list) > 1:
offset = os.path.join(*loc_list[1:])
logger.debug('offset=%s' % offset)
return (name, offset)
input_loc = self.input_exists(run_settings)
if input_loc:
location = getval(run_settings, input_loc)
input_storage_name, input_storage_offset = \
_parse_input_location(run_settings, location)
setval(run_settings, '%s/platform/storage/input/platform_url' % RMIT_SCHEMA,
input_storage_name)
# store offsets
setval(run_settings,
'%s/platform/storage/input/offset' % RMIT_SCHEMA,
input_storage_offset)
# TODO: replace with scratch space computation platform space
self.scratch_platform = '%s%s%s' % (
manage.get_scratch_platform(), sweep_name,
contextid)
# mytardis
if output_loc:
try:
self.experiment_id = int(getval(run_settings, '%s/input/mytardis/experiment_id' % RMIT_SCHEMA))
except KeyError:
self.experiment_id = 0
except ValueError:
self.experiment_id = 0
try:
curate_data = getval(run_settings, '%s/input/mytardis/curate_data' % RMIT_SCHEMA)
except SettingNotFoundException:
curate_data = False
if curate_data:
self.experiment_id = self.curate_data(run_settings, location, self.experiment_id)
setval(run_settings,
'%s/input/mytardis/experiment_id' % RMIT_SCHEMA,
str(self.experiment_id))
# generate all variations
map_text = getval(run_settings, '%s/input/sweep/sweep_map' % RMIT_SCHEMA)
# map_text = run_settings[RMIT_SCHEMA + '/input/sweep']['sweep_map']
sweep_map = json.loads(map_text)
logger.debug("sweep_map=%s" % pformat(sweep_map))
runs = _expand_variations(maps=[sweep_map], values={})
logger.debug("runs=%s" % runs)
# Create random numbers if needed
# TODO: move iseed out of hrmc into separate generic schema
# to use on any sweepable connector and make this function
# completely hrmc independent.
rands = []
try:
self.rand_index = getval(run_settings, '%s/input/hrmc/iseed' % RMIT_SCHEMA)
logger.debug("rand_index=%s" % self.rand_index)
except SettingNotFoundException:
pass
else:
# prep random seeds for each run based off original iseed
# FIXME: inefficient for large random file
# TODO, FIXME: this is potentially problematic if different
# runs end up overlapping in the random numbers they utilise.
# solution is to have separate random files per run or partition
# big file up.
try:
num_url = getval(run_settings, "%s/system/random_numbers" % RMIT_SCHEMA)
logger.debug('num_url=%s' % num_url)
except SettingNotFoundException:
pass
else:
try:
local_settings['random_numbers'] = num_url
rands = generate_rands(settings=local_settings,
start_range=0,
end_range=-1,
num_required=len(runs),
start_index=self.rand_index)
logger.debug("rands=%s" % rands)
except Exception, e:
logger.debug('error')
logger.error(e)
raise