def merge_description (self, source) :
"""
merge additional information into the unit description -- such as
resource information, or application specific data
"""
# print 'merging unit %s' % self.id
# print ' with %s' % source
# print ' and %s' % self.as_dict()
# we only allow this in DESCRIBED or BOUND state
if not self.state in [DESCRIBED, BOUND] :
raise RuntimeError ('unit is not in DESCRIBED state (%s)' \
% self.state)
ud_dict = self.as_dict ()
ru.dict_merge (ud_dict, source, policy='overwrite')
ru.dict_stringexpand (ud_dict)
ru.dict_stringexpand (ud_dict, self.session.cfg)
# print '-------------'
# import pprint
# pprint.pprint (ud_dict)
# print '-------------'
# pprint.pprint (self.session.cfg)
# print '-------------'
# exit()
for (key, val) in ud_dict.iteritems () :
try :
self.set_attribute (key, val)
except :
pass
def expand_description (self, session) :
# This will apply any application configuration wildcards we find in the
# session config.
td_dict = self.as_dict()
print td_dict
ru.dict_stringexpand (td_dict, session.cfg)
print td_dict
# we need to re-initialize our properties with the expanded dict values
tu.Properties.__init__ (self, td_dict)
def test_dict_stringexpand () :
target = {'workdir' : '%(home)s/work/',
'resource' : '%(resource)s'}
source = {'user' : 'peer_gynt',
'protocol' : 'ssh',
'host' : 'localhost',
'home' : '/home/%(user)s',
'resource' : '%(protocol)s://%(host)s/'}
ru.dict_stringexpand (target, source)
assert (target.keys() == ['workdir', 'resource'])
assert (target['workdir'] == '/home/peer_gynt/work/')
assert (target['resource'] == 'ssh://localhost/')
def merge_description (self, source) :
"""
merge additional information into the pilot description -- such as
resource information, or application specific data
"""
# we only allow this in DESCRIBED or BOUND state
if not self.state in [DESCRIBED, BOUND] :
raise RuntimeError ('pilot is not in DESCRIBED state (%s)' \
% self.state)
pd_dict = self.description.as_dict ()
ru.dict_merge (pd_dict, source, policy='overwrite')
ru.dict_stringexpand (pd_dict)
ru.dict_stringexpand (pd_dict, self.session.cfg)
self.description = troy.PilotDescription (pd_dict)
def expand_workload (self, workload):
"""
Inspect all workload tasks, check if cardinality is set and larger than
1. If so, created that many identical tasks, assign them an `cardinal`
index, and have that set replace the original task. All new tasks
will have a cardinality of 1.
"""
task_descriptions = list()
for task_id in workload.tasks :
task = workload.tasks[task_id]
task_dict = task.as_dict ()
# make sure all known config vars are expanded
ru.dict_stringexpand (task_dict, self.session.cfg)
if 'cardinality' in task_dict :
cardinality = int(task_dict['cardinality'])
for c in range(cardinality) :
new_task_dict = copy.deepcopy(task_dict)
new_task_dict['cardinality'] = 1 # avoid repeated expansion
new_task_dict['cardinal'] = c
task_descriptions.append (troy.TaskDescription (new_task_dict))
else :
task_descriptions.append (task.as_dict ())
# remove old tasks
workload.tasks = dict()
# and add fresh ones
workload.add_task (task_descriptions)
troy._logger.info ("planner expand wl cardinality: %s" % workload)
def parse (self, workload_description):
"""
Parse the given json into task and relation descriptions, and create
a workload out of them.
"""
troy._logger.info ("parsing workload")
tasks = None
relations = None
workload_dict = ru.parse_json (workload_description)
if 'tasks' in workload_dict :
tasks = workload_dict['tasks']
if 'relations' in workload_dict :
relations = workload_dict['relations']
if not tasks and not relations :
raise ValueError ("Cannot handle workload description")
task_descriptions = list()
relation_descriptions = list()
if tasks :
for task_descr in tasks :
# make sure we use all current information
ru.dict_stringexpand (task_descr, self.session.cfg)
task_descriptions.append (troy.TaskDescription (task_descr))
if relations :
for relation_descr in relations :
# make sure we use all current information
ru.dict_stringexpand (relation_descr, self.session.cfg)
relation_descriptions.append (troy.RelationDescription (relation_descr))
return task_descriptions, relation_descriptions
def get_config (params) :
"""
This method attempts to obtain configuration settings from a variety of
sources, depending on the parameter. it can point to an env var, or to
a directory containing configuration files, or to a single configuration
file, or to a list of any above, or it is a config dict already, or a list
of such dicts. In all cases, the config is obtained from the respective
source (which is assumed json formatted in the case of config files), and
a single merged and expanded dict is returned.
"""
ret = dict()
# always make params list for simpler code below
if not isinstance(params, list) :
params = [params]
for param in params :
if not param or None == param :
# we silently accept None's, to save some
# repetetetetive checks on the calling side
continue
elif isinstance (param, dict) :
# simply merge it into the result
ru.dict_merge (ret, param, policy='overwrite')
elif isinstance (param, basestring) :
# check if the string points to an env variable
if param in os.environ :
# assume that the value of the env var is what we really want
param = os.environ[param]
# is string, is not env, must be a dir or a file
if os.path.isdir (param) :
# config dir
cfg_files = glob.glob ("%s/*" % param)
# print 'is dir %s/*' % param
# print cfg_files
elif os.path.isfile (param) :
# single config file
cfg_files = [param]
else :
troy._logger.warning ("cannot handle config location %s" % param)
cfg_files = list()
print 'files: %s' % cfg_files
# read and merge all config files
for cfg_file in cfg_files :
cfg_dict = dict()
try :
cfg_dict = ru.read_json (cfg_file)
troy._logger.info ("reading config in %s" % cfg_file)
except Exception as e :
troy._logger.critical ("skipping config in %s (%s)" % (cfg_file, e))
raise
# import pprint
# print '================'
# print cfg_file
# pprint.pprint (cfg_dict)
# print '================'
ru.dict_merge (ret, cfg_dict, policy='overwrite')
else :
raise TypeError ("get_config parameter must be (list of) dict or "
"string, not %s" % type(param))
# print '================================'
# pprint.pprint (ret)
# print '================================'
# expand config(s) before returning
ru.dict_stringexpand (ret)
return ret