def merge_description (self, source) :
        """
        merge additional information into the unit description -- such as
        resource information, or application specific data
        """

      # print 'merging  unit %s' % self.id
      # print '         with %s' % source
      # print '          and %s' % self.as_dict()

        # we only allow this in DESCRIBED or BOUND state
        if  not self.state in [DESCRIBED, BOUND] :
            raise RuntimeError ('unit is not in DESCRIBED state (%s)' \
                             % self.state)

        ud_dict = self.as_dict ()

        ru.dict_merge        (ud_dict, source, policy='overwrite')
        ru.dict_stringexpand (ud_dict)
        ru.dict_stringexpand (ud_dict, self.session.cfg)

      # print '-------------'
      # import pprint
      # pprint.pprint (ud_dict)
      # print '-------------'
      # pprint.pprint (self.session.cfg)
      # print '-------------'
      # exit()

        for (key, val) in ud_dict.iteritems () :
            try :
                self.set_attribute (key, val)
            except :
                pass
    def expand_description (self, session) :

        # This will apply any application configuration wildcards we find in the
        # session config.
        
        td_dict = self.as_dict()
        print td_dict
        ru.dict_stringexpand (td_dict, session.cfg)
        print td_dict

        # we need to re-initialize our properties with the expanded dict values
        tu.Properties.__init__ (self, td_dict)
Example #3
0
def test_dict_stringexpand () :

    target = {'workdir'  : '%(home)s/work/',
              'resource' : '%(resource)s'}
    source = {'user'     : 'peer_gynt',
              'protocol' : 'ssh',
              'host'     : 'localhost',
              'home'     : '/home/%(user)s', 
              'resource' : '%(protocol)s://%(host)s/'}

    ru.dict_stringexpand (target, source)

    assert (target.keys()      == ['workdir', 'resource'])
    assert (target['workdir']  == '/home/peer_gynt/work/')
    assert (target['resource'] == 'ssh://localhost/')
Example #4
0
    def merge_description (self, source) :
        """
        merge additional information into the pilot description -- such as
        resource information, or application specific data
        """

        # we only allow this in DESCRIBED or BOUND state
        if  not self.state in [DESCRIBED, BOUND] :
            raise RuntimeError ('pilot is not in DESCRIBED state (%s)' \
                             % self.state)

        pd_dict = self.description.as_dict ()
        ru.dict_merge        (pd_dict, source, policy='overwrite')
        ru.dict_stringexpand (pd_dict)
        ru.dict_stringexpand (pd_dict, self.session.cfg)

        self.description = troy.PilotDescription (pd_dict)
    def expand_workload (self, workload):
        """
        Inspect all workload tasks, check if cardinality is set and larger than
        1.  If so, created that many identical tasks, assign them an `cardinal`
        index, and have that set replace the original task.  All new tasks
        will have a cardinality of 1.
        """

        task_descriptions = list()

        for task_id in workload.tasks :

            task = workload.tasks[task_id]
            task_dict = task.as_dict ()

            # make sure all known config vars are expanded
            ru.dict_stringexpand (task_dict, self.session.cfg)

            if  'cardinality' in task_dict :
                cardinality = int(task_dict['cardinality'])

                for c in range(cardinality) :

                    new_task_dict = copy.deepcopy(task_dict)
                    new_task_dict['cardinality'] = 1 # avoid repeated expansion
                    new_task_dict['cardinal']    = c
                    task_descriptions.append (troy.TaskDescription (new_task_dict))

            else :
                task_descriptions.append (task.as_dict ())

        # remove old tasks
        workload.tasks = dict()

        # and add fresh ones
        workload.add_task (task_descriptions)

        troy._logger.info ("planner  expand wl cardinality: %s" % workload)
    def parse (self, workload_description):
        """
        Parse the given json into  task and relation descriptions, and create
        a workload out of them.
        """

        troy._logger.info ("parsing workload")

        tasks     = None
        relations = None

        workload_dict = ru.parse_json (workload_description)

        if 'tasks'     in workload_dict : 
            tasks      =  workload_dict['tasks']
        if 'relations' in workload_dict : 
            relations  =  workload_dict['relations']

        if  not tasks and not relations :
            raise ValueError ("Cannot handle workload description")

        task_descriptions     = list()
        relation_descriptions = list()

        if tasks :
            for task_descr in tasks :
                # make sure we use all current information
                ru.dict_stringexpand (task_descr, self.session.cfg)
                task_descriptions.append (troy.TaskDescription (task_descr))

        if relations :
            for relation_descr in relations :
                # make sure we use all current information
                ru.dict_stringexpand (relation_descr, self.session.cfg)
                relation_descriptions.append (troy.RelationDescription (relation_descr))

        return task_descriptions, relation_descriptions
Example #7
0
def get_config (params) :
    """
    This method attempts to obtain configuration settings from a variety of
    sources, depending on the parameter. it can point to an env var, or to
    a directory containing configuration files, or to a single configuration
    file, or to a list of any above, or it is a config dict already, or a list
    of such dicts.  In all cases, the config is obtained from the respective
    source (which is assumed json formatted in the case of config files), and
    a single merged and expanded dict is returned.
    """


    ret = dict()

    # always make params list for simpler code below
    if  not isinstance(params, list) :
        params = [params]


    for param in params :

        if  not param or None == param : 

            # we silently accept None's, to save some 
            # repetetetetive checks on the calling side
            continue


        elif isinstance (param, dict) :

            # simply merge it into the result
            ru.dict_merge (ret, param, policy='overwrite')


        elif isinstance (param, basestring) :
        
            # check if the string points to an env variable
            if  param in os.environ :
                # assume that the value of the env var is what we really want
                param = os.environ[param]

            # is string, is not env, must be a dir or a file
            if  os.path.isdir (param) :
                # config dir
                cfg_files = glob.glob ("%s/*" % param)
              # print 'is dir %s/*' % param
              # print cfg_files

            elif os.path.isfile (param) :
                # single config file
                cfg_files = [param]

            else :
                troy._logger.warning ("cannot handle config location %s" % param)
                cfg_files = list()

            print 'files: %s' % cfg_files
            # read and merge all config files
            for cfg_file in cfg_files :
                cfg_dict = dict()
                try :
                    cfg_dict = ru.read_json (cfg_file)
                    troy._logger.info ("reading  config in %s" % cfg_file)
                except Exception as e :
                    troy._logger.critical ("skipping config in %s (%s)" % (cfg_file, e))
                    raise

              # import pprint
              # print '================'
              # print cfg_file
              # pprint.pprint (cfg_dict)
              # print '================'

                ru.dict_merge (ret, cfg_dict, policy='overwrite')



        else :
            raise TypeError ("get_config parameter must be (list of) dict or "
                             "string, not %s" % type(param))

  # print '================================'
  # pprint.pprint (ret)
  # print '================================'

    # expand config(s) before returning
    ru.dict_stringexpand (ret)

    return ret