Exemple #1
0
def connect_to_wallaby(broker_addr='127.0.0.1', port=5672, username='', passwd='', mechanisms='ANONYMOUS PLAIN GSSAPI'):
    global supported_api_versions

    # set up session for wallaby
    session = Session()

    if username != '' and passwd != '':
        broker_str = '%s/%s@%s:%d' % (username, passwd, broker_addr, port)
    elif username != '':
        broker_str = '%s@%s:%d' % (username, broker_addr, port)
    else:
        broker_str = '%s:%d' % (broker_addr, port)

    sys.stdout.write("Connecting to broker %s:\n" % (broker_str))

    try:
        broker = session.addBroker('amqp://%s' % broker_str, mechanisms=mechanisms)
    except:
        sys.stderr.write('Unable to connect to broker "%s"\n' % broker_str)
        raise

    # Retrieve the config store object
    sys.stdout.write("Connecting to wallaby store:\n")
    try:
        (store_agent, config_store) = WallabyHelpers.get_store_objs(session)
    except WallabyStoreError, error:
        sys.stderr.write('Error: %s\n' % error.error_str)
        session.delBroker(broker)
        raise
Exemple #2
0
    def build_accounting_group_feature(self, feature_name, group_tuple_list):
        sys.stdout.write("building acct group feature %s\n"%(feature_name))
        sys.stdout.flush()
        
        self.assert_feature(feature_name)

        params = {}
        params["GROUP_NAMES"] = ",".join([x[0] for x in group_tuple_list])
        
        for tuple in group_tuple_list:
            (name, is_static, quota, accept_surplus) = tuple
            if is_static: params["GROUP_QUOTA_%s"%(name)] = "%d"%(int(quota))
            else:         params["GROUP_QUOTA_DYNAMIC_%s"%(name)] = "%f"%(float(quota))
            if accept_surplus: params["GROUP_AUTOREGROUP_%s"%(name)] = "TRUE"

        # make sure parameters are declared
        splist = params.keys()
        splist.sort()
        for p in splist: self.assert_param(p)

        feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
        result = feat_obj.modifyParams('replace', params, {})
        if result.status != 0:
            sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
            raise WallabyStoreError("Failed to add feature")
Exemple #3
0
    def build_access_feature(self, feature_name, collector_host=None, condor_host=None):
        if condor_host==None: condor_host = self.params.collector_addr
        if collector_host==None: collector_host = condor_host
        sys.stdout.write("building access feature %s\n"%(feature_name))
        sys.stdout.flush()

        self.assert_feature(feature_name)

        params={}
        params["CONDOR_HOST"] = condor_host
        params["COLLECTOR_HOST"] = collector_host
        params["ALLOW_WRITE"] = "*"
        params["ALLOW_READ"] = "*"
        params["SEC_DEFAULT_AUTHENTICATION_METHODS"] = "CLAIMTOBE"

        # make sure parameters are declared
        splist = params.keys()
        splist.sort()
        for p in splist: self.assert_param(p)

        feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
        result = feat_obj.modifyParams('replace', params, {})
        if result.status != 0:
            sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
            raise WallabyStoreError("Failed to add feature")
Exemple #4
0
 def assert_node_groups(self, group_names, node_names, mod_op='replace'):
     # apply the groups to the nodes
     for name in node_names:
         node_obj = WallabyHelpers.get_node(self.session, self.config_store, name)
         result = node_obj.modifyMemberships(mod_op, group_names, {})
         if result.status != 0:
             sys.stderr.write("Failed to set groups for %s: (%d, %s)\n" % (name, result.status, result.text))
             raise WallabyStoreError(result.text)
Exemple #5
0
    def assert_node_features(self, feature_names, node_names, mod_op='replace'):
        for feat in feature_names:
            if not feat in self.feat_names:
                emsg = "Feature %s not in config store" % (feat)
                raise Exception(emsg)

        # apply feature list to nodes
        for name in node_names:
            node_obj = WallabyHelpers.get_node(self.session, self.config_store, name)
            group_name = WallabyHelpers.get_id_group_name(node_obj, self.session)
            group_obj = WallabyHelpers.get_group(self.session, self.config_store, group_name)
            if mod_op == 'insert':
                result = group_obj.modifyFeatures('replace', feature_names + group_obj.features, {})
            else:
                result = group_obj.modifyFeatures(mod_op, feature_names, {})
            if result.status != 0:
                sys.stderr.write("Failed to set features for %s: (%d, %s)\n" % (name, result.status, result.text))
                raise WallabyStoreError(result.text)
Exemple #6
0
    def clear_nodes(self, node_names):
        for name in node_names:
            node_obj = WallabyHelpers.get_node(self.session, self.config_store, name)
            result = node_obj.modifyMemberships('replace', [], {})
            if result.status != 0:
                sys.stderr.write("Failed to clear groups from %s: (%d, %s)\n" % (name, result.status, result.text))
                raise WallabyStoreError("Failed to clear groups")

            group_name = WallabyHelpers.get_id_group_name(node_obj, self.session)
            group_obj = WallabyHelpers.get_group(self.session, self.config_store, group_name)
            result = group_obj.modifyFeatures('replace', [], {})
            if result.status != 0:
                sys.stderr.write("Failed to clear features from %s: (%d, %s)\n" % (name, result.status, result.text))
                raise WallabyStoreError("Failed to clear features")

            result = group_obj.modifyParams('replace', {}, {})
            if result.status != 0:
                sys.stderr.write("Failed to clear params from %s: (%d, %s)\n" % (name, result.status, result.text))
                raise WallabyStoreError("Failed to clear params")
Exemple #7
0
    def clear_default_group(self):
            group_obj = WallabyHelpers.get_group(self.session, self.config_store, '+++DEFAULT')
            result = group_obj.modifyFeatures('replace', [], {})
            if result.status != 0:
                sys.stderr.write("Failed to clear features from %s: (%d, %s)\n" % (name, result.status, result.text))
                raise WallabyStoreError("Failed to clear features")

            result = group_obj.modifyParams('replace', {}, {})
            if result.status != 0:
                sys.stderr.write("Failed to clear params from %s: (%d, %s)\n" % (name, result.status, result.text))
                raise WallabyStoreError("Failed to clear params")
Exemple #8
0
    def list_nodes(self, with_all_feats=None, without_any_feats=None, with_all_groups=None, without_any_groups=None, checkin_since=None):
        r = []
        for node in self.node_names:
            node_obj = WallabyHelpers.get_node(self.session, self.config_store, node)

            sys.stdout.write("    list_nodes: node=%s   checkin= %s\n" % (node, node_obj.last_checkin))

            if (checkin_since != None) and ((node_obj.last_checkin / 1000000) < checkin_since): continue

            nodefeats = []
            if (with_all_feats != None) or (without_any_feats != None):
                nodefeats = WallabyHelpers.get_node_features(node_obj, self.session, self.config_store)
            if (with_all_feats != None) and (False in [x in nodefeats for x in with_all_feats]): continue
            if (without_any_feats != None) and (True in [x in nodefeats for x in without_any_feats]): continue

            nodegroups = []
            if (with_all_groups != None) or (without_any_groups != None):
                nodegroups = [WallabyHelpers.get_id_group_name(node_obj, self.session)] + node_obj.memberships + ['+++DEFAULT']
            if (with_all_groups != None) and (False in [x in nodegroups for x in with_all_groups]): continue
            if (without_any_groups != None) and (True in [x in nodegroups for x in without_any_groups]): continue

            r += [node]
        return r
Exemple #9
0
    def tag_test_feature(self, feature_name, param_name):
        # ensure parameter name exists
        self.assert_param(param_name)

        # ensure that parameter requires restart
        param_obj = WallabyHelpers.get_param(self.session, self.config_store, param_name)
        result = param_obj.setRequiresRestart(True)
        if result.status != 0:
            sys.stderr.write("Failed to set restart for %s: (%d, %s)\n" % (param_name, result.status, result.text))
            raise WallabyStoreError("Failed to set restart")

        # set this param to a new value, to ensure a restart on activation
        feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
        result = feat_obj.modifyParams('add', {param_name:("%s"%(time.time()))}, {})
        if result.status != 0:
            sys.stderr.write("Failed to add param %s to %s: (%d, %s)\n" % (param_name, feature_name, result.status, result.text))
            raise WallabyStoreError("Failed to add param")

        # make sure master is tagged for restart via this parameter
        subsys_obj = WallabyHelpers.get_subsys(self.session, self.config_store, 'master')
        result = subsys_obj.modifyParams('add', [param_name], {})
        if result.status != 0:
            sys.stderr.write("Failed to add param %s to master: (%d, %s)\n" % (param_name, result.status, result.text))
            raise WallabyStoreError("Failed to add param")
Exemple #10
0
    def build_feature(self, feature_name, params={}, mod_op='replace'):
        sys.stdout.write("building feature %s\n"%(feature_name))
        sys.stdout.flush()

        self.assert_feature(feature_name)

        # make sure parameters are declared
        splist = params.keys()
        splist.sort()
        for p in splist: self.assert_param(p)

        feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
        result = feat_obj.modifyParams(mod_op, params, {})
        if result.status != 0:
            sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
            raise WallabyStoreError("Failed to add feature")
Exemple #11
0
    def build_collector_feature(self, feature_name, n_coll=1, portstart=10000, dl_append=True, collector_host=None, condor_host=None, disable_plugins=True):
        if condor_host==None: condor_host = self.params.collector_addr
        if collector_host==None: collector_host = condor_host
        sys.stdout.write("building collector feature %s with %d sub-collectors\n"%(feature_name, n_coll))
        sys.stdout.flush()
        
        self.assert_feature(feature_name)

        collector_names = []
        params={}
        if dl_append: daemon_list = ">= "
        else:         daemon_list = "MASTER"
        for s in xrange(n_coll):
            tag = "%03d"%(s)
            port=portstart+s
            locname = "COLLECTOR%s"%(tag)
            collector_names += ["%s:%d"%(collector_host,port)]
            if (s > 0) or not dl_append: daemon_list += ","
            daemon_list += "COLLECTOR%s"%(tag)
            params["COLLECTOR%s"%(tag)] = "$(COLLECTOR)"
            params["COLLECTOR%s_ARGS"%(tag)] = "-f -p %d -local-name %s"%(port,locname)
            params["COLLECTOR%s_ENVIRONMENT"%(tag)] = "_CONDOR_COLLECTOR_LOG=$(LOG)/CollectorLog%s"%(tag)
            params["COLLECTOR.%s.COLLECTOR_NAME"%(locname)] = locname
            params["COLLECTOR.%s.CONDOR_VIEW_HOST"%(locname)] = "$(COLLECTOR_HOST)"
            params["COLLECTOR.%s.COLLECTOR_ADDRESS_FILE"%(locname)] = "$(LOG)/.collector%s-address"%(tag)
            if disable_plugins:
                params["COLLECTOR.%s.PLUGINS"%(locname)] = ""

        params["DAEMON_LIST"] = daemon_list

        # make sure parameters are declared
        splist = params.keys()
        splist.sort()
        for p in splist: self.assert_param(p)

        feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
        result = feat_obj.modifyParams('replace', params, {})
        if result.status != 0:
            sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
            raise WallabyStoreError("Failed to add feature")

        return collector_names
Exemple #12
0
    def assert_group_features(self, feature_names, group_names, mod_op='replace'):
        # ensure these actually exist
        for grp in group_names:
            if not grp in self.group_names:
                sys.stdout.write("Adding group %s to store:\n" % (grp))
                result = self.config_store.addExplicitGroup(grp)
                if result.status != 0:
                    sys.stderr.write("Failed to create group %s: (%d, %s)\n" % (grp, result.status, result.text))
                    raise WallabyStoreError(result.text)

        # In principle, could automatically install features if they aren't found
        for feat in feature_names:
            if not feat in self.feat_names: raise Exception("Feature %s not in config store" % (feat))

        # apply feature list to group
        for name in group_names:
            group_obj = WallabyHelpers.get_group(self.session, self.config_store, name)
            result = group_obj.modifyFeatures(mod_op, feature_names, {})
            if result.status != 0:
                sys.stderr.write("Failed to set features for %s: (%d, %s)\n" % (name, result.status, result.text))
                raise WallabyStoreError(result.text)
Exemple #13
0
    def build_scheduler_feature(self, feature_name, n_schedd=1, dl_append=True):
        sys.stdout.write("building scheduler feature %s with %d schedds\n"%(feature_name, n_schedd))
        sys.stdout.flush()
        
        self.assert_feature(feature_name)

        schedd_names = []
        params={}
        params["USE_PROCD"] = "FALSE"
        if dl_append: daemon_list = ">= "
        else:         daemon_list = "MASTER"
        for s in xrange(n_schedd):
            tag = "%03d"%(s)
            locname = "SCHEDD%s"%(tag)
            schedd_names += [locname]
            if (s > 0) or not dl_append: daemon_list += ","
            daemon_list += "SCHEDD%s"%(tag)
            params["SCHEDD%s"%(tag)] = "$(SCHEDD)"
            params["SCHEDD%s_ARGS"%(tag)] = "-f -local-name %s"%(locname)
            params["SCHEDD.%s.SCHEDD_NAME"%(locname)] = locname
            params["SCHEDD.%s.SCHEDD_LOG"%(locname)] = "$(LOG)/SchedLog%s"%(tag)
            params["SCHEDD.%s.SCHEDD_ADDRESS_FILE"%(locname)] = "$(LOG)/.schedd%s-address"%(tag)
            params["SCHEDD.%s.SPOOL"%(locname)] = "$(SPOOL).%s"%(tag)
            params["SCHEDD.%s.HISTORY"%(locname)] = "$(SPOOL)/history.s%s"%(tag)
            #params["SCHEDD.%s.SCHEDD_DAEMON_AD_FILE"%(locname)] = "$(LOG)/.schedd%s-daemon-ad"%(tag)

        params["DAEMON_LIST"] = daemon_list

        # make sure parameters are declared
        splist = params.keys()
        splist.sort()
        for p in splist: self.assert_param(p)

        feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
        result = feat_obj.modifyParams('replace', params, {})
        if result.status != 0:
            sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
            raise WallabyStoreError("Failed to add feature")

        return schedd_names
   env['CONDOR_CONFIG'] = '/etc/condor/condor_config'
   env['_CONDOR_LOCAL_CONFIG_DIR'] = config_dir
   env['_CONDOR_CONFIGD_CHECK_INTERVAL'] = str(checkin_time)
   env['_CONDOR_QMF_BROKER_HOST'] = '127.0.0.1'
   env['_CONDOR_QMF_BROKER_PORT'] = '5672'
   env['_CONDOR_CONFIGD_OVERRIDE_DIR'] = override_dir
   env['_CONDOR_LOG'] = '.'
   env['PATH'] = os.environ['PATH']
   cmd = shlex.split('../condor_configd -d -l %s -m %s -h %s' % (log_file, config_file, nodename))
   os.execve(cmd[0], cmd, env)
   sys.exit(0)

# Setup the connection to the store
store = []
try:
   (agent, store) = WallabyHelpers.get_store_objs(session)
except Exception, error:
   print 'Error: %s' % error.error_str

if store == []:
   os.kill(store_pid, 9)
   os.kill(configd_pid, 9)
   os.kill(broker_pid, 9)
   sys.exit(1)

# Wait until the Node object is created in the store, which means the configd
# has started up and contacted the store
print "Waiting for the node object to be created in the store"
result = store.checkNodeValidity([nodename])
while result.invalidNodes != []:
   time.sleep(1)
Exemple #15
0
    def build_execute_feature(self, feature_name, n_startd=1, n_slots=1, n_dynamic=0, dl_append=True, dedicated=True, preemption=False, ad_machine=True):
        sys.stdout.write("building execute feature %s -- n_startd=%d  n_slots=%d  n_dynamic=%d\n"%(feature_name, n_startd, n_slots, n_dynamic))
        sys.stdout.flush()

        self.assert_feature(feature_name)

        params={}
        params["USE_PROCD"] = "FALSE"

        if dedicated:
            params["START"] = "TRUE"
            params["SUSPEND"] = "FALSE"
            params["KILL"] = "FALSE"
            params["CONTINUE"] = "TRUE"
            params["WANT_VACATE"] = "FALSE"
            params["WANT_SUSPEND"] = "FALSE"

        if not preemption:
            params["MAXJOBRETIREMENTTIME"] = "3600 * 24"
            params["PREEMPT"] = "FALSE"
            params["PREEMPTION_REQUIREMENTS"] = "FALSE"
            params["RANK"] = "0"
            params["NEGOTIATOR_CONSIDER_PREEMPTION"] = "FALSE"

        if n_dynamic > 0:
            params["SLOT_TYPE_1"] = "cpus=%d"%(n_dynamic)
            params["SLOT_TYPE_1_PARTITIONABLE"] = "TRUE"
            params["NUM_SLOTS_TYPE_1"] = "%d"%(n_slots)
            params["NUM_CPUS"] = "%d"%(n_slots * n_dynamic)
        else:
            params["NUM_SLOTS"] = "%d"%(n_slots)
            params["NUM_CPUS"] = "%d"%(n_slots)

        if dl_append: daemon_list = ">= "
        else:         daemon_list = "MASTER"
        for s in xrange(n_startd):
            tag = "%03d"%(s)
            locname = "STARTD%s"%(tag)
            if (s > 0) or not dl_append: daemon_list += ","
            daemon_list += "STARTD%s"%(tag)
            params["STARTD%s"%(tag)] = "$(STARTD)"
            params["STARTD%s_ARGS"%(tag)] = "-f -local-name %s"%(locname)
            params["STARTD.%s.STARTD_NAME"%(locname)] = locname
            params["STARTD.%s.STARTD_ADDRESS_FILE"%(locname)] = "$(LOG)/.startd%s-address"%(tag)
            params["STARTD.%s.STARTD_LOG"%(locname)] = "$(LOG)/StartLog%s"%(tag)
            #params["STARTD.%s.EXECUTE"%(locname)] = "$(EXECUTE)/%s"%(locname)
            if ad_machine:
                params["STARTD%s.STARTD_ATTRS"%(tag)] = "$(STARTD_ATTRS), Machine"
                params["STARTD%s.Machine"%(tag)] = "\"s%s.$(FULL_HOSTNAME)\""%(tag)

        params["DAEMON_LIST"] = daemon_list

        # make sure parameters are declared
        splist = params.keys()
        splist.sort()
        for p in splist: self.assert_param(p)

        feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
        result = feat_obj.modifyParams('replace', params, {})
        if result.status != 0:
            sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
            raise WallabyStoreError("Failed to add feature")

        tslots = n_startd * n_slots
        return (tslots, tslots * n_dynamic)
Exemple #16
0
    except:
        sys.stderr.write('Unable to connect to broker "%s"\n' % broker_str)
        raise

    # Retrieve the config store object
    sys.stdout.write("Connecting to wallaby store:\n")
    try:
        (store_agent, config_store) = WallabyHelpers.get_store_objs(session)
    except WallabyStoreError, error:
        sys.stderr.write('Error: %s\n' % error.error_str)
        session.delBroker(broker)
        raise

    # Check API version number
    try:
        WallabyHelpers.verify_store_api(config_store, supported_api_versions)
    except WallabyUnsupportedAPI, error:
        if error.minor == 0:
            store_api_version = error.major
        else:
            store_api_version = '%s.%s' % (error.major, error.minor)
        sys.stderr.write('The store is using an API version that is not supported (%s)\n' % store_api_version)
        session.delBroker(broker)
        raise

    # return all the connection objects
    return (session, broker, store_agent, config_store)


# A base class for our unit tests -- defines snapshot/restore for the pool
class condor_unit_test(unittest.TestCase):