def connect_to_wallaby(broker_addr='127.0.0.1', port=5672, username='', passwd='', mechanisms='ANONYMOUS PLAIN GSSAPI'):
global supported_api_versions
# set up session for wallaby
session = Session()
if username != '' and passwd != '':
broker_str = '%s/%s@%s:%d' % (username, passwd, broker_addr, port)
elif username != '':
broker_str = '%s@%s:%d' % (username, broker_addr, port)
else:
broker_str = '%s:%d' % (broker_addr, port)
sys.stdout.write("Connecting to broker %s:\n" % (broker_str))
try:
broker = session.addBroker('amqp://%s' % broker_str, mechanisms=mechanisms)
except:
sys.stderr.write('Unable to connect to broker "%s"\n' % broker_str)
raise
# Retrieve the config store object
sys.stdout.write("Connecting to wallaby store:\n")
try:
(store_agent, config_store) = WallabyHelpers.get_store_objs(session)
except WallabyStoreError, error:
sys.stderr.write('Error: %s\n' % error.error_str)
session.delBroker(broker)
raise
def build_accounting_group_feature(self, feature_name, group_tuple_list):
sys.stdout.write("building acct group feature %s\n"%(feature_name))
sys.stdout.flush()
self.assert_feature(feature_name)
params = {}
params["GROUP_NAMES"] = ",".join([x[0] for x in group_tuple_list])
for tuple in group_tuple_list:
(name, is_static, quota, accept_surplus) = tuple
if is_static: params["GROUP_QUOTA_%s"%(name)] = "%d"%(int(quota))
else: params["GROUP_QUOTA_DYNAMIC_%s"%(name)] = "%f"%(float(quota))
if accept_surplus: params["GROUP_AUTOREGROUP_%s"%(name)] = "TRUE"
# make sure parameters are declared
splist = params.keys()
splist.sort()
for p in splist: self.assert_param(p)
feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
result = feat_obj.modifyParams('replace', params, {})
if result.status != 0:
sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
raise WallabyStoreError("Failed to add feature")
def build_access_feature(self, feature_name, collector_host=None, condor_host=None):
if condor_host==None: condor_host = self.params.collector_addr
if collector_host==None: collector_host = condor_host
sys.stdout.write("building access feature %s\n"%(feature_name))
sys.stdout.flush()
self.assert_feature(feature_name)
params={}
params["CONDOR_HOST"] = condor_host
params["COLLECTOR_HOST"] = collector_host
params["ALLOW_WRITE"] = "*"
params["ALLOW_READ"] = "*"
params["SEC_DEFAULT_AUTHENTICATION_METHODS"] = "CLAIMTOBE"
# make sure parameters are declared
splist = params.keys()
splist.sort()
for p in splist: self.assert_param(p)
feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
result = feat_obj.modifyParams('replace', params, {})
if result.status != 0:
sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
raise WallabyStoreError("Failed to add feature")
def assert_node_groups(self, group_names, node_names, mod_op='replace'):
# apply the groups to the nodes
for name in node_names:
node_obj = WallabyHelpers.get_node(self.session, self.config_store, name)
result = node_obj.modifyMemberships(mod_op, group_names, {})
if result.status != 0:
sys.stderr.write("Failed to set groups for %s: (%d, %s)\n" % (name, result.status, result.text))
raise WallabyStoreError(result.text)
def assert_node_features(self, feature_names, node_names, mod_op='replace'):
for feat in feature_names:
if not feat in self.feat_names:
emsg = "Feature %s not in config store" % (feat)
raise Exception(emsg)
# apply feature list to nodes
for name in node_names:
node_obj = WallabyHelpers.get_node(self.session, self.config_store, name)
group_name = WallabyHelpers.get_id_group_name(node_obj, self.session)
group_obj = WallabyHelpers.get_group(self.session, self.config_store, group_name)
if mod_op == 'insert':
result = group_obj.modifyFeatures('replace', feature_names + group_obj.features, {})
else:
result = group_obj.modifyFeatures(mod_op, feature_names, {})
if result.status != 0:
sys.stderr.write("Failed to set features for %s: (%d, %s)\n" % (name, result.status, result.text))
raise WallabyStoreError(result.text)
def clear_nodes(self, node_names):
for name in node_names:
node_obj = WallabyHelpers.get_node(self.session, self.config_store, name)
result = node_obj.modifyMemberships('replace', [], {})
if result.status != 0:
sys.stderr.write("Failed to clear groups from %s: (%d, %s)\n" % (name, result.status, result.text))
raise WallabyStoreError("Failed to clear groups")
group_name = WallabyHelpers.get_id_group_name(node_obj, self.session)
group_obj = WallabyHelpers.get_group(self.session, self.config_store, group_name)
result = group_obj.modifyFeatures('replace', [], {})
if result.status != 0:
sys.stderr.write("Failed to clear features from %s: (%d, %s)\n" % (name, result.status, result.text))
raise WallabyStoreError("Failed to clear features")
result = group_obj.modifyParams('replace', {}, {})
if result.status != 0:
sys.stderr.write("Failed to clear params from %s: (%d, %s)\n" % (name, result.status, result.text))
raise WallabyStoreError("Failed to clear params")
def clear_default_group(self):
group_obj = WallabyHelpers.get_group(self.session, self.config_store, '+++DEFAULT')
result = group_obj.modifyFeatures('replace', [], {})
if result.status != 0:
sys.stderr.write("Failed to clear features from %s: (%d, %s)\n" % (name, result.status, result.text))
raise WallabyStoreError("Failed to clear features")
result = group_obj.modifyParams('replace', {}, {})
if result.status != 0:
sys.stderr.write("Failed to clear params from %s: (%d, %s)\n" % (name, result.status, result.text))
raise WallabyStoreError("Failed to clear params")
def list_nodes(self, with_all_feats=None, without_any_feats=None, with_all_groups=None, without_any_groups=None, checkin_since=None):
r = []
for node in self.node_names:
node_obj = WallabyHelpers.get_node(self.session, self.config_store, node)
sys.stdout.write(" list_nodes: node=%s checkin= %s\n" % (node, node_obj.last_checkin))
if (checkin_since != None) and ((node_obj.last_checkin / 1000000) < checkin_since): continue
nodefeats = []
if (with_all_feats != None) or (without_any_feats != None):
nodefeats = WallabyHelpers.get_node_features(node_obj, self.session, self.config_store)
if (with_all_feats != None) and (False in [x in nodefeats for x in with_all_feats]): continue
if (without_any_feats != None) and (True in [x in nodefeats for x in without_any_feats]): continue
nodegroups = []
if (with_all_groups != None) or (without_any_groups != None):
nodegroups = [WallabyHelpers.get_id_group_name(node_obj, self.session)] + node_obj.memberships + ['+++DEFAULT']
if (with_all_groups != None) and (False in [x in nodegroups for x in with_all_groups]): continue
if (without_any_groups != None) and (True in [x in nodegroups for x in without_any_groups]): continue
r += [node]
return r
def tag_test_feature(self, feature_name, param_name):
# ensure parameter name exists
self.assert_param(param_name)
# ensure that parameter requires restart
param_obj = WallabyHelpers.get_param(self.session, self.config_store, param_name)
result = param_obj.setRequiresRestart(True)
if result.status != 0:
sys.stderr.write("Failed to set restart for %s: (%d, %s)\n" % (param_name, result.status, result.text))
raise WallabyStoreError("Failed to set restart")
# set this param to a new value, to ensure a restart on activation
feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
result = feat_obj.modifyParams('add', {param_name:("%s"%(time.time()))}, {})
if result.status != 0:
sys.stderr.write("Failed to add param %s to %s: (%d, %s)\n" % (param_name, feature_name, result.status, result.text))
raise WallabyStoreError("Failed to add param")
# make sure master is tagged for restart via this parameter
subsys_obj = WallabyHelpers.get_subsys(self.session, self.config_store, 'master')
result = subsys_obj.modifyParams('add', [param_name], {})
if result.status != 0:
sys.stderr.write("Failed to add param %s to master: (%d, %s)\n" % (param_name, result.status, result.text))
raise WallabyStoreError("Failed to add param")
def build_feature(self, feature_name, params={}, mod_op='replace'):
sys.stdout.write("building feature %s\n"%(feature_name))
sys.stdout.flush()
self.assert_feature(feature_name)
# make sure parameters are declared
splist = params.keys()
splist.sort()
for p in splist: self.assert_param(p)
feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
result = feat_obj.modifyParams(mod_op, params, {})
if result.status != 0:
sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
raise WallabyStoreError("Failed to add feature")
def build_collector_feature(self, feature_name, n_coll=1, portstart=10000, dl_append=True, collector_host=None, condor_host=None, disable_plugins=True):
if condor_host==None: condor_host = self.params.collector_addr
if collector_host==None: collector_host = condor_host
sys.stdout.write("building collector feature %s with %d sub-collectors\n"%(feature_name, n_coll))
sys.stdout.flush()
self.assert_feature(feature_name)
collector_names = []
params={}
if dl_append: daemon_list = ">= "
else: daemon_list = "MASTER"
for s in xrange(n_coll):
tag = "%03d"%(s)
port=portstart+s
locname = "COLLECTOR%s"%(tag)
collector_names += ["%s:%d"%(collector_host,port)]
if (s > 0) or not dl_append: daemon_list += ","
daemon_list += "COLLECTOR%s"%(tag)
params["COLLECTOR%s"%(tag)] = "$(COLLECTOR)"
params["COLLECTOR%s_ARGS"%(tag)] = "-f -p %d -local-name %s"%(port,locname)
params["COLLECTOR%s_ENVIRONMENT"%(tag)] = "_CONDOR_COLLECTOR_LOG=$(LOG)/CollectorLog%s"%(tag)
params["COLLECTOR.%s.COLLECTOR_NAME"%(locname)] = locname
params["COLLECTOR.%s.CONDOR_VIEW_HOST"%(locname)] = "$(COLLECTOR_HOST)"
params["COLLECTOR.%s.COLLECTOR_ADDRESS_FILE"%(locname)] = "$(LOG)/.collector%s-address"%(tag)
if disable_plugins:
params["COLLECTOR.%s.PLUGINS"%(locname)] = ""
params["DAEMON_LIST"] = daemon_list
# make sure parameters are declared
splist = params.keys()
splist.sort()
for p in splist: self.assert_param(p)
feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
result = feat_obj.modifyParams('replace', params, {})
if result.status != 0:
sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
raise WallabyStoreError("Failed to add feature")
return collector_names
def assert_group_features(self, feature_names, group_names, mod_op='replace'):
# ensure these actually exist
for grp in group_names:
if not grp in self.group_names:
sys.stdout.write("Adding group %s to store:\n" % (grp))
result = self.config_store.addExplicitGroup(grp)
if result.status != 0:
sys.stderr.write("Failed to create group %s: (%d, %s)\n" % (grp, result.status, result.text))
raise WallabyStoreError(result.text)
# In principle, could automatically install features if they aren't found
for feat in feature_names:
if not feat in self.feat_names: raise Exception("Feature %s not in config store" % (feat))
# apply feature list to group
for name in group_names:
group_obj = WallabyHelpers.get_group(self.session, self.config_store, name)
result = group_obj.modifyFeatures(mod_op, feature_names, {})
if result.status != 0:
sys.stderr.write("Failed to set features for %s: (%d, %s)\n" % (name, result.status, result.text))
raise WallabyStoreError(result.text)
def build_scheduler_feature(self, feature_name, n_schedd=1, dl_append=True):
sys.stdout.write("building scheduler feature %s with %d schedds\n"%(feature_name, n_schedd))
sys.stdout.flush()
self.assert_feature(feature_name)
schedd_names = []
params={}
params["USE_PROCD"] = "FALSE"
if dl_append: daemon_list = ">= "
else: daemon_list = "MASTER"
for s in xrange(n_schedd):
tag = "%03d"%(s)
locname = "SCHEDD%s"%(tag)
schedd_names += [locname]
if (s > 0) or not dl_append: daemon_list += ","
daemon_list += "SCHEDD%s"%(tag)
params["SCHEDD%s"%(tag)] = "$(SCHEDD)"
params["SCHEDD%s_ARGS"%(tag)] = "-f -local-name %s"%(locname)
params["SCHEDD.%s.SCHEDD_NAME"%(locname)] = locname
params["SCHEDD.%s.SCHEDD_LOG"%(locname)] = "$(LOG)/SchedLog%s"%(tag)
params["SCHEDD.%s.SCHEDD_ADDRESS_FILE"%(locname)] = "$(LOG)/.schedd%s-address"%(tag)
params["SCHEDD.%s.SPOOL"%(locname)] = "$(SPOOL).%s"%(tag)
params["SCHEDD.%s.HISTORY"%(locname)] = "$(SPOOL)/history.s%s"%(tag)
#params["SCHEDD.%s.SCHEDD_DAEMON_AD_FILE"%(locname)] = "$(LOG)/.schedd%s-daemon-ad"%(tag)
params["DAEMON_LIST"] = daemon_list
# make sure parameters are declared
splist = params.keys()
splist.sort()
for p in splist: self.assert_param(p)
feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
result = feat_obj.modifyParams('replace', params, {})
if result.status != 0:
sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
raise WallabyStoreError("Failed to add feature")
return schedd_names
env['CONDOR_CONFIG'] = '/etc/condor/condor_config'
env['_CONDOR_LOCAL_CONFIG_DIR'] = config_dir
env['_CONDOR_CONFIGD_CHECK_INTERVAL'] = str(checkin_time)
env['_CONDOR_QMF_BROKER_HOST'] = '127.0.0.1'
env['_CONDOR_QMF_BROKER_PORT'] = '5672'
env['_CONDOR_CONFIGD_OVERRIDE_DIR'] = override_dir
env['_CONDOR_LOG'] = '.'
env['PATH'] = os.environ['PATH']
cmd = shlex.split('../condor_configd -d -l %s -m %s -h %s' % (log_file, config_file, nodename))
os.execve(cmd[0], cmd, env)
sys.exit(0)
# Setup the connection to the store
store = []
try:
(agent, store) = WallabyHelpers.get_store_objs(session)
except Exception, error:
print 'Error: %s' % error.error_str
if store == []:
os.kill(store_pid, 9)
os.kill(configd_pid, 9)
os.kill(broker_pid, 9)
sys.exit(1)
# Wait until the Node object is created in the store, which means the configd
# has started up and contacted the store
print "Waiting for the node object to be created in the store"
result = store.checkNodeValidity([nodename])
while result.invalidNodes != []:
time.sleep(1)
def build_execute_feature(self, feature_name, n_startd=1, n_slots=1, n_dynamic=0, dl_append=True, dedicated=True, preemption=False, ad_machine=True):
sys.stdout.write("building execute feature %s -- n_startd=%d n_slots=%d n_dynamic=%d\n"%(feature_name, n_startd, n_slots, n_dynamic))
sys.stdout.flush()
self.assert_feature(feature_name)
params={}
params["USE_PROCD"] = "FALSE"
if dedicated:
params["START"] = "TRUE"
params["SUSPEND"] = "FALSE"
params["KILL"] = "FALSE"
params["CONTINUE"] = "TRUE"
params["WANT_VACATE"] = "FALSE"
params["WANT_SUSPEND"] = "FALSE"
if not preemption:
params["MAXJOBRETIREMENTTIME"] = "3600 * 24"
params["PREEMPT"] = "FALSE"
params["PREEMPTION_REQUIREMENTS"] = "FALSE"
params["RANK"] = "0"
params["NEGOTIATOR_CONSIDER_PREEMPTION"] = "FALSE"
if n_dynamic > 0:
params["SLOT_TYPE_1"] = "cpus=%d"%(n_dynamic)
params["SLOT_TYPE_1_PARTITIONABLE"] = "TRUE"
params["NUM_SLOTS_TYPE_1"] = "%d"%(n_slots)
params["NUM_CPUS"] = "%d"%(n_slots * n_dynamic)
else:
params["NUM_SLOTS"] = "%d"%(n_slots)
params["NUM_CPUS"] = "%d"%(n_slots)
if dl_append: daemon_list = ">= "
else: daemon_list = "MASTER"
for s in xrange(n_startd):
tag = "%03d"%(s)
locname = "STARTD%s"%(tag)
if (s > 0) or not dl_append: daemon_list += ","
daemon_list += "STARTD%s"%(tag)
params["STARTD%s"%(tag)] = "$(STARTD)"
params["STARTD%s_ARGS"%(tag)] = "-f -local-name %s"%(locname)
params["STARTD.%s.STARTD_NAME"%(locname)] = locname
params["STARTD.%s.STARTD_ADDRESS_FILE"%(locname)] = "$(LOG)/.startd%s-address"%(tag)
params["STARTD.%s.STARTD_LOG"%(locname)] = "$(LOG)/StartLog%s"%(tag)
#params["STARTD.%s.EXECUTE"%(locname)] = "$(EXECUTE)/%s"%(locname)
if ad_machine:
params["STARTD%s.STARTD_ATTRS"%(tag)] = "$(STARTD_ATTRS), Machine"
params["STARTD%s.Machine"%(tag)] = "\"s%s.$(FULL_HOSTNAME)\""%(tag)
params["DAEMON_LIST"] = daemon_list
# make sure parameters are declared
splist = params.keys()
splist.sort()
for p in splist: self.assert_param(p)
feat_obj = WallabyHelpers.get_feature(self.session, self.config_store, feature_name)
result = feat_obj.modifyParams('replace', params, {})
if result.status != 0:
sys.stderr.write("Failed to modify params for %s: (%d, %s)\n" % (feature_name, result.status, result.text))
raise WallabyStoreError("Failed to add feature")
tslots = n_startd * n_slots
return (tslots, tslots * n_dynamic)
except:
sys.stderr.write('Unable to connect to broker "%s"\n' % broker_str)
raise
# Retrieve the config store object
sys.stdout.write("Connecting to wallaby store:\n")
try:
(store_agent, config_store) = WallabyHelpers.get_store_objs(session)
except WallabyStoreError, error:
sys.stderr.write('Error: %s\n' % error.error_str)
session.delBroker(broker)
raise
# Check API version number
try:
WallabyHelpers.verify_store_api(config_store, supported_api_versions)
except WallabyUnsupportedAPI, error:
if error.minor == 0:
store_api_version = error.major
else:
store_api_version = '%s.%s' % (error.major, error.minor)
sys.stderr.write('The store is using an API version that is not supported (%s)\n' % store_api_version)
session.delBroker(broker)
raise
# return all the connection objects
return (session, broker, store_agent, config_store)
# A base class for our unit tests -- defines snapshot/restore for the pool
class condor_unit_test(unittest.TestCase):