def fetch(self,constraint=None,format_list=None):
if constraint==None:
constraint_str=""
else:
constraint_str="-constraint '%s'"%constraint
full_xml=(format_list==None)
if format_list!=None:
format_arr=["-format '<c>' ClusterId"] #clusterid is always there, so this will always be printed out
for format_el in format_list:
attr_name,attr_type=format_el
attr_format={'s':'%s','i':'%i','r':'%f','b':'%i'}[attr_type]
format_arr.append('-format \'<a n="%s"><%s>%s</%s></a>\' %s'%(attr_name,attr_type,attr_format,attr_type,attr_name))
format_arr.append("-format '</c>' ClusterId") #clusterid is always there, so this will always be printed out
format_str=string.join(format_arr," ")
# set environment for security settings
self.security_obj.save_state()
self.security_obj.enforce_requests()
if full_xml:
xml_data = condorExe.exe_cmd(self.exe_name, "%s -xml %s %s" % (self.resource_str, self.pool_str, constraint_str))
else:
xml_data = condorExe.exe_cmd(self.exe_name, "%s %s %s %s" % (self.resource_str, format_str, self.pool_str, constraint_str))
xml_data = ['<?xml version="1.0"?><classads>'] + xml_data + ["</classads>"]
# restore old values
self.security_obj.restore_state()
list_data = xml2list(xml_data)
del xml_data
dict_data = list2dict(list_data, self.group_attribute)
return dict_data
def fetch(self,constraint=None,format_list=None):
if constraint is None:
constraint_str=""
else:
constraint_str="-constraint '%s'"%constraint
full_xml=(format_list is None)
if format_list is not None:
format_arr=[]
for format_el in format_list:
attr_name,attr_type=format_el
attr_format={'s':'%s','i':'%i','r':'%f','b':'%i'}[attr_type]
format_arr.append('-format "%s" "%s"'%(attr_format,attr_name))
format_str=string.join(format_arr," ")
# set environment for security settings
self.security_obj.save_state()
try:
self.security_obj.enforce_requests()
if full_xml:
xml_data = condorExe.exe_cmd(self.exe_name,"%s -xml %s %s"%(self.resource_str,self.pool_str,constraint_str),env=self.env);
else:
xml_data = condorExe.exe_cmd(self.exe_name,"%s %s -xml %s %s"%(self.resource_str,format_str,self.pool_str,constraint_str),env=self.env);
finally:
# restore old values
self.security_obj.restore_state()
list_data = xml2list(xml_data)
del xml_data
dict_data = list2dict(list_data, self.group_attribute)
return dict_data
def removeGlideins(schedd_name,jid_list,force=False):
####
# We are assuming the gfactory to be
# a condor superuser and thus does not need
# identity switching to remove jobs
####
global factoryConfig
removed_jids=[]
schedd_str=schedd_name2str(schedd_name)
is_not_first=0
for jid in jid_list:
if is_not_first:
is_not_first=1
time.sleep(factoryConfig.remove_sleep)
try:
condorExe.exe_cmd("condor_rm","%s %li.%li"%(schedd_str,jid[0],jid[1]))
removed_jids.append(jid)
# Force the removal if requested
if force == True:
try:
log_files.logActivity("Forcing the removal of glideins in X state")
condorExe.exe_cmd("condor_rm","-forcex %s %li.%li"%(schedd_str,jid[0],jid[1]))
except condorExe.ExeError, e:
log_files.logWarning("Forcing the removal of glideins in %s.%s state failed" % (jid[0],jid[1]))
except condorExe.ExeError, e:
# silently ignore errors, and try next one
log_files.logWarning("removeGlidein(%s,%li.%li): %s"%(schedd_name,jid[0],jid[1],e))
if len(removed_jids)>=factoryConfig.max_removes:
break # limit reached, stop
def releaseGlideins(schedd_name,jid_list):
####
# We are assuming the gfactory to be
# a condor superuser and thus does not need
# identity switching to release jobs
####
global factoryConfig
released_jids=[]
schedd_str=schedd_name2str(schedd_name)
is_not_first=0
for jid in jid_list:
if is_not_first:
is_not_first=1
time.sleep(factoryConfig.release_sleep)
try:
condorExe.exe_cmd("condor_release","%s %li.%li"%(schedd_str,jid[0],jid[1]))
released_jids.append(jid)
except condorExe.ExeError, e:
log_files.logWarning("releaseGlidein(%s,%li.%li): %s"%(schedd_name,jid[0],jid[1],e))
if len(released_jids)>=factoryConfig.max_releases:
break # limit reached, stop
def advertizeWork(client_name,req_name,glidein_name,min_glideins,glidein_params={}):
# get a 9 digit number that will stay 9 digit for the next 25 years
short_time = time.time()-1.05e9
tmpnam="/tmp/aw_%li_%li"%(short_time,os.getpid())
fd=file(tmpnam,"w")
try:
try:
fd.write('MyType = "%s"\n'%client_id)
fd.write('GlideMyType = "%s"\n'%client_id)
fd.write('Name = "%s@%s"\n'%(req_name,client_name))
fd.write('ClientName = "%s"\n'%client_name)
fd.write('ReqName = "%s"\n'%req_name)
fd.write('ReqGlidein = "%s"\n'%glidein_name)
fd.write('ReqIdleGlideins=%i\n'%min_glideins)
# write out both the attributes, prefixes and counts
prefix="GlideinParam"
data=glidein_params
for attr in data.keys():
el=data[attr]
if type(el)==type(1):
# don't quote ints
fd.write('%s%s = %s\n'%(prefix,attr,el))
else:
fd.write('%s%s = "%s"\n'%(prefix,attr,el))
finally:
fd.close()
condorExe.exe_cmd("../sbin/condor_advertise","UPDATE_MASTER_AD %s"%tmpnam)
finally:
os.remove(tmpnam)
def condorq_attrs(q_constraint, attribute_list):
"""
Retrieves a list of a single item from the all the factory queues.
"""
attr_str = ""
for attr in attribute_list:
attr_str += " -attr %s" % attr
xml_data = condorExe.exe_cmd("condor_q","-g -l %s -xml -constraint '%s'" % (attr_str, q_constraint))
classads_xml = []
tmp_list = []
for line in xml_data:
# look for the xml header
if line[:5] == "<?xml":
if len(tmp_list) > 0:
classads_xml.append(tmp_list)
tmp_list = []
tmp_list.append(line)
q_proxy_list = []
for ad_xml in classads_xml:
cred_list = xml2list(ad_xml)
q_proxy_list.extend(cred_list)
return q_proxy_list
def exe_privsep(cmd, options):
switchboard_stderr = os.dup( sys.stderr.fileno() )
# we duplicate stderr because condor_root_switchboard closes it automatically.
output = condorExe.exe_cmd("../sbin/condor_root_switchboard", "%s 0 %d" % (cmd, switchboard_stderr), options)
os.close(switchboard_stderr)
return output
def fetch_using_exe(self, constraint=None, format_list=None):
"""
Return the results obtained from executing the HTCondor query command
@param constraint: Constraints to be applied to the query
@type constraint: string
@param format_list: Classad attr & type. [(attr1, 'i'), ('attr2', 's')]
@type format_list: list
@return: Dict containing the results
@rtype: dict
"""
if constraint is None:
constraint_str=""
else:
constraint_str="-constraint '%s'"%constraint
full_xml=(format_list is None)
if format_list is not None:
format_arr=[]
for format_el in format_list:
attr_name,attr_type=format_el
attr_format={'s':'%s','i':'%i','r':'%f','b':'%i'}[attr_type]
format_arr.append('-format "%s" "%s"'%(attr_format,attr_name))
format_str=string.join(format_arr," ")
# set environment for security settings
self.security_obj.save_state()
try:
self.security_obj.enforce_requests()
if full_xml:
xml_data = condorExe.exe_cmd(self.exe_name,"%s -xml %s %s"%(self.resource_str,self.pool_str,constraint_str),env=self.env);
else:
xml_data = condorExe.exe_cmd(self.exe_name,"%s %s -xml %s %s"%(self.resource_str,format_str,self.pool_str,constraint_str),env=self.env);
finally:
# restore old security context
self.security_obj.restore_state()
list_data = xml2list(xml_data)
del xml_data
dict_data = list2dict(list_data, self.group_attribute)
return dict_data
def cached_exe_cmd(cmd, arg_str, schedd_name, pool_name, schedd_lookup_cache):
if schedd_lookup_cache == None:
schedd_lookup_cache = condorMonitor.NoneScheddCache()
schedd_str, env = schedd_lookup_cache.getScheddId(schedd_name, pool_name)
opts = "%s%s%s" % (pool2str(pool_name), schedd_str, arg_str)
return condorExe.exe_cmd(cmd, opts, env=env)
def condorSubmitOne(submit_file,schedd_name=None,pool_name=None):
submit_opts="%s%s%s"%(pool2str(pool_name),schedd2str(schedd_name),submit_file)
outstr=condorExe.exe_cmd("condor_submit",submit_opts)
#extract 'submitted to cluster xxx.' part
j = re.search(r'submitted to cluster [0-9]+\.', string.join(outstr))
sstr = j.string[j.start(0):j.end(0)]
#extract the number
j = re.search(r'[0-9]+', sstr)
idstr = j.string[j.start(0):j.end(0)]
return int(idstr)
def test_exe_cmd(self):
"""
exe_cmd is a wrapper for iexe_cmd. See test_iexe_cmd docstring for
full details.
"""
# Execution should proceed normally and exit with no exceptions.
try:
for script in self.normal_exit_scripts:
output = exe_cmd(script, self.dummy_args)
except Exception, e:
self.fail("Exception Occurred: %s" % str(e))
def exe_privsep(cmd, options):
switchboard_stderr = os.dup( sys.stderr.fileno() )
# we duplicate stderr because condor_root_switchboard closes it automatically.
try:
output = condorExe.exe_cmd("../sbin/condor_root_switchboard", "%s 0 %d" % (cmd, switchboard_stderr), options)
except:
try:
exc_info = sys.exc_info()
os.close(switchboard_stderr)
except OSError:
# Ignore if it is already closed, raise original exception
raise exc_info[0], exc_info[1], exc_info[2]
raise
os.close(switchboard_stderr)
return output
def condorReschedule(schedd_name=None,pool_name=None):
cmd_opts="%s%s"%(pool2str(pool_name),schedd2str(schedd_name))
condorExe.exe_cmd("condor_reschedule",cmd_opts)
return
def condorRemoveOne(cluster_or_uname,schedd_name=None,pool_name=None):
rm_opts="%s%s%s"%(pool2str(pool_name),schedd2str(schedd_name),cluster_or_uname)
return condorExe.exe_cmd("condor_rm",rm_opts)
def condorRemove(constraint,schedd_name=None,pool_name=None):
rm_opts="%s%s-constraint '%s'"%(pool2str(pool_name),schedd2str(schedd_name),constraint)
return condorExe.exe_cmd("condor_rm",rm_opts)
def exe_privsep(cmd,options):
return condorExe.exe_cmd("../sbin/condor_root_switchboard","%s 0 2"%cmd,
options)
def create_client_condor_config(config_fname, mapfile_fname, collector_nodes, classad_proxy):
attrs = condorExe.exe_cmd('condor_config_val','-dump')
def_attrs = filter_unwanted_config_attrs(attrs)
fd=open(config_fname,"w")
try:
fd.write("############################################\n")
fd.write("#\n")
fd.write("# Condor config file used by the VO Frontend\n")
fd.write("#\n")
fd.write("# This file is generated at each reconfig\n")
fd.write("# Do not change by hand!\n")
fd.write("#\n")
fd.write("############################################\n\n")
fd.write("###########################\n")
fd.write("# Base config values\n")
fd.write("# obtained from\n")
fd.write("# condor_config_val -dump\n")
fd.write("# at config time.\n")
fd.write("###########################\n\n")
fd.writelines(def_attrs)
fd.write("\n##################################\n")
fd.write("# Add Frontend specific attributes\n")
fd.write("##################################\n")
fd.write("\n###########################\n")
fd.write("# Pool collector(s)\n")
fd.write("###########################\n")
fd.write("COLLECTOR_HOST = %s\n"%string.join(collector_nodes,","))
fd.write("\n###########################\n")
fd.write("# Authentication settings\n")
fd.write("############################\n")
fd.write("\n# Force GSI authentication\n")
fd.write("SEC_DEFAULT_AUTHENTICATION_METHODS = GSI\n")
fd.write("SEC_DEFAULT_AUTHENTICATION = REQUIRED\n")
fd.write("\n#################################\n")
fd.write("# Where to find ID->uid mappings\n")
fd.write("# (also disable any GRIDMAP)\n")
fd.write("#################################\n")
fd.write("# This is a fake file, redefine at runtime\n")
fd.write("CERTIFICATE_MAPFILE=%s\n"%mapfile_fname)
fd.write("\n# Specify that we trust anyone but not anonymous\n")
fd.write("# I.e. we only talk to servers that have \n")
fd.write("# a DN mapped in our mapfile\n")
for context in condorSecurity.CONDOR_CONTEXT_LIST:
fd.write("DENY_%s = anonymous@*\n"%context)
fd.write("\n")
for context in condorSecurity.CONDOR_CONTEXT_LIST:
fd.write("ALLOW_%s = *@*\n"%context)
fd.write("\n")
fd.write("\n# Unset all the tool specifics\n")
fd.write("\n# Force integrity\n")
fd.write("SEC_DEFAULT_INTEGRITY = REQUIRED\n")
fd.write("\n######################################################\n")
fd.write("## If someone tried to use this config to start a master\n")
fd.write("## make sure it is not used to run any daemons\n")
fd.write("######################################################\n")
fd.write("DAEMON_LIST=MASTER\n")
fd.write("DAEMON_SHUTDOWN=True\n")
fd.write("\n######################################################\n")
fd.write("## If condor is allowed to use VOMS attributes, it will\n")
fd.write("## map COLLECTOR DN to anonymous. Just disable it.\n")
fd.write("######################################################\n")
fd.write("USE_VOMS_ATTRIBUTES = False\n")
fd.write("\n######################################################\n")
fd.write("## Add GSI DAEMON PROXY based on the frontend config and \n")
fd.write("## not what is in the condor configs from install \n")
fd.write("########################################################\n")
fd.write("GSI_DAEMON_PROXY = %s\n" % classad_proxy)
finally:
fd.close()
return