def deadvertizeAllGlideinClientMonitoring(factory_name,
glidein_name,
entry_name,
factory_collector=DEFAULT_VAL):
"""
Deadvertize monitoring classads for the given entry.
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfc')
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factoryclient_id)
fd.write(
'Requirements = (ReqGlidein == "%s@%s@%s")&&(GlideinMyType == "%s")\n'
% (entry_name, glidein_name, factory_name,
factoryConfig.factoryclient_id))
finally:
fd.close()
exe_condor_advertise(tmpnam,
"INVALIDATE_LICENSE_ADS",
factory_collector=factory_collector)
finally:
os.remove(tmpnam)
def writeToMultiClassadFile(self, filename=None, append=True):
# filename: Name of the file to write classads to
# append: Wether the classads need to be appended to the file
# If we create file append is in a way ignored
if filename is None:
filename = classadSupport.generate_classad_filename(
prefix='gfi_adm_gfc')
append = False
for el in self.client_data:
createGlideinClientMonitoringFile(filename,
self.factory_name,
self.glidein_name,
self.entry_name,
el['client_name'],
el['client_int_name'],
el['client_int_req'],
self.glidein_attrs,
el['client_params'],
el['client_monitors'],
el['limits_triggered'],
do_append=append)
# Append from here on anyways
append = True
return filename
def advertizeGlideinClientMonitoring(
factory_name,
glidein_name,
entry_name,
client_name,
client_int_name,
client_int_req,
glidein_attrs={},
client_params={},
client_monitors={},
factory_collector=DEFAULT_VAL,
):
tmpnam = classadSupport.generate_classad_filename(prefix="gfi_adm_gfc")
createGlideinClientMonitoringFile(
tmpnam,
factory_name,
glidein_name,
entry_name,
client_name,
client_int_name,
client_int_req,
glidein_attrs,
client_params,
client_monitors,
)
advertizeGlideinClientMonitoringFromFile(tmpnam, remove_file=True, factory_collector=factory_collector)
def deadvertizeGlidein(factory_name,
glidein_name,
entry_name,
factory_collector=DEFAULT_VAL):
"""
Removes the glidefactory classad advertising the entry from the WMS Collector.
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gf')
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factory_id)
fd.write(
'Requirements = (Name == "%s@%s@%s")&&(GlideinMyType == "%s")\n'
% (entry_name, glidein_name, factory_name,
factoryConfig.factory_id))
finally:
fd.close()
exe_condor_advertise(tmpnam,
"INVALIDATE_ADS_GENERIC",
factory_collector=factory_collector)
finally:
os.remove(tmpnam)
def advertizeGlobal(factory_name, glidein_name, supported_signtypes,
pub_key_obj, stats_dict={}, factory_collector=DEFAULT_VAL):
"""
Creates the glidefactoryglobal classad and advertises.
@type factory_name: string
@param factory_name: the name of the factory
@type glidein_name: string
@param glidein_name: name of the glidein
@type supported_signtypes: string
@param supported_signtypes: suppported sign types, i.e. sha1
@type pub_key_obj: GlideinKey
@param pub_key_obj: for the frontend to use in encryption
@type stats_dict: dict
@param stats_dict: completed jobs statistics
@type factory_collector: string or None
@param factory_collector: the collector to query, special value 'default' will get it from the global config
@todo add factory downtime?
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_ad_gfg')
gfg_classad = FactoryGlobalClassad(factory_name, glidein_name,
supported_signtypes, pub_key_obj)
try:
gfg_classad.writeToFile(tmpnam, append=False)
exe_condor_advertise(tmpnam, gfg_classad.adAdvertiseCmd,
factory_collector=factory_collector)
finally:
# Unable to write classad
_remove_if_there(tmpnam)
def writeToMultiClassadFile(self, filename=None, append=True):
# filename: Name of the file to write classads to
# append: Wether the classads need to be appended to the file
# If we create file append is in a way ignored
if filename is None:
filename = classadSupport.generate_classad_filename(prefix="gfi_adm_gfc")
append = False
for el in self.client_data:
createGlideinClientMonitoringFile(
filename,
self.factory_name,
self.glidein_name,
self.entry_name,
el["client_name"],
el["client_int_name"],
el["client_int_req"],
self.glidein_attrs,
el["client_params"],
el["client_monitors"],
el["limits_triggered"],
do_append=append,
)
# Append from here on anyways
append = True
return filename
def do_advertize_multi(self):
tmpnam = classadSupport.generate_classad_filename(prefix="gfi_adm_gfc")
ap = False
for el in self.client_data:
createGlideinClientMonitoringFile(
tmpnam,
self.factory_name,
self.glidein_name,
self.entry_name,
el["client_name"],
el["client_int_name"],
el["client_int_req"],
self.glidein_attrs,
el["client_params"],
el["client_monitors"],
do_append=ap,
)
ap = True # Append from here on
if ap:
error_arr = []
try:
advertizeGlideinClientMonitoringFromFile(
tmpnam, remove_file=True, is_multi=True, factory_collector=self.factory_collector
)
except condorExe.ExeError, e:
error_arr.append(e)
if len(error_arr) > 0:
raise MultiExeError(error_arr)
def do_advertize_multi(self):
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_adm_gfc')
ap = False
for el in self.client_data:
createGlideinClientMonitoringFile(tmpnam,
self.factory_name,
self.glidein_name,
self.entry_name,
el['client_name'],
el['client_int_name'],
el['client_int_req'],
self.glidein_attrs,
el['client_params'],
el['client_monitors'],
do_append=ap)
ap = True # Append from here on
if ap:
error_arr = []
try:
advertizeGlideinClientMonitoringFromFile(
tmpnam,
remove_file=True,
is_multi=True,
factory_collector=self.factory_collector)
except condorExe.ExeError as e:
error_arr.append(e)
if len(error_arr) > 0:
raise MultiExeError(error_arr)
def advertizeGlideinClientMonitoring(factory_name, glidein_name, entry_name,
client_name, client_int_name, client_int_req,
glidein_attrs={}, client_params={}, client_monitors={}):
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_adm_gfc')
createGlideinClientMonitoringFile(tmpnam, factory_name, glidein_name, entry_name,
client_name, client_int_name, client_int_req,
glidein_attrs, client_params, client_monitors)
advertizeGlideinClientMonitoringFromFile(tmpnam, remove_file=True)
def advertizeGlideinClientMonitoring(factory_name, glidein_name, entry_name,
client_name, client_int_name, client_int_req,
glidein_attrs={}, client_params={}, client_monitors={},
factory_collector=DEFAULT_VAL):
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_adm_gfc')
createGlideinClientMonitoringFile(tmpnam, factory_name, glidein_name, entry_name,
client_name, client_int_name, client_int_req,
glidein_attrs, client_params, client_monitors)
advertizeGlideinClientMonitoringFromFile(tmpnam, remove_file=True, factory_collector=factory_collector)
def deadvertizeGlobal(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
"""
Removes the glidefactoryglobal classad advertising the factory globals from the WMS Collector.
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfg')
# TODO: use tempfile
try:
with open(tmpnam, "w") as fd:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factory_global)
fd.write('Requirements = (Name == "%s@%s")&&(GlideinMyType == "%s")\n' % (glidein_name, factory_name, factoryConfig.factory_id))
exe_condor_advertise(tmpnam, "INVALIDATE_ADS_GENERIC", factory_collector=factory_collector)
finally:
_remove_if_there(tmpnam)
def deadvertizeFactory(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
"""
Deadvertize all entry and global classads for this factory.
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_fact')
# TODO: use tempfile
try:
with open(tmpnam, "w") as fd:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factory_id)
fd.write('Requirements = (FactoryName =?= "%s")&&(GlideinName =?= "%s")\n' % (factory_name, glidein_name))
exe_condor_advertise(tmpnam, "INVALIDATE_ADS_GENERIC", factory_collector=factory_collector)
finally:
_remove_if_there(tmpnam)
def do_advertize_iterate(self):
error_arr = []
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_ad_gfc')
for el in self.client_data:
createGlideinClientMonitoringFile(
tmpnam, self.factory_name, self.glidein_name, self.entry_name,
el['client_name'], el['client_int_name'], el['client_int_req'],
self.glidein_attrs, el['client_params'], el['client_monitors'])
try:
advertizeGlideinClientMonitoringFromFile(tmpnam,
remove_file=True)
except condorExe.ExeError, e:
error_arr.append(e)
def advertizeGlobal(factory_name, glidein_name, supported_signtypes,
pub_key_obj, factory_collector=DEFAULT_VAL):
"""
Creates the glidefactoryglobal classad and advertises.
@type factory_name: string
@param factory_name: the name of the factory
@type glidein_name: string
@param glidein_name: name of the glidein
@type supported_signtypes: string
@param supported_signtypes: suppported sign types, i.e. sha1
@type pub_key_obj: GlideinKey
@param pub_key_obj: for the frontend to use in encryption
@type factory_collector: string or None
@param factory_collector: the collector to query, special value 'default' will get it from the global config
@todo add factory downtime?
"""
global factoryConfig
global advertizeGlobalCounter
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_ad_gfg')
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "%s"\n' % factoryConfig.factory_global)
fd.write('GlideinMyType = "%s"\n' % factoryConfig.factory_global)
fd.write('GlideinWMSVersion = "%s"\n' % factoryConfig.glideinwms_version)
fd.write('Name = "%s@%s"\n' % (glidein_name, factory_name))
fd.write('FactoryName = "%s"\n' % factory_name)
fd.write('GlideinName = "%s"\n' % glidein_name)
fd.write('%s = "%s"\n' % (factoryConfig.factory_signtype_id, string.join(supported_signtypes, ',')))
fd.write('PubKeyID = "%s"\n' % pub_key_obj.get_pub_key_id())
fd.write('PubKeyType = "%s"\n' % pub_key_obj.get_pub_key_type())
fd.write('PubKeyValue = "%s"\n' % string.replace(pub_key_obj.get_pub_key_value(), '\n', '\\n'))
fd.write('DaemonStartTime = %li\n' % start_time)
fd.write('UpdateSequenceNumber = %i\n' % advertizeGlobalCounter)
advertizeGlobalCounter += 1
finally:
fd.close()
exe_condor_advertise(tmpnam, "UPDATE_MASTER_AD", factory_collector=factory_collector)
finally:
os.remove(tmpnam)
def deadvertizeFactoryClientMonitoring(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
"""
Deadvertize all monitoring classads for this factory.
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfc')
# TODO: use tempfile
try:
with open(tmpnam, "w") as fd:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factoryclient_id)
fd.write('Requirements = (ReqFactoryName=?="%s")&&(ReqGlideinName=?="%s")&&(GlideinMyType == "%s")' %
(factory_name, glidein_name, factoryConfig.factoryclient_id))
exe_condor_advertise(tmpnam, "INVALIDATE_LICENSE_ADS", factory_collector=factory_collector)
finally:
_remove_if_there(tmpnam)
def do_advertize(self):
"""
Do the actual advertizing
"""
idx = 0
for factory_pool in self.factory_queue.keys():
idx = idx + 1
self.unique_id = 1
self.adname = classadSupport.generate_classad_filename(prefix="gfi_ad_gc_%li" % (idx))
# this should be done in parallel, but keep it serial for now
filename_arr = []
if frontendConfig.advertise_use_multi == True:
filename_arr.append(self.adname)
for el in self.factory_queue[factory_pool]:
params_obj, key_obj = el
try:
filename_arr_el = self.createAdvertizeWorkFile(factory_pool, params_obj, key_obj)
for f in filename_arr_el:
if f not in filename_arr:
filename_arr.append(f)
except NoCredentialException:
filename_arr = [] # don't try to advertise
logSupport.log.warning(
"No security credentials match for factory pool %s, not advertising request" % factory_pool
)
except condorExe.ExeError:
filename_arr = [] # don't try to advertise
logSupport.log.exception(
"Error creating request files for factory pool %s, unable to advertise: " % factory_pool
)
logSupport.log.error(
"Error creating request files for factory pool %s, unable to advertise" % factory_pool
)
# Advertize all the files (if multi, should only be one)
for filename in filename_arr:
try:
advertizeWorkFromFile(
factory_pool, filename, remove_file=True, is_multi=frontendConfig.advertise_use_multi
)
except condorExe.ExeError:
logSupport.log.exception("Advertising request failed for factory pool %s: " % factory_pool)
self.factory_queue = {} # clean queue
def deadvertizeAllGlideinClientMonitoring(factory_name, glidein_name, entry_name):
"""
Deadvertize monitoring classads for the given entry.
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfc')
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factoryclient_id)
fd.write('Requirements = (ReqGlidein == "%s@%s@%s")&&(GlideinMyType == "%s")\n' % (entry_name, glidein_name, factory_name, factoryConfig.factoryclient_id))
finally:
fd.close()
exe_condor_advertise(tmpnam, "INVALIDATE_LICENSE_ADS")
finally:
os.remove(tmpnam)
def deadvertizeGlobal(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
"""
Removes the glidefactoryglobal classad advertising the factory globals from the WMS Collector.
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfg')
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factory_global)
fd.write('Requirements = (Name == "%s@%s")&&(GlideinMyType == "%s")\n' % (glidein_name, factory_name, factoryConfig.factory_id))
finally:
fd.close()
exe_condor_advertise(tmpnam, "INVALIDATE_MASTER_ADS", factory_collector=factory_collector)
finally:
os.remove(tmpnam)
def deadvertizeFactoryClientMonitoring(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
"""
Deadvertize all monitoring classads for this factory.
"""
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_de_gfc')
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factoryclient_id)
fd.write('Requirements = (ReqFactoryName=?="%s")&&(ReqGlideinName=?="%s")&&(GlideinMyType == "%s")' % (factory_name, glidein_name, factoryConfig.factoryclient_id))
finally:
fd.close()
exe_condor_advertise(tmpnam, "INVALIDATE_LICENSE_ADS", factory_collector=factory_collector)
finally:
os.remove(tmpnam)
def deadvertizeFactory(factory_name, glidein_name, factory_collector=DEFAULT_VAL):
"""
Deadvertize all entry and global classads for this factory.
"""
tmpnam = classadSupport.generate_classad_filename(prefix="gfi_de_fact")
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % factoryConfig.factory_id)
fd.write('Requirements = (FactoryName =?= "%s")&&(GlideinName =?= "%s")\n' % (factory_name, glidein_name))
finally:
fd.close()
exe_condor_advertise(tmpnam, "INVALIDATE_ADS_GENERIC", factory_collector=factory_collector)
finally:
os.remove(tmpnam)
def deadvertizeAllWork(factory_pool, my_name):
"""
Removes all work requests for the client in the factory.
"""
global frontendConfig
tmpnam = classadSupport.generate_classad_filename(prefix="gfi_de_gc")
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "Query"\n')
fd.write('TargetType = "%s"\n' % frontendConfig.client_id)
fd.write(
'Requirements = (ClientName == "%s") && (GlideinMyType == "%s")\n' % (my_name, frontendConfig.client_id)
)
finally:
fd.close()
exe_condor_advertise(tmpnam, "INVALIDATE_MASTER_ADS", factory_pool)
finally:
os.remove(tmpnam)
def advertizeGlobal(
factory_name, glidein_name, supported_signtypes, pub_key_obj, stats_dict={}, factory_collector=DEFAULT_VAL
):
"""
Creates the glidefactoryglobal classad and advertises.
@type factory_name: string
@param factory_name: the name of the factory
@type glidein_name: string
@param glidein_name: name of the glidein
@type supported_signtypes: string
@param supported_signtypes: suppported sign types, i.e. sha1
@type pub_key_obj: GlideinKey
@param pub_key_obj: for the frontend to use in encryption
@type stats_dict: dict
@param stats_dict: completed jobs statistics
@type factory_collector: string or None
@param factory_collector: the collector to query, special value 'default' will get it from the global config
@todo add factory downtime?
"""
tmpnam = classadSupport.generate_classad_filename(prefix="gfi_ad_gfg")
gfg_classad = FactoryGlobalClassad(factory_name, glidein_name, supported_signtypes, pub_key_obj)
try:
gfg_classad.writeToFile(tmpnam, append=False)
exe_condor_advertise(tmpnam, gfg_classad.adAdvertiseCmd, factory_collector=factory_collector)
finally:
# Unable to write classad
try:
os.remove(tmpnam)
except:
# Do the possible to remove the file if there
pass
def advertizeGlidein(factory_name, glidein_name, entry_name, trust_domain,
auth_method, supported_signtypes, pub_key_obj,
glidein_attrs={}, glidein_params={}, glidein_monitors={},
factory_collector=DEFAULT_VAL):
"""
Creates the glideclient classad and advertises.
@type factory_name: string
@param factory_name: the name of the factory
@type glidein_name: string
@param glidein_name: name of the glidein
@type entry_name: string
@param entry_name: name of the entry
@type trust_domain: string
@param trust_domain: trust domain for this entry
@type auth_method: string
@param auth_method: the authentication methods this entry supports in glidein submission, i.e. grid_proxy
@type supported_signtypes: string
@param supported_signtypes: suppported sign types, i.e. sha1
@type glidein_attrs: dict
@param glidein_attrs: glidein attrs to be published, not be overwritten by Frontends
@type glidein_params: dict
@param glidein_params: params to be published, can be overwritten by Frontends
@type glidein_monitors: dict
@param glidein_monitors: monitor attrs to be published
@type pub_key_obj: GlideinKey
@param pub_key_obj: for the frontend to use in encryption
@type factory_collector: string or None
@param factory_collector: the collector to query, special value 'default' will get it from the global config
"""
global factoryConfig, advertizeGlideinCounter
tmpnam = classadSupport.generate_classad_filename(prefix='gfi_ad_gf')
fd = file(tmpnam, "w")
try:
try:
fd.write('MyType = "%s"\n' % factoryConfig.factory_id)
fd.write('GlideinMyType = "%s"\n' % factoryConfig.factory_id)
fd.write('GlideinWMSVersion = "%s"\n' % factoryConfig.glideinwms_version)
fd.write('Name = "%s@%s@%s"\n' % (entry_name, glidein_name, factory_name))
fd.write('FactoryName = "%s"\n' % factory_name)
fd.write('GlideinName = "%s"\n' % glidein_name)
fd.write('EntryName = "%s"\n' % entry_name)
fd.write('%s = "%s"\n' % (factoryConfig.factory_signtype_id, string.join(supported_signtypes, ',')))
# Must have a key to communicate
fd.write('PubKeyID = "%s"\n' % pub_key_obj.get_pub_key_id())
fd.write('PubKeyType = "%s"\n' % pub_key_obj.get_pub_key_type())
fd.write('PubKeyValue = "%s"\n' % string.replace(pub_key_obj.get_pub_key_value(), '\n', '\\n'))
if 'grid_proxy' in auth_method:
fd.write('GlideinAllowx509_Proxy = %s\n' % True)
fd.write('GlideinRequirex509_Proxy = %s\n' % True)
fd.write('GlideinRequireGlideinProxy = %s\n' % False)
else:
fd.write('GlideinAllowx509_Proxy = %s\n' % False)
fd.write('GlideinRequirex509_Proxy = %s\n' % False)
fd.write('GlideinRequireGlideinProxy = %s\n' % True)
fd.write('DaemonStartTime = %li\n' % start_time)
fd.write('UpdateSequenceNumber = %i\n' % advertizeGlideinCounter)
advertizeGlideinCounter += 1
# write out both the attributes, params and monitors
for (prefix, data) in ((factoryConfig.glidein_attr_prefix, glidein_attrs),
(factoryConfig.glidein_param_prefix, glidein_params),
(factoryConfig.glidein_monitor_prefix, glidein_monitors)):
for attr in data.keys():
el = data[attr]
if type(el) == type(1):
# don't quote ints
fd.write('%s%s = %s\n' % (prefix, attr, el))
else:
escaped_el = string.replace(string.replace(str(el), '"', '\\"'), '\n', '\\n')
fd.write('%s%s = "%s"\n' % (prefix, attr, escaped_el))
finally:
fd.close()
exe_condor_advertise(tmpnam, "UPDATE_MASTER_AD", factory_collector=factory_collector)
finally:
os.remove(tmpnam)
def iterate_one(do_advertize, factory_in_downtime, glideinDescript,
frontendDescript, group_name, my_entries):
"""
One iteration of the entry group
@type do_advertize: boolean
@param do_advertize: True if glidefactory classads should be advertised
@type factory_in_downtime: boolean
@param factory_in_downtime: True if factory is in downtime
@type glideinDescript: dict
@param glideinDescript: Factory glidein config values
@type frontendDescript: dict
@param frontendDescript: Security mappings for frontend identities, security classes, and usernames
@type group_name: string
@param group_name: Name of the group
@type my_entries: dict
@param my_entries: Dictionary of entry objects (glideFactoryEntry.Entry) keyed on entry name
"""
groupwork_done = {}
done_something = 0
for entry in my_entries.values():
entry.initIteration(factory_in_downtime)
try:
groupwork_done = find_and_perform_work(do_advertize,
factory_in_downtime,
glideinDescript,
frontendDescript, group_name,
my_entries)
except:
logSupport.log.warning(
"Error occurred while trying to find and do work.")
logSupport.log.exception("Exception: ")
logSupport.log.debug("Group Work done: %s" % groupwork_done)
# Classad files to use
gf_filename = classadSupport.generate_classad_filename(prefix='gfi_adm_gf')
gfc_filename = classadSupport.generate_classad_filename(
prefix='gfi_adm_gfc')
logSupport.log.info(
"Generating glidefactory (%s) and glidefactoryclient (%s) classads as needed"
% (gf_filename, gfc_filename))
entries_to_advertise = []
for entry in my_entries.values():
# Write classads to file if work was done or if advertise flag is set
# Actual advertise is done using multi classad advertisement
entrywork_done = 0
if ((entry.name in groupwork_done)
and ('work_done' in groupwork_done[entry.name])):
entrywork_done = groupwork_done[entry.name]['work_done']
done_something += entrywork_done
if ((do_advertize) or (entrywork_done > 0)):
entries_to_advertise.append(entry.name)
entry.writeClassadsToFile(factory_in_downtime, gf_filename,
gfc_filename)
entry.unsetInDowntime()
if ((do_advertize) or (done_something > 0)):
logSupport.log.debug(
"Generated glidefactory and glidefactoryclient classads for entries: %s"
% string.join(entries_to_advertise, ', '))
# ADVERTISE: glidefactory classads
gfi.advertizeGlideinFromFile(gf_filename,
remove_file=True,
is_multi=True)
# ADVERTISE: glidefactoryclient classads
gfi.advertizeGlideinClientMonitoringFromFile(gfc_filename,
remove_file=True,
is_multi=True)
else:
logSupport.log.info(
"Not advertising glidefactory and glidefactoryclient classads this round"
)
return done_something
def iterate_one(do_advertize, factory_in_downtime, glideinDescript,
frontendDescript, group_name, my_entries):
"""
One iteration of the entry group
@type do_advertize: boolean
@param do_advertize: True if glidefactory classads should be advertised
@type factory_in_downtime: boolean
@param factory_in_downtime: True if factory is in downtime
@type glideinDescript: dict
@param glideinDescript: Factory glidein config values
@type frontendDescript: dict
@param frontendDescript: Security mappings for frontend identities, security classes, and usernames for privsep
@type group_name: string
@param group_name: Name of the group
@type my_entries: dict
@param my_entries: Dictionary of entry objects keyed on entry name
"""
groupwork_done = {}
done_something = 0
for entry in my_entries.values():
entry.initIteration(factory_in_downtime)
try:
groupwork_done = find_and_perform_work(factory_in_downtime,
glideinDescript,
frontendDescript,
group_name, my_entries)
except:
logSupport.log.warning("Error occurred while trying to find and do work.")
logSupport.log.exception("Exception: ")
logSupport.log.debug("Group Work done: %s" % groupwork_done)
# Classad files to use
gf_filename = classadSupport.generate_classad_filename(prefix='gfi_adm_gf')
gfc_filename = classadSupport.generate_classad_filename(prefix='gfi_adm_gfc')
logSupport.log.info("Generating glidefactory (%s) and glidefactoryclient (%s) classads as needed" % (gf_filename, gfc_filename))
entries_to_advertise = []
for entry in my_entries.values():
# Write classads to file if work was done or if advertise flag is set
# Actual advertise is done using multi classad advertisement
entrywork_done = 0
if ( (entry.name in groupwork_done) and
('work_done' in groupwork_done[entry.name]) ):
entrywork_done = groupwork_done[entry.name]['work_done']
done_something += entrywork_done
if ( (do_advertize) or (entrywork_done > 0) ):
entries_to_advertise.append(entry.name)
entry.writeClassadsToFile(factory_in_downtime, gf_filename,
gfc_filename)
entry.unsetInDowntime()
if ((do_advertize) or (done_something > 0)):
logSupport.log.debug("Generated glidefactory and glidefactoryclient classads for entries: %s" % string.join(entries_to_advertise, ', '))
# ADVERTISE: glidefactory classads
gfi.advertizeGlideinFromFile(gf_filename,
remove_file=True,
is_multi=True)
# ADVERTISE: glidefactoryclient classads
gfi.advertizeGlideinClientMonitoringFromFile(gfc_filename,
remove_file=True,
is_multi=True)
else:
logSupport.log.info("Not advertising glidefactory and glidefactoryclient classads this round")
return done_something
def do_global_advertize(self):
"""
Advertize globals with credentials
"""
global advertizeGCGounter
for factory_pool in self.global_pool:
tmpname = classadSupport.generate_classad_filename(prefix="gfi_ad_gcg")
glidein_params_to_encrypt = {}
fd = file(tmpname, "w")
x509_proxies_data = []
if self.descript_obj.x509_proxies_plugin is not None:
x509_proxies_data = self.descript_obj.x509_proxies_plugin.get_credentials()
nr_credentials = len(x509_proxies_data)
glidein_params_to_encrypt["NumberOfCredentials"] = "%s" % nr_credentials
else:
nr_credentials = 0
request_name = "Global"
if factory_pool in self.global_params:
request_name, security_name = self.global_params[factory_pool]
glidein_params_to_encrypt["SecurityName"] = security_name
classad_name = "%s@%s" % (request_name, self.descript_obj.my_name)
fd.write('MyType = "%s"\n' % frontendConfig.client_global)
fd.write('GlideinMyType = "%s"\n' % frontendConfig.client_global)
fd.write('GlideinWMSVersion = "%s"\n' % frontendConfig.glideinwms_version)
fd.write('Name = "%s"\n' % classad_name)
fd.write('FrontendName = "%s"\n' % self.descript_obj.frontend_name)
fd.write('GroupName = "%s"\n' % self.descript_obj.group_name)
fd.write('ClientName = "%s"\n' % self.descript_obj.my_name)
for i in range(nr_credentials):
cred_el = x509_proxies_data[i]
cred_el.renew()
cred_el.advertize = True
if hasattr(cred_el, "filename"):
try:
if (not os.path.exists(cred_el.filename)) and (cred_el.creation_script is not None):
logSupport.log.debug(
"Proxy %s didn not exist, calling creation_script %s"
% (cred_el.filename, cred_el.creation_script)
)
condorExe.iexe_cmd(cred_el.creation_script)
data_fd = open(cred_el.filename)
cred_data = data_fd.read()
data_fd.close()
except:
cred_el.advertize = False
logSupport.log.exception("Advertising global credential %s failed" % cred_el.filename)
continue
glidein_params_to_encrypt[cred_el.file_id(cred_el.filename)] = cred_data
if hasattr(cred_el, "security_class"):
# Convert the sec class to a string so the Factory can interpret the value correctly
glidein_params_to_encrypt["SecurityClass" + cred_el.file_id(cred_el.filename)] = str(
cred_el.security_class
)
if hasattr(cred_el, "key_fname"):
try:
data_fd = open(cred_el.key_fname)
cred_data = data_fd.read()
data_fd.close()
except:
cred_el.advertize = False
logSupport.log.exception("Advertising global credential %s failed: " % cred_el.filename)
continue
glidein_params_to_encrypt[cred_el.file_id(cred_el.key_fname)] = cred_data
if hasattr(cred_el, "security_class"):
# Convert the sec class to a string so the Factory can interpret the value correctly
glidein_params_to_encrypt["SecurityClass" + cred_el.file_id(cred_el.key_fname)] = str(
cred_el.security_class
)
if hasattr(cred_el, "pilot_fname"):
try:
data_fd = open(cred_el.pilot_fname)
cred_data = data_fd.read()
data_fd.close()
except:
cred_el.advertize = False
logSupport.log.exception("Advertising global credential %s failed: " % cred_el.filename)
continue
glidein_params_to_encrypt[cred_el.file_id(cred_el.pilot_fname)] = cred_data
if hasattr(cred_el, "security_class"):
# Convert the sec class to a string so the Factory can interpret the value correctly
glidein_params_to_encrypt["SecurityClass" + cred_el.file_id(cred_el.pilot_fname)] = str(
cred_el.security_class
)
if factory_pool in self.global_key:
key_obj = self.global_key[factory_pool]
if key_obj is not None:
fd.write(string.join(key_obj.get_key_attrs(), "\n") + "\n")
for attr in glidein_params_to_encrypt.keys():
# logSupport.log.debug("Encrypting (%s,%s)"%(attr,glidein_params_to_encrypt[attr]))
el = key_obj.encrypt_hex(glidein_params_to_encrypt[attr])
escaped_el = string.replace(string.replace(str(el), '"', '\\"'), "\n", "\\n")
fd.write('%s%s = "%s"\n' % (frontendConfig.encrypted_param_prefix, attr, escaped_el))
# Update Sequence number information
if advertizeGCGounter.has_key(classad_name):
advertizeGCGounter[classad_name] += 1
else:
advertizeGCGounter[classad_name] = 0
fd.write("UpdateSequenceNumber = %s\n" % advertizeGCGounter[classad_name])
fd.close()
try:
advertizeWorkFromFile(factory_pool, tmpname, remove_file=True)
except condorExe.ExeError:
logSupport.log.exception("Advertising globals failed for factory pool %s: " % factory_pool)