def populate(self,params=None):
if params is None:
params=self.params
# put default files in place first
self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
self.dicts['preentry_file_list'].add_placeholder(cWConsts.GRIDMAP_FILE,allow_overwrite=True) # this one must be loaded before factory runs setup_x509.sh
# follow by the blacklist file
file_name = cWConsts.BLACKLIST_FILE
self.dicts['preentry_file_list'].add_from_file(file_name,
cWDictFile.FileDictFile.make_val_tuple(file_name, "nocache",
config_out='BLACKLIST_FILE'),
os.path.join(params.src_dir, file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh', 'check_blacklist.sh'):
self.dicts['preentry_file_list'].add_from_file(script_name,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), 'exec'),
os.path.join(params.src_dir, script_name))
# put user files in stage
for user_file in params.files:
add_file_unparsed(user_file,self.dicts, False)
# start expr is special
start_expr=None
# put user attributes into config files
for attr_name in params.attrs.keys():
if attr_name in ('GLIDECLIENT_Start','GLIDECLIENT_Group_Start'):
if start_expr is None:
start_expr=params.attrs[attr_name].value
elif not (params.attrs[attr_name].value in (None,'True')):
start_expr="(%s)&&(%s)"%(start_expr,params.attrs[attr_name].value)
# delete from the internal structure... will use it in match section
del params.data['attrs'][attr_name]
else:
add_attr_unparsed(attr_name, params,self.dicts,"main")
real_start_expr=params.match.start_expr
if start_expr is not None:
if real_start_expr!='True':
real_start_expr="(%s)&&(%s)"%(real_start_expr,start_expr)
else:
real_start_expr=start_expr
# since I removed the attributes, roll back into the match.start_expr
params.data['match']['start_expr']=real_start_expr
self.dicts['consts'].add('GLIDECLIENT_Start',real_start_expr)
# create GLIDEIN_Collector attribute
self.dicts['params'].add_extended('GLIDEIN_Collector', False, str(calc_glidein_collectors(params.collectors)))
# create GLIDEIN_CCB attribute only if CCBs list is in config file
tmp_glidein_ccbs_string = str(calc_glidein_ccbs(params.ccbs))
if tmp_glidein_ccbs_string:
self.dicts['params'].add_extended('GLIDEIN_CCB', False, tmp_glidein_ccbs_string)
populate_gridmap(params,self.dicts['gridmap'])
if self.dicts['preentry_file_list'].is_placeholder(cWConsts.GRIDMAP_FILE): # gridmapfile is optional, so if not loaded, remove the placeholder
self.dicts['preentry_file_list'].remove(cWConsts.GRIDMAP_FILE)
# populate complex files
populate_frontend_descript(self.work_dir,self.dicts['frontend_descript'],self.active_sub_list,params)
populate_common_descript(self.dicts['frontend_descript'],params)
# Apply multicore policy so frontend can deal with multicore
# glideins and requests correctly
apply_multicore_policy(self.dicts['frontend_descript'])
# populate the monitor files
javascriptrrd_dir = params.monitor.javascriptRRD_dir
for mfarr in ((params.src_dir,'frontend_support.js'),
(javascriptrrd_dir,'javascriptrrd.wlibs.js')):
mfdir,mfname=mfarr
parent_dir = self.find_parent_dir(mfdir,mfname)
mfobj=cWDictFile.SimpleFile(parent_dir,mfname)
mfobj.load()
self.monitor_jslibs.append(mfobj)
for mfarr in ((params.src_dir,'frontendRRDBrowse.html'),
(params.src_dir,'frontendRRDGroupMatrix.html'),
(params.src_dir,'frontendGroupGraphStatusNow.html'),
(params.src_dir,'frontendStatus.html')):
mfdir,mfname=mfarr
mfobj=cWDictFile.SimpleFile(mfdir,mfname)
mfobj.load()
self.monitor_htmls.append(mfobj)
mfobj = cWDictFile.SimpleFile(params.src_dir + '/frontend', 'index.html')
mfobj.load()
self.monitor_htmls.append(mfobj)
for imgfil in ('frontendGroupGraphsNow.small.png',
'frontendRRDBrowse.small.png',
'frontendRRDGroupMatix.small.png',
'frontendStatus.small.png'):
mfobj = cWDictFile.SimpleFile(params.src_dir + '/frontend/images', imgfil)
mfobj.load()
self.monitor_htmls.append(mfobj)
# Tell condor to advertise GLIDECLIENT_ReqNode
self.dicts['vars'].add_extended('GLIDECLIENT_ReqNode','string',None,None,False,True,False)
# derive attributes
populate_common_attrs(self.dicts)
# populate security data
populate_main_security(self.client_security,params)
def populate(self,params=None):
if params is None:
params=self.params
# put default files in place first
self.dicts['file_list'].add_placeholder('error_gen.sh',allow_overwrite=True)
self.dicts['file_list'].add_placeholder('error_augment.sh',allow_overwrite=True)
self.dicts['file_list'].add_placeholder('setup_script.sh',allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
self.dicts['file_list'].add_placeholder(cWConsts.GRIDMAP_FILE,allow_overwrite=True) # this one must be loaded before setup_x509.sh is run
#load system files
for file_name in ('error_gen.sh', 'error_augment.sh', 'parse_starterlog.awk', 'advertise_failure.helper',
'condor_config', 'condor_config.multi_schedd.include',
'condor_config.dedicated_starter.include', 'condor_config.check.include',
'condor_config.monitor.include'):
self.dicts['file_list'].add_from_file(file_name,
(cWConsts.insert_timestr(file_name), 'regular', 0, 'TRUE', 'FALSE'),
os.path.join(params.src_dir, file_name))
self.dicts['description'].add("condor_config","condor_config")
self.dicts['description'].add("condor_config.multi_schedd.include","condor_config_multi_include")
self.dicts['description'].add("condor_config.dedicated_starter.include","condor_config_main_include")
self.dicts['description'].add("condor_config.monitor.include","condor_config_monitor_include")
self.dicts['description'].add("condor_config.check.include","condor_config_check_include")
self.dicts['vars'].load(params.src_dir,'condor_vars.lst',change_self=False,set_not_changed=False)
#
# Note:
# We expect the condor platform info to be coming in as parameters
# as FE provided consts file is not available at this time
#
# add the basic standard params
self.dicts['params'].add("GLIDEIN_Report_Failed",'NEVER')
self.dicts['params'].add("CONDOR_OS",'default')
self.dicts['params'].add("CONDOR_ARCH",'default')
self.dicts['params'].add("CONDOR_VERSION",'default')
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('setup_script.sh', 'cat_consts.sh', 'condor_platform_select.sh'):
self.dicts['file_list'].add_from_file(script_name,
(cWConsts.insert_timestr(script_name), 'exec', 0, 'TRUE', 'FALSE'),
os.path.join(params.src_dir, script_name))
#load condor tarballs
# only one will be downloaded in the end... based on what condor_platform_select.sh decides
for condor_idx in range(len(params.condor_tarballs)):
condor_el=params.condor_tarballs[condor_idx]
# condor_el now is a combination of csv version+os+arch
# Get list of valid tarballs for this condor_el
# Register the tarball, but make download conditional to cond_name
condor_el_valid_tarballs = get_valid_condor_tarballs([condor_el])
condor_fname = cWConsts.insert_timestr(cgWConsts.CONDOR_FILE % condor_idx)
condor_tarfile = ""
condor_fd = None
if condor_el.tar_file is not None:
# Condor tarball available. Just add it to the list of tarballs
# with every possible condor_platform string
condor_tarfile = condor_el.tar_file
else:
# Create a new tarball as usual
condor_fd = cgWCreate.create_condor_tar_fd(condor_el.base_dir)
condor_tarfile = os.path.join(self.dicts['file_list'].dir,
condor_fname)
# insert the newly created tarball fname back into the config
params.subparams.data['condor_tarballs'][condor_idx]['tar_file'] = condor_tarfile
for tar in condor_el_valid_tarballs:
condor_platform = "%s-%s-%s" % (tar['version'], tar['os'],
tar['arch'])
cond_name = "CONDOR_PLATFORM_%s" % condor_platform
condor_platform_fname = cgWConsts.CONDOR_FILE % condor_platform
if condor_fd is None:
# tar file exists. Just use it
self.dicts['file_list'].add_from_file(
condor_platform_fname,
(condor_fname, 'untar', 0, cond_name, cgWConsts.CONDOR_ATTR),
condor_el.tar_file
)
else:
# This is addition of new tarfile
# Need to rewind fd everytime
condor_fd.seek(0)
self.dicts['file_list'].add_from_fd(
condor_platform_fname,
(condor_fname, 'untar', 0, cond_name, cgWConsts.CONDOR_ATTR),
condor_fd
)
self.dicts['untar_cfg'].add(condor_platform_fname,
cgWConsts.CONDOR_DIR)
# Add cond_name in the config, so that it is known
# But leave it disabled by default
self.dicts['consts'].add(cond_name, "0",
allow_overwrite=False)
if condor_fd is not None:
condor_fd.close()
#
# Note:
# We expect the collector info to be coming in as parameter
# as FE consts file is not available at this time
#
# add the basic standard params
self.dicts['params'].add("GLIDEIN_Collector",'Fake')
# add the factory monitoring collector parameter, if any collectors are defined
# this is purely a factory thing
factory_monitoring_collector=calc_monitoring_collectors_string(params.monitoring_collectors)
if factory_monitoring_collector is not None:
self.dicts['params'].add('GLIDEIN_Factory_Collector',str(factory_monitoring_collector))
populate_gridmap(params,self.dicts['gridmap'])
file_list_scripts = ['collector_setup.sh',
'create_temp_mapfile.sh',
'setup_x509.sh',
cgWConsts.CONDOR_STARTUP_FILE]
after_file_list_scripts = ['check_proxy.sh',
'create_mapfile.sh',
'validate_node.sh',
'setup_network.sh',
'gcb_setup.sh',
'glexec_setup.sh',
'java_setup.sh',
'glidein_memory_setup.sh',
'glidein_cpus_setup.sh',
'glidein_sitewms_setup.sh',
'script_wrapper.sh',
'smart_partitionable.sh']
# Only execute scripts once
duplicate_scripts = set(file_list_scripts).intersection(after_file_list_scripts)
if duplicate_scripts:
raise RuntimeError, "Duplicates found in the list of files to execute '%s'" % ','.join(duplicate_scripts)
# Load more system scripts
for script_name in file_list_scripts:
self.dicts['file_list'].add_from_file(script_name,
(cWConsts.insert_timestr(script_name), 'exec', 0, 'TRUE', 'FALSE'),
os.path.join(params.src_dir, script_name))
# make sure condor_startup does not get executed ahead of time under normal circumstances
# but must be loaded early, as it also works as a reporting script in case of error
self.dicts['description'].add(cgWConsts.CONDOR_STARTUP_FILE,"last_script")
#
# At this point in the glideins, condor_advertize should be able to talk to the FE collector
#
# put user files in stage
for file in params.files:
add_file_unparsed(file, self.dicts, True)
# put user attributes into config files
for attr_name in params.attrs.keys():
add_attr_unparsed(attr_name, params,self.dicts,"main")
# add additional system scripts
for script_name in after_file_list_scripts:
self.dicts['after_file_list'].add_from_file(script_name,
(cWConsts.insert_timestr(script_name), 'exec', 0, 'TRUE', 'FALSE'),
os.path.join(params.src_dir, script_name))
# populate complex files
populate_factory_descript(self.work_dir,self.dicts['glidein'],self.active_sub_list,params)
populate_frontend_descript(self.dicts['frontend_descript'],params)
# populate the monitor files
javascriptrrd_dir = params.monitor.javascriptRRD_dir
for mfarr in ((params.src_dir,'factory_support.js'),
(javascriptrrd_dir,'javascriptrrd.wlibs.js')):
mfdir,mfname=mfarr
parent_dir = self.find_parent_dir(mfdir,mfname)
mfobj=cWDictFile.SimpleFile(parent_dir,mfname)
mfobj.load()
self.monitor_jslibs.append(mfobj)
for mfarr in ((params.src_dir,'factoryRRDBrowse.html'),
(params.src_dir,'factoryRRDEntryMatrix.html'),
(params.src_dir,'factoryStatus.html'),
(params.src_dir,'factoryLogStatus.html'),
(params.src_dir,'factoryCompletedStats.html'),
(params.src_dir,'factoryStatusNow.html'),
(params.src_dir,'factoryEntryStatusNow.html')):
mfdir,mfname=mfarr
mfobj=cWDictFile.SimpleFile(mfdir,mfname)
mfobj.load()
self.monitor_htmls.append(mfobj)
# add the index page and its images
mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/', 'index.html')
mfobj.load()
self.monitor_htmls.append(mfobj)
for imgfile in ('factoryCompletedStats.png',
'factoryEntryStatusNow.png',
'factoryLogStatus.png',
'factoryRRDBrowse.png',
'factoryRRDEntryMatrix.png',
'factoryStatus.png',
'factoryStatusNow.png'):
mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/images/', imgfile)
mfobj.load()
self.monitor_images.append(mfobj)
def populate(self,params=None):
if params is None:
params=self.params
sub_params=params.groups[self.sub_name]
# put default files in place first
self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE, allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE, allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE, allow_overwrite=True) # this one must be loaded before any tarball
# follow by the blacklist file
file_name = cWConsts.BLACKLIST_FILE
self.dicts['preentry_file_list'].add_from_file(file_name,
cWDictFile.FileDictFile.make_val_tuple(file_name, "nocache",
config_out='BLACKLIST_FILE'),
os.path.join(params.src_dir, file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh', "check_blacklist.sh"):
self.dicts['preentry_file_list'].add_from_file(script_name,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), 'exec'),
os.path.join(params.src_dir, script_name))
# put user files in stage
for user_file in sub_params.files:
add_file_unparsed(user_file, self.dicts, False)
# start expr is special
start_expr=None
# put user attributes into config files
for attr_name in sub_params.attrs.keys():
if attr_name in ('GLIDECLIENT_Group_Start','GLIDECLIENT_Start'):
if start_expr is None:
start_expr=sub_params.attrs[attr_name].value
elif sub_params.attrs[attr_name].value is not None:
start_expr="(%s)&&(%s)"%(start_expr,sub_params.attrs[attr_name].value)
# delete from the internal structure... will use it in match section
del sub_params.data['attrs'][attr_name]
else:
add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name)
real_start_expr=sub_params.match.start_expr
if start_expr is not None:
if real_start_expr!='True':
real_start_expr="(%s)&&(%s)"%(real_start_expr,start_expr)
else:
real_start_expr=start_expr
# since I removed the attributes, roll back into the match.start_expr
sub_params.data['match']['start_expr']=real_start_expr
self.dicts['consts'].add('GLIDECLIENT_Group_Start',real_start_expr)
# derive attributes
populate_common_attrs(self.dicts)
# populate complex files
populate_group_descript(self.work_dir,self.dicts['group_descript'],
self.sub_name,sub_params)
populate_common_descript(self.dicts['group_descript'],sub_params)
# Apply group specific glexec policy
apply_group_glexec_policy(self.dicts['group_descript'], sub_params, params)
# populate security data
populate_main_security(self.client_security,params)
populate_group_security(self.client_security,params,sub_params)
def populate(self,params=None):
if params is None:
params=self.params
sub_params=params.entries[self.sub_name]
# put default files in place first
self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
# follow by the blacklist file
file_name=cWConsts.BLACKLIST_FILE
self.dicts['file_list'].add_from_file(file_name,
(file_name, 'nocache', 0, 'TRUE', 'BLACKLIST_FILE'),
os.path.join(params.src_dir, file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh', 'check_blacklist.sh'):
self.dicts['file_list'].add_from_file(script_name,
(cWConsts.insert_timestr(script_name), 'exec', 0, 'TRUE', 'FALSE'),
os.path.join(params.src_dir, script_name))
#load system files
self.dicts['vars'].load(params.src_dir,'condor_vars.lst.entry',change_self=False,set_not_changed=False)
# put user files in stage
for user_file in sub_params.files:
add_file_unparsed(user_file, self.dicts, True)
# Add attribute for voms
# put user attributes into config files
for attr_name in sub_params.attrs.keys():
add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name)
# put standard attributes into config file
# override anything the user set
for dtype in ('attrs','consts'):
self.dicts[dtype].add("GLIDEIN_Gatekeeper",sub_params.gatekeeper,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_GridType",sub_params.gridtype,allow_overwrite=True)
# MERGENOTE:
# GLIDEIN_REQUIRE_VOMS publishes an attribute so that users without VOMS proxies
# can avoid sites that require VOMS proxies (using the normal Condor Requirements
# string.
self.dicts[dtype].add("GLIDEIN_REQUIRE_VOMS",sub_params.config.restrictions.require_voms_proxy,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_REQUIRE_GLEXEC_USE",sub_params.config.restrictions.require_glidein_glexec_use,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_TrustDomain",sub_params.trust_domain,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_SupportedAuthenticationMethod",sub_params.auth_method,allow_overwrite=True)
if sub_params.rsl is not None:
self.dicts[dtype].add('GLIDEIN_GlobusRSL',sub_params.rsl,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_SlotsLayout", sub_params.config.submit.slots_layout, allow_overwrite=True)
self.dicts['vars'].add_extended("GLIDEIN_REQUIRE_VOMS","boolean",sub_params.config.restrictions.require_voms_proxy,None,False,True,True)
self.dicts['vars'].add_extended("GLIDEIN_REQUIRE_GLEXEC_USE","boolean",sub_params.config.restrictions.require_glidein_glexec_use,None,False,True,True)
# populate infosys
for infosys_ref in sub_params.infosys_refs:
self.dicts['infosys'].add_extended(infosys_ref['type'],infosys_ref['server'],infosys_ref['ref'],allow_overwrite=True)
# populate monitorgroups
for monitorgroup in sub_params.monitorgroups:
self.dicts['mongroup'].add_extended(monitorgroup['group_name'],allow_overwrite=True)
# populate complex files
populate_job_descript(self.work_dir,self.dicts['job_descript'],
self.sub_name,sub_params)
################################################################################################################
# This is the original function call:
#
# self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE,
# params.factory_name,params.glidein_name,self.sub_name,
# sub_params.gridtype,sub_params.gatekeeper, sub_params.rsl, sub_params.auth_method,
# params.web_url,sub_params.proxy_url,sub_params.work_dir,
# params.submit.base_client_log_dir, sub_params.submit.submit_attrs)
#
# Almost all of the parameters are attributes of params and/or sub_params. Instead of maintaining an ever
# increasing parameter list for this function, lets just pass params, sub_params, and the 2 other parameters
# to the function and call it a day.
################################################################################################################
self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE, self.sub_name, params, sub_params)
