def get_main_dicts(submit_dir,stage_dir):
main_dicts=get_common_dicts(submit_dir,stage_dir)
main_dicts['summary_signature']=cWDictFile.SummarySHA1DictFile(submit_dir,cWConsts.SUMMARY_SIGNATURE_FILE)
main_dicts['glidein']=cWDictFile.StrDictFile(submit_dir,cgWConsts.GLIDEIN_FILE)
main_dicts['frontend_descript']=cWDictFile.ReprDictFile(submit_dir,cgWConsts.FRONTEND_DESCRIPT_FILE)
main_dicts['gridmap']=cWDictFile.GridMapDict(stage_dir,cWConsts.insert_timestr(cWConsts.GRIDMAP_FILE))
main_dicts['after_file_list']=cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cgWConsts.AFTER_FILE_LISTFILE),fname_idx=cgWConsts.AFTER_FILE_LISTFILE)
return main_dicts
def get_main_dicts(work_dir,stage_dir,simple_work_dir,assume_groups):
main_dicts=get_common_dicts(work_dir,stage_dir,simple_work_dir)
main_dicts['summary_signature']=cWDictFile.SummarySHA1DictFile(work_dir,cWConsts.SUMMARY_SIGNATURE_FILE)
main_dicts['frontend_descript']=cWDictFile.StrDictFile(work_dir,cvWConsts.FRONTEND_DESCRIPT_FILE)
main_dicts['gridmap']=GridMapDict(stage_dir,cWConsts.insert_timestr(cvWConsts.GRIDMAP_FILE))
if assume_groups:
main_dicts['aftergroup_file_list']=cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cvWConsts.AFTERGROUP_FILE_LISTFILE),fname_idx=cvWConsts.AFTERGROUP_FILE_LISTFILE)
main_dicts['aftergroup_preentry_file_list']=cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cvWConsts.AFTERGROUP_PREENTRY_FILE_LISTFILE),fname_idx=cvWConsts.AFTERGROUP_PREENTRY_FILE_LISTFILE)
return main_dicts
def populate(self,params=None):
if params==None:
params=self.params
sub_params=params.entries[self.sub_name]
# put default files in place first
self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
# follow by the blacklist file
file_name=cWConsts.BLACKLIST_FILE
self.dicts['file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh',"check_blacklist.sh"):
self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
#load system files
self.dicts['vars'].load(params.src_dir,'condor_vars.lst.entry',change_self=False,set_not_changed=False)
# put user files in stage
for file in sub_params.files:
add_file_unparsed(file,self.dicts)
# put user attributes into config files
for attr_name in sub_params.attrs.keys():
add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name)
# put standard attributes into config file
# override anything the user set
for dtype in ('attrs','consts'):
self.dicts[dtype].add("GLIDEIN_Gatekeeper",sub_params.gatekeeper,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_GridType",sub_params.gridtype,allow_overwrite=True)
if sub_params.rsl!=None:
self.dicts[dtype].add('GLIDEIN_GlobusRSL',sub_params.rsl,allow_overwrite=True)
# populate infosys
for infosys_ref in sub_params.infosys_refs:
self.dicts['infosys'].add_extended(infosys_ref['type'],infosys_ref['server'],infosys_ref['ref'],allow_overwrite=True)
# populate monitorgroups
for monitorgroup in sub_params.monitorgroups:
self.dicts['mongroup'].add_extended(monitorgroup['group_name'],allow_overwrite=True)
# populate complex files
populate_job_descript(self.work_dir,self.dicts['job_descript'],
self.sub_name,sub_params)
self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE,
params.factory_name,params.glidein_name,self.sub_name,
sub_params.gridtype,sub_params.gatekeeper,sub_params.rsl,
params.web_url,sub_params.proxy_url,sub_params.work_dir,
params.submit.base_client_log_dir)
def get_common_dicts(work_dir,stage_dir,
simple_work_dir): # if True, do not create params
common_dicts={'description':cWDictFile.DescriptionDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.DESCRIPTION_FILE),fname_idx=cWConsts.DESCRIPTION_FILE),
'consts':cWDictFile.StrDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.CONSTS_FILE),fname_idx=cWConsts.CONSTS_FILE),
'vars':cWDictFile.VarsDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.VARS_FILE),fname_idx=cWConsts.VARS_FILE),
'untar_cfg':cWDictFile.StrDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.UNTAR_CFG_FILE),fname_idx=cWConsts.UNTAR_CFG_FILE),
'file_list':cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.FILE_LISTFILE),fname_idx=cWConsts.FILE_LISTFILE),
'preentry_file_list':cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cvWConsts.PREENTRY_FILE_LISTFILE),fname_idx=cvWConsts.PREENTRY_FILE_LISTFILE),
"signature":cWDictFile.SHA1DictFile(stage_dir,cWConsts.insert_timestr(cWConsts.SIGNATURE_FILE),fname_idx=cWConsts.SIGNATURE_FILE)}
if not simple_work_dir:
common_dicts['params']=ParamsDictFile(work_dir,cvWConsts.PARAMS_FILE)
refresh_description(common_dicts)
return common_dicts
def get_common_dicts(submit_dir,stage_dir):
common_dicts={'attrs':cWDictFile.ReprDictFile(submit_dir,cgWConsts.ATTRS_FILE),
'description':cWDictFile.DescriptionDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.DESCRIPTION_FILE),fname_idx=cWConsts.DESCRIPTION_FILE),
'consts':cWDictFile.StrDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.CONSTS_FILE),fname_idx=cWConsts.CONSTS_FILE),
'params':cWDictFile.ReprDictFile(submit_dir,cgWConsts.PARAMS_FILE),
'vars':cWDictFile.VarsDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.VARS_FILE),fname_idx=cWConsts.VARS_FILE),
'untar_cfg':cWDictFile.StrDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.UNTAR_CFG_FILE),fname_idx=cWConsts.UNTAR_CFG_FILE),
'file_list':cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.FILE_LISTFILE),fname_idx=cWConsts.FILE_LISTFILE),
"signature":cWDictFile.SHA1DictFile(stage_dir,cWConsts.insert_timestr(cWConsts.SIGNATURE_FILE),fname_idx=cWConsts.SIGNATURE_FILE)}
refresh_description(common_dicts)
return common_dicts
def populate(self,params=None):
if params==None:
params=self.params
sub_params=params.groups[self.sub_name]
# put default files in place first
self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
# follow by the blacklist file
file_name=cWConsts.BLACKLIST_FILE
self.dicts['preentry_file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh',"check_blacklist.sh"):
self.dicts['preentry_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
# put user files in stage
for file in sub_params.files:
add_file_unparsed(file,self.dicts)
# put user attributes into config files
for attr_name in sub_params.attrs.keys():
add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name)
# populate complex files
populate_group_descript(self.work_dir,self.dicts['group_descript'],
self.sub_name,sub_params)
populate_common_descript(self.dicts['group_descript'],sub_params)
# populate security data
populate_main_security(self.client_security,params)
populate_group_security(self.client_security,params,sub_params)
def populate(self,params=None):
if params==None:
params=self.params
# put default files in place first
self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
self.dicts['preentry_file_list'].add_placeholder(cWConsts.GRIDMAP_FILE,allow_overwrite=True) # this one must be loaded before factory runs setup_x509.sh
# follow by the blacklist file
file_name=cWConsts.BLACKLIST_FILE
self.dicts['preentry_file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh',"check_blacklist.sh"):
self.dicts['preentry_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
# put user files in stage
for file in params.files:
add_file_unparsed(file,self.dicts)
# put user attributes into config files
for attr_name in params.attrs.keys():
add_attr_unparsed(attr_name, params,self.dicts,"main")
# create GLIDEIN_Collector attribute
self.dicts['params'].add_extended('GLIDEIN_Collector',False,str(calc_glidein_collectors(params.collectors)))
populate_gridmap(params,self.dicts['gridmap'])
if self.dicts['preentry_file_list'].is_placeholder(cWConsts.GRIDMAP_FILE): # gridmapfile is optional, so if not loaded, remove the placeholder
self.dicts['preentry_file_list'].remove(cWConsts.GRIDMAP_FILE)
# populate complex files
populate_frontend_descript(self.work_dir,self.dicts['frontend_descript'],self.active_sub_list,params)
populate_common_descript(self.dicts['frontend_descript'],params)
# populate the monitor files
javascriptrrd_dir = params.monitor.javascriptRRD_dir
for mfarr in ((params.src_dir,'frontend_support.js'),
(javascriptrrd_dir,'rrdFlot.js'),
(javascriptrrd_dir,'rrdFlotMatrix.js'),
(javascriptrrd_dir,'rrdFlotSupport.js'),
(javascriptrrd_dir,'rrdFile.js'),
(javascriptrrd_dir,'rrdFilter.js'),
(javascriptrrd_dir,'binaryXHR.js'),
(params.monitor.flot_dir,'jquery.flot.js'),
(params.monitor.flot_dir,'jquery.flot.selection.js'),
(params.monitor.flot_dir,'excanvas.js'),
(params.monitor.jquery_dir,'jquery.js')):
mfdir,mfname=mfarr
parent_dir = self.find_parent_dir(mfdir,mfname)
mfobj=cWDictFile.SimpleFile(parent_dir,mfname)
mfobj.load()
self.monitor_jslibs.append(mfobj)
for mfarr in ((params.src_dir,'frontendRRDBrowse.html'),
(params.src_dir,'frontendRRDGroupMatrix.html'),
(params.src_dir,'frontendGroupGraphStatusNow.html'),
(params.src_dir,'frontendStatus.html')):
mfdir,mfname=mfarr
mfobj=cWDictFile.SimpleFile(mfdir,mfname)
mfobj.load()
self.monitor_htmls.append(mfobj)
# populate security data
populate_main_security(self.client_security,params)
def populate(self,params=None):
if params is None:
params=self.params
# put default files in place first
self.dicts['file_list'].add_placeholder('error_gen.sh',allow_overwrite=True)
self.dicts['file_list'].add_placeholder('error_augment.sh',allow_overwrite=True)
self.dicts['file_list'].add_placeholder('setup_script.sh',allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
#load system files
for file_name in ('error_gen.sh','error_augment.sh','parse_starterlog.awk', 'advertise_failure.helper',
"condor_config", "condor_config.multi_schedd.include", "condor_config.dedicated_starter.include", "condor_config.check.include", "condor_config.monitor.include"):
self.dicts['file_list'].add_from_file(file_name,(cWConsts.insert_timestr(file_name),"regular","TRUE",'FALSE'),os.path.join(params.src_dir,file_name))
self.dicts['description'].add("condor_config","condor_config")
self.dicts['description'].add("condor_config.multi_schedd.include","condor_config_multi_include")
self.dicts['description'].add("condor_config.dedicated_starter.include","condor_config_main_include")
self.dicts['description'].add("condor_config.monitor.include","condor_config_monitor_include")
self.dicts['description'].add("condor_config.check.include","condor_config_check_include")
self.dicts['vars'].load(params.src_dir,'condor_vars.lst',change_self=False,set_not_changed=False)
#
# Note:
# We expect the condor platform info to be coming in as parameters
# as FE provided consts file is not available at this time
#
# add the basic standard params
self.dicts['params'].add("GLIDEIN_Report_Failed",'NEVER')
self.dicts['params'].add("CONDOR_OS",'default')
self.dicts['params'].add("CONDOR_ARCH",'default')
self.dicts['params'].add("CONDOR_VERSION",'default')
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('setup_script.sh','cat_consts.sh','condor_platform_select.sh'):
self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
#load condor tarballs
# only one will be downloaded in the end... based on what condor_platform_select.sh decides
for condor_idx in range(len(params.condor_tarballs)):
condor_el=params.condor_tarballs[condor_idx]
# condor_el now is a combination of csv version+os+arch
# Get list of valid tarballs for this condor_el
# Register the tarball, but make download conditional to cond_name
condor_el_valid_tarballs = get_valid_condor_tarballs([condor_el])
condor_fname = cWConsts.insert_timestr(cgWConsts.CONDOR_FILE % condor_idx)
condor_tarfile = ""
condor_fd = None
if condor_el.tar_file is not None:
# Condor tarball available. Just add it to the list of tarballs
# with every possible condor_platform string
condor_tarfile = condor_el.tar_file
else:
# Create a new tarball as usual
condor_fd = cgWCreate.create_condor_tar_fd(condor_el.base_dir)
condor_tarfile = os.path.join(self.dicts['file_list'].dir,
condor_fname)
# insert the newly created tarball fname back into the config
params.subparams.data['condor_tarballs'][condor_idx]['tar_file'] = condor_tarfile
for tar in condor_el_valid_tarballs:
condor_platform = "%s-%s-%s" % (tar['version'], tar['os'],
tar['arch'])
cond_name = "CONDOR_PLATFORM_%s" % condor_platform
condor_platform_fname = cgWConsts.CONDOR_FILE % condor_platform
if condor_fd is None:
# tar file exists. Just use it
self.dicts['file_list'].add_from_file(
condor_platform_fname, (condor_fname,
"untar", cond_name,
cgWConsts.CONDOR_ATTR),
condor_el.tar_file)
else:
# This is addition of new tarfile
# Need to rewind fd everytime
condor_fd.seek(0)
self.dicts['file_list'].add_from_fd(
condor_platform_fname,
(condor_fname,"untar",cond_name,cgWConsts.CONDOR_ATTR),
condor_fd)
self.dicts['untar_cfg'].add(condor_platform_fname,
cgWConsts.CONDOR_DIR)
# Add cond_name in the config, so that it is known
# But leave it disabled by default
self.dicts['consts'].add(cond_name, "0",
allow_overwrite=False)
if condor_fd is not None:
condor_fd.close()
#
# Note:
# We expect the collector info to be coming in as parameter
# as FE consts file is not available at this time
#
# add the basic standard params
self.dicts['params'].add("GLIDEIN_Collector",'Fake')
file_list_scripts = ['collector_setup.sh',
'create_temp_mapfile.sh',
'setup_x509.sh',
cgWConsts.CONDOR_STARTUP_FILE]
after_file_list_scripts = ['check_proxy.sh',
'create_mapfile.sh',
'validate_node.sh',
'gcb_setup.sh',
'glexec_setup.sh',
'java_setup.sh',
'glidein_memory_setup.sh',
'glidein_cpus_setup.sh']
# Only execute scripts once
duplicate_scripts = set(file_list_scripts).intersection(after_file_list_scripts)
if duplicate_scripts:
raise RuntimeError, "Duplicates found in the list of files to execute '%s'" % ','.join(duplicate_scripts)
# Load more system scripts
for script_name in file_list_scripts:
self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
# make sure condor_startup does not get executed ahead of time under normal circumstances
# but must be loaded early, as it also works as a reporting script in case of error
self.dicts['description'].add(cgWConsts.CONDOR_STARTUP_FILE,"last_script")
#
# At this point in the glideins, condor_advertize should be able to talk to the FE collector
#
# put user files in stage
for file in params.files:
add_file_unparsed(file,self.dicts)
# put user attributes into config files
for attr_name in params.attrs.keys():
add_attr_unparsed(attr_name, params,self.dicts,"main")
# add additional system scripts
for script_name in after_file_list_scripts:
self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
# populate complex files
populate_factory_descript(self.work_dir,self.dicts['glidein'],self.active_sub_list,params)
populate_frontend_descript(self.dicts['frontend_descript'],params)
# populate the monitor files
javascriptrrd_dir = params.monitor.javascriptRRD_dir
for mfarr in ((params.src_dir,'factory_support.js'),
(javascriptrrd_dir,'rrdFlot.js'),
(javascriptrrd_dir,'rrdFlotMatrix.js'),
(javascriptrrd_dir,'rrdFlotSupport.js'),
(javascriptrrd_dir,'rrdFile.js'),
(javascriptrrd_dir,'rrdMultiFile.js'),
(javascriptrrd_dir,'rrdFilter.js'),
(javascriptrrd_dir,'binaryXHR.js'),
(params.monitor.flot_dir,'jquery.flot.js'),
(params.monitor.flot_dir,'jquery.flot.selection.js'),
(params.monitor.flot_dir,'jquery.flot.tooltip.js'),
(params.monitor.flot_dir,'excanvas.js'),
(params.monitor.jquery_dir,'jquery.js')):
mfdir,mfname=mfarr
parent_dir = self.find_parent_dir(mfdir,mfname)
mfobj=cWDictFile.SimpleFile(parent_dir,mfname)
mfobj.load()
self.monitor_jslibs.append(mfobj)
for mfarr in ((params.src_dir,'factoryRRDBrowse.html'),
(params.src_dir,'factoryRRDEntryMatrix.html'),
(params.src_dir,'factoryStatus.html'),
(params.src_dir,'factoryLogStatus.html'),
(params.src_dir,'factoryCompletedStats.html'),
(params.src_dir,'factoryStatusNow.html'),
(params.src_dir,'factoryEntryStatusNow.html')):
mfdir,mfname=mfarr
mfobj=cWDictFile.SimpleFile(mfdir,mfname)
mfobj.load()
self.monitor_htmls.append(mfobj)
# add the index page and its images
mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/', 'index.html')
mfobj.load()
self.monitor_htmls.append(mfobj)
for imgfile in ('factoryCompletedStats.png',
'factoryEntryStatusNow.png',
'factoryLogStatus.png',
'factoryRRDBrowse.png',
'factoryRRDEntryMatrix.png',
'factoryStatus.png',
'factoryStatusNow.png'):
mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/images/', imgfile)
mfobj.load()
self.monitor_htmls.append(mfobj)
def populate(self,params=None):
if params is None:
params=self.params
sub_params=params.entries[self.sub_name]
# put default files in place first
self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
# follow by the blacklist file
file_name=cWConsts.BLACKLIST_FILE
self.dicts['file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh',"check_blacklist.sh"):
self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
#load system files
self.dicts['vars'].load(params.src_dir,'condor_vars.lst.entry',change_self=False,set_not_changed=False)
# put user files in stage
for user_file in sub_params.files:
add_file_unparsed(user_file,self.dicts)
# Add attribute for voms
# put user attributes into config files
for attr_name in sub_params.attrs.keys():
add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name)
# put standard attributes into config file
# override anything the user set
for dtype in ('attrs','consts'):
self.dicts[dtype].add("GLIDEIN_Gatekeeper",sub_params.gatekeeper,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_GridType",sub_params.gridtype,allow_overwrite=True)
# MERGENOTE:
# GLIDEIN_REQUIRE_VOMS publishes an attribute so that users without VOMS proxies
# can avoid sites that require VOMS proxies (using the normal Condor Requirements
# string.
self.dicts[dtype].add("GLIDEIN_REQUIRE_VOMS",sub_params.config.restrictions.require_voms_proxy,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_REQUIRE_GLEXEC_USE",sub_params.config.restrictions.require_glidein_glexec_use,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_TrustDomain",sub_params.trust_domain,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_SupportedAuthenticationMethod",sub_params.auth_method,allow_overwrite=True)
if sub_params.rsl is not None:
self.dicts[dtype].add('GLIDEIN_GlobusRSL',sub_params.rsl,allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_SlotsLayout", sub_params.config.submit.slots_layout, allow_overwrite=True)
self.dicts['vars'].add_extended("GLIDEIN_REQUIRE_VOMS","boolean",sub_params.config.restrictions.require_voms_proxy,None,False,True,True)
self.dicts['vars'].add_extended("GLIDEIN_REQUIRE_GLEXEC_USE","boolean",sub_params.config.restrictions.require_glidein_glexec_use,None,False,True,True)
# populate infosys
for infosys_ref in sub_params.infosys_refs:
self.dicts['infosys'].add_extended(infosys_ref['type'],infosys_ref['server'],infosys_ref['ref'],allow_overwrite=True)
# populate monitorgroups
for monitorgroup in sub_params.monitorgroups:
self.dicts['mongroup'].add_extended(monitorgroup['group_name'],allow_overwrite=True)
# populate complex files
populate_job_descript(self.work_dir,self.dicts['job_descript'],
self.sub_name,sub_params)
self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE,
params.factory_name,params.glidein_name,self.sub_name,
sub_params.gridtype,sub_params.gatekeeper, sub_params.rsl, sub_params.auth_method,
params.web_url,sub_params.proxy_url,sub_params.work_dir,
params.submit.base_client_log_dir)
def add_file_unparsed(user_file, dicts, is_factory):
absfname = user_file["absfname"]
if "relfname" not in user_file:
relfname = os.path.basename(absfname) # defualt is the final part of absfname
else:
relfname = user_file["relfname"]
is_const = eval(user_file["const"])
is_executable = eval(user_file["executable"])
is_wrapper = eval(user_file["wrapper"])
do_untar = eval(user_file["untar"])
prefix = user_file["prefix"]
period_value = int(user_file["period"])
file_list_idx = "file_list"
if "after_entry" in user_file:
if eval(user_file["after_entry"]):
file_list_idx = "after_file_list"
if is_executable: # a script
dicts[file_list_idx].add_from_file(
relfname,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "exec", period_value, prefix),
absfname,
)
elif is_wrapper: # a sourceable script for the wrapper
dicts[file_list_idx].add_from_file(
relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "wrapper"), absfname
)
elif do_untar: # a tarball
untar_opts = user_file.get_child("untar_options")
if u"dir" in untar_opts:
wnsubdir = untar_opts["dir"]
else:
wnsubdir = string.split(relfname, ".", 1)[0] # deafult is relfname up to the first .
if "absdir_outattr" in untar_opts:
config_out = untar_opts["absdir_outattr"]
else:
config_out = "FALSE"
cond_attr = untar_opts["cond_attr"]
dicts[file_list_idx].add_from_file(
relfname,
cWDictFile.FileDictFile.make_val_tuple(
cWConsts.insert_timestr(relfname), "untar", cond_download=cond_attr, config_out=config_out
),
absfname,
)
dicts["untar_cfg"].add(relfname, wnsubdir)
else: # not executable nor tarball => simple file
if is_const:
val = "regular"
dicts[file_list_idx].add_from_file(
relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), val), absfname
)
else:
val = "nocache"
dicts[file_list_idx].add_from_file(
relfname, cWDictFile.FileDictFile.make_val_tuple(relfname, val), absfname
) # no timestamp if it can be modified
def populate(self, other=None):
# put default files in place first
self.dicts["file_list"].add_placeholder("error_gen.sh", allow_overwrite=True)
self.dicts["file_list"].add_placeholder("error_augment.sh", allow_overwrite=True)
self.dicts["file_list"].add_placeholder("setup_script.sh", allow_overwrite=True)
self.dicts["file_list"].add_placeholder(cWConsts.CONSTS_FILE, allow_overwrite=True)
self.dicts["file_list"].add_placeholder(cWConsts.VARS_FILE, allow_overwrite=True)
self.dicts["file_list"].add_placeholder(
cWConsts.UNTAR_CFG_FILE, allow_overwrite=True
) # this one must be loaded before any tarball
self.dicts["file_list"].add_placeholder(
cWConsts.GRIDMAP_FILE, allow_overwrite=True
) # this one must be loaded before setup_x509.sh is run
# load system files
for file_name in (
"error_gen.sh",
"error_augment.sh",
"parse_starterlog.awk",
"advertise_failure.helper",
"condor_config",
"condor_config.multi_schedd.include",
"condor_config.dedicated_starter.include",
"condor_config.check.include",
"condor_config.monitor.include",
):
self.dicts["file_list"].add_from_file(
file_name,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(file_name), "regular"),
os.path.join(cgWConsts.WEB_BASE_DIR, file_name),
)
self.dicts["description"].add("condor_config", "condor_config")
self.dicts["description"].add("condor_config.multi_schedd.include", "condor_config_multi_include")
self.dicts["description"].add("condor_config.dedicated_starter.include", "condor_config_main_include")
self.dicts["description"].add("condor_config.monitor.include", "condor_config_monitor_include")
self.dicts["description"].add("condor_config.check.include", "condor_config_check_include")
self.dicts["vars"].load(cgWConsts.WEB_BASE_DIR, "condor_vars.lst", change_self=False, set_not_changed=False)
#
# Note:
# We expect the condor platform info to be coming in as parameters
# as FE provided consts file is not available at this time
#
# add the basic standard params
self.dicts["params"].add("GLIDEIN_Report_Failed", "NEVER")
self.dicts["params"].add("CONDOR_OS", "default")
self.dicts["params"].add("CONDOR_ARCH", "default")
self.dicts["params"].add("CONDOR_VERSION", "default")
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ("setup_script.sh", "cat_consts.sh", "condor_platform_select.sh"):
self.dicts["file_list"].add_from_file(
script_name,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"),
os.path.join(cgWConsts.WEB_BASE_DIR, script_name),
)
# load condor tarballs
# only one will be downloaded in the end... based on what condor_platform_select.sh decides
condor_tarballs = self.conf.get_child_list(u"condor_tarballs")
prev_tar_dir_map = {}
if other is not None and other.main_dicts.dicts["glidein"].has_key("CondorTarballDirMap"):
prev_tar_dir_map = eval(other.main_dicts.dicts["glidein"]["CondorTarballDirMap"])
tar_dir_map = {}
for condor_idx in range(len(condor_tarballs)):
condor_el = condor_tarballs[condor_idx]
# condor_el now is a combination of csv version+os+arch
# Get list of valid tarballs for this condor_el
# Register the tarball, but make download conditional to cond_name
condor_el_valid_tarballs = get_valid_condor_tarballs([condor_el])
condor_fname = cWConsts.insert_timestr(cgWConsts.CONDOR_FILE % condor_idx)
condor_fd = None
if u"tar_file" in condor_el:
# Condor tarball available. Just add it to the list of tarballs
# with every possible condor_platform string
condor_tarfile = condor_el[u"tar_file"]
# already built this tarball, just reuse it
elif condor_el[u"base_dir"] in prev_tar_dir_map:
condor_tarfile = prev_tar_dir_map[condor_el[u"base_dir"]]
tar_dir_map[condor_el[u"base_dir"]] = condor_tarfile
else:
# Create a new tarball as usual
condor_fd = cgWCreate.create_condor_tar_fd(condor_el[u"base_dir"])
tar_dir_map[condor_el[u"base_dir"]] = os.path.join(self.dicts["file_list"].dir, condor_fname)
for tar in condor_el_valid_tarballs:
condor_platform = "%s-%s-%s" % (tar["version"], tar["os"], tar["arch"])
cond_name = "CONDOR_PLATFORM_%s" % condor_platform
condor_platform_fname = cgWConsts.CONDOR_FILE % condor_platform
if condor_fd is None:
# tar file exists. Just use it
self.dicts["file_list"].add_from_file(
condor_platform_fname,
cWDictFile.FileDictFile.make_val_tuple(
condor_fname, "untar", cond_download=cond_name, config_out=cgWConsts.CONDOR_ATTR
),
condor_tarfile,
)
else:
# This is addition of new tarfile
# Need to rewind fd every time
condor_fd.seek(0)
self.dicts["file_list"].add_from_fd(
condor_platform_fname,
cWDictFile.FileDictFile.make_val_tuple(
condor_fname, "untar", cond_download=cond_name, config_out=cgWConsts.CONDOR_ATTR
),
condor_fd,
)
self.dicts["untar_cfg"].add(condor_platform_fname, cgWConsts.CONDOR_DIR)
# Add cond_name in the config, so that it is known
# But leave it disabled by default
self.dicts["consts"].add(cond_name, "0", allow_overwrite=False)
if condor_fd is not None:
condor_fd.close()
self.dicts["glidein"].add("CondorTarballDirMap", str(tar_dir_map))
#
# Note:
# We expect the collector info to be coming in as parameter
# as FE consts file is not available at this time
#
# add the basic standard params
self.dicts["params"].add("GLIDEIN_Collector", "Fake")
# add the factory monitoring collector parameter, if any collectors are defined
# this is purely a factory thing
factory_monitoring_collector = calc_monitoring_collectors_string(
self.conf.get_child_list(u"monitoring_collectors")
)
if factory_monitoring_collector is not None:
self.dicts["params"].add("GLIDEIN_Factory_Collector", str(factory_monitoring_collector))
populate_gridmap(self.conf, self.dicts["gridmap"])
file_list_scripts = [
"collector_setup.sh",
"create_temp_mapfile.sh",
"setup_x509.sh",
cgWConsts.CONDOR_STARTUP_FILE,
]
after_file_list_scripts = [
"check_proxy.sh",
"create_mapfile.sh",
"validate_node.sh",
"setup_network.sh",
"gcb_setup.sh",
"glexec_setup.sh",
"java_setup.sh",
"glidein_memory_setup.sh",
"glidein_cpus_setup.sh",
"glidein_sitewms_setup.sh",
"script_wrapper.sh",
"smart_partitionable.sh",
]
# Only execute scripts once
duplicate_scripts = set(file_list_scripts).intersection(after_file_list_scripts)
if duplicate_scripts:
raise RuntimeError, "Duplicates found in the list of files to execute '%s'" % ",".join(duplicate_scripts)
# Load more system scripts
for script_name in file_list_scripts:
self.dicts["file_list"].add_from_file(
script_name,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"),
os.path.join(cgWConsts.WEB_BASE_DIR, script_name),
)
# Add the drainer script
drain_script = "check_wn_drainstate.sh"
self.dicts["file_list"].add_from_file(
drain_script,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(drain_script), "exec", 60),
os.path.join(cgWConsts.WEB_BASE_DIR, drain_script),
)
# make sure condor_startup does not get executed ahead of time under normal circumstances
# but must be loaded early, as it also works as a reporting script in case of error
self.dicts["description"].add(cgWConsts.CONDOR_STARTUP_FILE, "last_script")
#
# At this point in the glideins, condor_advertize should be able to talk to the FE collector
#
# put user files in stage
for file in self.conf.get_child_list(u"files"):
add_file_unparsed(file, self.dicts, True)
# put user attributes into config files
for attr in self.conf.get_child_list(u"attrs"):
add_attr_unparsed(attr, self.dicts, "main")
# add additional system scripts
for script_name in after_file_list_scripts:
self.dicts["after_file_list"].add_from_file(
script_name,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"),
os.path.join(cgWConsts.WEB_BASE_DIR, script_name),
)
# populate complex files
populate_factory_descript(
self.work_dir, self.dicts["glidein"], self.active_sub_list, self.disabled_sub_list, self.conf
)
populate_frontend_descript(self.dicts["frontend_descript"], self.conf)
# populate the monitor files
javascriptrrd_dir = self.conf.get_child(u"monitor")[u"javascriptRRD_dir"]
for mfarr in ((cgWConsts.WEB_BASE_DIR, "factory_support.js"), (javascriptrrd_dir, "javascriptrrd.wlibs.js")):
mfdir, mfname = mfarr
parent_dir = self.find_parent_dir(mfdir, mfname)
mfobj = cWDictFile.SimpleFile(parent_dir, mfname)
mfobj.load()
self.monitor_jslibs.append(mfobj)
for mfarr in (
(cgWConsts.WEB_BASE_DIR, "factoryRRDBrowse.html"),
(cgWConsts.WEB_BASE_DIR, "factoryRRDEntryMatrix.html"),
(cgWConsts.WEB_BASE_DIR, "factoryStatus.html"),
(cgWConsts.WEB_BASE_DIR, "factoryLogStatus.html"),
(cgWConsts.WEB_BASE_DIR, "factoryCompletedStats.html"),
(cgWConsts.WEB_BASE_DIR, "factoryStatusNow.html"),
(cgWConsts.WEB_BASE_DIR, "factoryEntryStatusNow.html"),
):
mfdir, mfname = mfarr
mfobj = cWDictFile.SimpleFile(mfdir, mfname)
mfobj.load()
self.monitor_htmls.append(mfobj)
# add the index page and its images
mfobj = cWDictFile.SimpleFile(cgWConsts.WEB_BASE_DIR + "/factory/", "index.html")
mfobj.load()
self.monitor_htmls.append(mfobj)
for imgfile in (
"factoryCompletedStats.png",
"factoryEntryStatusNow.png",
"factoryLogStatus.png",
"factoryRRDBrowse.png",
"factoryRRDEntryMatrix.png",
"factoryStatus.png",
"factoryStatusNow.png",
):
mfobj = cWDictFile.SimpleFile(cgWConsts.WEB_BASE_DIR + "/factory/images/", imgfile)
mfobj.load()
self.monitor_images.append(mfobj)
def populate(self,params=None):
if params is None:
params=self.params
# put default files in place first
self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
self.dicts['preentry_file_list'].add_placeholder(cWConsts.GRIDMAP_FILE,allow_overwrite=True) # this one must be loaded before factory runs setup_x509.sh
# follow by the blacklist file
file_name=cWConsts.BLACKLIST_FILE
self.dicts['preentry_file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh',"check_blacklist.sh"):
self.dicts['preentry_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
# put user files in stage
for user_file in params.files:
add_file_unparsed(user_file,self.dicts)
# start expr is special
start_expr=None
# put user attributes into config files
for attr_name in params.attrs.keys():
if attr_name in ('GLIDECLIENT_Start','GLIDECLIENT_Group_Start'):
if start_expr is None:
start_expr=params.attrs[attr_name].value
elif not (params.attrs[attr_name].value in (None,'True')):
start_expr="(%s)&&(%s)"%(start_expr,params.attrs[attr_name].value)
# delete from the internal structure... will use it in match section
del params.data['attrs'][attr_name]
else:
add_attr_unparsed(attr_name, params,self.dicts,"main")
real_start_expr=params.match.start_expr
if start_expr is not None:
if real_start_expr!='True':
real_start_expr="(%s)&&(%s)"%(real_start_expr,start_expr)
else:
real_start_expr=start_expr
# since I removed the attributes, roll back into the match.start_expr
params.data['match']['start_expr']=real_start_expr
self.dicts['consts'].add('GLIDECLIENT_Start',real_start_expr)
# create GLIDEIN_Collector attribute
self.dicts['params'].add_extended('GLIDEIN_Collector',False,str(calc_glidein_collectors(params.collectors)))
populate_gridmap(params,self.dicts['gridmap'])
if self.dicts['preentry_file_list'].is_placeholder(cWConsts.GRIDMAP_FILE): # gridmapfile is optional, so if not loaded, remove the placeholder
self.dicts['preentry_file_list'].remove(cWConsts.GRIDMAP_FILE)
# populate complex files
populate_frontend_descript(self.work_dir,self.dicts['frontend_descript'],self.active_sub_list,params)
populate_common_descript(self.dicts['frontend_descript'],params)
# populate the monitor files
javascriptrrd_dir = params.monitor.javascriptRRD_dir
for mfarr in ((params.src_dir,'frontend_support.js'),
(javascriptrrd_dir,'rrdFlot.js'),
(javascriptrrd_dir,'rrdFlotMatrix.js'),
(javascriptrrd_dir,'rrdFlotSupport.js'),
(javascriptrrd_dir,'rrdFile.js'),
(javascriptrrd_dir,'rrdFilter.js'),
(javascriptrrd_dir,'binaryXHR.js'),
(params.monitor.flot_dir,'jquery.flot.js'),
(params.monitor.flot_dir,'jquery.flot.selection.js'),
(params.monitor.flot_dir,'jquery.flot.tooltip.js'),
(params.monitor.flot_dir,'excanvas.js'),
(params.monitor.jquery_dir,'jquery.js')):
mfdir,mfname=mfarr
parent_dir = self.find_parent_dir(mfdir,mfname)
mfobj=cWDictFile.SimpleFile(parent_dir,mfname)
mfobj.load()
self.monitor_jslibs.append(mfobj)
for mfarr in ((params.src_dir,'frontendRRDBrowse.html'),
(params.src_dir,'frontendRRDGroupMatrix.html'),
(params.src_dir,'frontendGroupGraphStatusNow.html'),
(params.src_dir,'frontendStatus.html')):
mfdir,mfname=mfarr
mfobj=cWDictFile.SimpleFile(mfdir,mfname)
mfobj.load()
self.monitor_htmls.append(mfobj)
spd = self.params.data
useMonitorIndexPage = True
if spd.has_key('frontend_monitor_index_page'):
useMonitorIndexPage = spd['frontend_monitor_index_page'] in ('True', 'true', '1')
if useMonitorIndexPage:
mfobj = cWDictFile.SimpleFile(params.src_dir + '/frontend', 'index.html')
mfobj.load()
self.monitor_htmls.append(mfobj)
for imgfil in ('frontendGroupGraphsNow.small.png',
'frontendRRDBrowse.small.png',
'frontendRRDGroupMatix.small.png',
'frontendStatus.small.png'):
mfobj = cWDictFile.SimpleFile(params.src_dir + '/frontend/images', imgfil)
mfobj.load()
self.monitor_htmls.append(mfobj)
# Tell condor to advertise GLIDECLIENT_ReqNode
self.dicts['vars'].add_extended('GLIDECLIENT_ReqNode','string',None,None,False,True,False)
# derive attributes
populate_common_attrs(self.dicts)
# populate security data
populate_main_security(self.client_security,params)
def populate(self,params=None):
if params is None:
params=self.params
sub_params=params.groups[self.sub_name]
# put default files in place first
self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
# follow by the blacklist file
file_name=cWConsts.BLACKLIST_FILE
self.dicts['preentry_file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name))
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh',"check_blacklist.sh"):
self.dicts['preentry_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
# put user files in stage
for user_file in sub_params.files:
add_file_unparsed(user_file,self.dicts)
# start expr is special
start_expr=None
# put user attributes into config files
for attr_name in sub_params.attrs.keys():
if attr_name in ('GLIDECLIENT_Group_Start','GLIDECLIENT_Start'):
if start_expr is None:
start_expr=sub_params.attrs[attr_name].value
elif sub_params.attrs[attr_name].value is not None:
start_expr="(%s)&&(%s)"%(start_expr,sub_params.attrs[attr_name].value)
# delete from the internal structure... will use it in match section
del sub_params.data['attrs'][attr_name]
else:
add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name)
real_start_expr=sub_params.match.start_expr
if start_expr is not None:
if real_start_expr!='True':
real_start_expr="(%s)&&(%s)"%(real_start_expr,start_expr)
else:
real_start_expr=start_expr
# since I removed the attributes, roll back into the match.start_expr
sub_params.data['match']['start_expr']=real_start_expr
self.dicts['consts'].add('GLIDECLIENT_Group_Start',real_start_expr)
# derive attributes
populate_common_attrs(self.dicts)
# populate complex files
populate_group_descript(self.work_dir,self.dicts['group_descript'],
self.sub_name,sub_params)
populate_common_descript(self.dicts['group_descript'],sub_params)
# populate security data
populate_main_security(self.client_security,params)
populate_group_security(self.client_security,params,sub_params)
def populate(self,params=None):
if params==None:
params=self.params
# put default files in place first
self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True)
self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ('cat_consts.sh','setup_x509.sh'):
self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
#load system files
for file_name in ('parse_starterlog.awk', "condor_config", "condor_config.multi_schedd.include", "condor_config.dedicated_starter.include", "condor_config.check.include", "condor_config.monitor.include"):
self.dicts['file_list'].add_from_file(file_name,(cWConsts.insert_timestr(file_name),"regular","TRUE",'FALSE'),os.path.join(params.src_dir,file_name))
self.dicts['description'].add("condor_config","condor_config")
self.dicts['description'].add("condor_config.multi_schedd.include","condor_config_multi_include")
self.dicts['description'].add("condor_config.dedicated_starter.include","condor_config_main_include")
self.dicts['description'].add("condor_config.monitor.include","condor_config_monitor_include")
self.dicts['description'].add("condor_config.check.include","condor_config_check_include")
self.dicts['vars'].load(params.src_dir,'condor_vars.lst',change_self=False,set_not_changed=False)
# put user files in stage
for file in params.files:
add_file_unparsed(file,self.dicts)
# put user attributes into config files
for attr_name in params.attrs.keys():
add_attr_unparsed(attr_name, params,self.dicts,"main")
# add the basic standard params
self.dicts['params'].add("GLIDEIN_Collector",'Fake')
# Prepare to load condor tarballs (after entry, as I may need data from there)
for script_name in ('validate_node.sh','condor_platform_select.sh'):
self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
#load condor tarballs
for condor_idx in range(len(params.condor_tarballs)):
condor_el=params.condor_tarballs[condor_idx]
condor_platform="%s-%s-%s"%(condor_el.version,condor_el.os,condor_el.arch)
cond_name="CONDOR_PLATFORM_%s"%condor_platform
condor_fname=cgWConsts.CONDOR_FILE%condor_platform
# register the tarball, but make the download conditional to cond_name
if condor_el.tar_file!=None: # condor tarball available
self.dicts['after_file_list'].add_from_file(condor_fname,(cWConsts.insert_timestr(condor_fname),"untar",cond_name,cgWConsts.CONDOR_ATTR),condor_el.tar_file)
else: # create a new tarball
condor_fd=cgWCreate.create_condor_tar_fd(condor_el.base_dir)
condor_fname=cWConsts.insert_timestr(condor_fname)
self.dicts['after_file_list'].add_from_fd(condor_fname,(condor_fname,"untar",cond_name,cgWConsts.CONDOR_ATTR),condor_fd)
condor_fd.close()
# insert the newly created tarball fname back into the config
params.subparams.data['condor_tarballs'][condor_idx]['tar_file']=os.path.join(self.dicts['file_list'].dir,condor_fname)
# add cond_name in the config, so that it is known it is there
# but leave it disabled by default
self.dicts['consts'].add(cond_name,"0",allow_overwrite=False)
self.dicts['untar_cfg'].add(condor_fname,cgWConsts.CONDOR_DIR)
# add additional system scripts
for script_name in ('create_mapfile.sh','collector_setup.sh','gcb_setup.sh','glexec_setup.sh','java_setup.sh'):
self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
# this must be the last script in the list
for script_name in (cgWConsts.CONDOR_STARTUP_FILE,):
self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name))
self.dicts['description'].add(cgWConsts.CONDOR_STARTUP_FILE,"last_script")
# populate complex files
populate_factory_descript(self.work_dir,self.dicts['glidein'],self.active_sub_list,params)
populate_frontend_descript(self.dicts['frontend_descript'],params)
# populate the monitor files
javascriptrrd_dir = params.monitor.javascriptRRD_dir
for mfarr in ((params.src_dir,'factory_support.js'),
(javascriptrrd_dir,'rrdFlot.js'),
(javascriptrrd_dir,'rrdFlotMatrix.js'),
(javascriptrrd_dir,'rrdFlotSupport.js'),
(javascriptrrd_dir,'rrdFile.js'),
(javascriptrrd_dir,'rrdMultiFile.js'),
(javascriptrrd_dir,'rrdFilter.js'),
(javascriptrrd_dir,'binaryXHR.js'),
(params.monitor.flot_dir,'jquery.flot.js'),
(params.monitor.flot_dir,'jquery.flot.selection.js'),
(params.monitor.flot_dir,'excanvas.js'),
(params.monitor.jquery_dir,'jquery.js')):
mfdir,mfname=mfarr
parent_dir = self.find_parent_dir(mfdir,mfname)
mfobj=cWDictFile.SimpleFile(parent_dir,mfname)
mfobj.load()
self.monitor_jslibs.append(mfobj)
for mfarr in ((params.src_dir,'factoryRRDBrowse.html'),
(params.src_dir,'factoryRRDEntryMatrix.html'),
(params.src_dir,'factoryStatus.html'),
(params.src_dir,'factoryLogStatus.html'),
(params.src_dir,'factoryCompletedStats.html'),
(params.src_dir,'factoryStatusNow.html'),
(params.src_dir,'factoryEntryStatusNow.html')):
mfdir,mfname=mfarr
mfobj=cWDictFile.SimpleFile(mfdir,mfname)
mfobj.load()
self.monitor_htmls.append(mfobj)
def populate(self, entry, schedd):
# put default files in place first
self.dicts["file_list"].add_placeholder(cWConsts.CONSTS_FILE, allow_overwrite=True)
self.dicts["file_list"].add_placeholder(cWConsts.VARS_FILE, allow_overwrite=True)
self.dicts["file_list"].add_placeholder(
cWConsts.UNTAR_CFG_FILE, allow_overwrite=True
) # this one must be loaded before any tarball
# follow by the blacklist file
file_name = cWConsts.BLACKLIST_FILE
self.dicts["file_list"].add_from_file(
file_name,
cWDictFile.FileDictFile.make_val_tuple(file_name, "nocache", config_out="BLACKLIST_FILE"),
os.path.join(cgWConsts.WEB_BASE_DIR, file_name),
)
# Load initial system scripts
# These should be executed before the other scripts
for script_name in ("cat_consts.sh", "check_blacklist.sh"):
self.dicts["file_list"].add_from_file(
script_name,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"),
os.path.join(cgWConsts.WEB_BASE_DIR, script_name),
)
# load system files
self.dicts["vars"].load(
cgWConsts.WEB_BASE_DIR, "condor_vars.lst.entry", change_self=False, set_not_changed=False
)
# put user files in stage
for user_file in entry.get_child_list(u"files"):
add_file_unparsed(user_file, self.dicts, True)
# Add attribute for voms
# put user attributes into config files
for attr in entry.get_child_list(u"attrs"):
add_attr_unparsed(attr, self.dicts, self.sub_name)
# put standard attributes into config file
# override anything the user set
config = entry.get_child(u"config")
restrictions = config.get_child(u"restrictions")
submit = config.get_child(u"submit")
for dtype in ("attrs", "consts"):
self.dicts[dtype].add("GLIDEIN_Gatekeeper", entry[u"gatekeeper"], allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_GridType", entry[u"gridtype"], allow_overwrite=True)
# MERGENOTE:
# GLIDEIN_REQUIRE_VOMS publishes an attribute so that users without VOMS proxies
# can avoid sites that require VOMS proxies (using the normal Condor Requirements
# string.
self.dicts[dtype].add("GLIDEIN_REQUIRE_VOMS", restrictions[u"require_voms_proxy"], allow_overwrite=True)
self.dicts[dtype].add(
"GLIDEIN_REQUIRE_GLEXEC_USE", restrictions[u"require_glidein_glexec_use"], allow_overwrite=True
)
self.dicts[dtype].add("GLIDEIN_TrustDomain", entry[u"trust_domain"], allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_SupportedAuthenticationMethod", entry[u"auth_method"], allow_overwrite=True)
if u"rsl" in entry:
self.dicts[dtype].add("GLIDEIN_GlobusRSL", entry[u"rsl"], allow_overwrite=True)
self.dicts[dtype].add("GLIDEIN_SlotsLayout", submit[u"slots_layout"], allow_overwrite=True)
self.dicts["vars"].add_extended(
"GLIDEIN_REQUIRE_VOMS", "boolean", restrictions[u"require_voms_proxy"], None, False, True, True
)
self.dicts["vars"].add_extended(
"GLIDEIN_REQUIRE_GLEXEC_USE",
"boolean",
restrictions[u"require_glidein_glexec_use"],
None,
False,
True,
True,
)
# populate infosys
for infosys_ref in entry.get_child_list(u"infosys_refs"):
self.dicts["infosys"].add_extended(
infosys_ref[u"type"], infosys_ref[u"server"], infosys_ref[u"ref"], allow_overwrite=True
)
# populate monitorgroups
for monitorgroup in entry.get_child_list(u"monitorgroups"):
self.dicts["mongroup"].add_extended(monitorgroup[u"group_name"], allow_overwrite=True)
# populate complex files
populate_job_descript(self.work_dir, self.dicts["job_descript"], self.sub_name, entry, schedd)
################################################################################################################
# This is the original function call:
#
# self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE,
# params.factory_name,params.glidein_name,self.sub_name,
# sub_params.gridtype,sub_params.gatekeeper, sub_params.rsl, sub_params.auth_method,
# params.web_url,sub_params.proxy_url,sub_params.work_dir,
# params.submit.base_client_log_dir, sub_params.submit.submit_attrs)
#
# Almost all of the parameters are attributes of params and/or sub_params. Instead of maintaining an ever
# increasing parameter list for this function, lets just pass params, sub_params, and the 2 other parameters
# to the function and call it a day.
################################################################################################################
self.dicts["condor_jdl"].populate(cgWConsts.STARTUP_FILE, self.sub_name, self.conf, entry)
def add_file_unparsed(file,dicts):
absfname=file.absfname
if absfname==None:
raise RuntimeError, "Found a file element without an absname: %s"%file
relfname=file.relfname
if relfname==None:
relfname=os.path.basename(absfname) # defualt is the final part of absfname
if len(relfname)<1:
raise RuntimeError, "Found a file element with an empty relfname: %s"%file
is_const=eval(file.const,{},{})
is_executable=eval(file.executable,{},{})
is_wrapper=eval(file.wrapper,{},{})
do_untar=eval(file.untar,{},{})
file_list_idx='file_list'
if file.has_key('after_entry'):
if eval(file.after_entry,{},{}):
file_list_idx='after_file_list'
if is_executable: # a script
if not is_const:
raise RuntimeError, "A file cannot be executable if it is not constant: %s"%file
if do_untar:
raise RuntimeError, "A tar file cannot be executable: %s"%file
if is_wrapper:
raise RuntimeError, "A wrapper file cannot be executable: %s"%file
dicts[file_list_idx].add_from_file(relfname,(cWConsts.insert_timestr(relfname),"exec","TRUE",'FALSE'),absfname)
elif is_wrapper: # a sourceable script for the wrapper
if not is_const:
raise RuntimeError, "A file cannot be a wrapper if it is not constant: %s"%file
if do_untar:
raise RuntimeError, "A tar file cannot be a wrapper: %s"%file
dicts[file_list_idx].add_from_file(relfname,(cWConsts.insert_timestr(relfname),"wrapper","TRUE",'FALSE'),absfname)
elif do_untar: # a tarball
if not is_const:
raise RuntimeError, "A file cannot be untarred if it is not constant: %s"%file
wnsubdir=file.untar_options.dir
if wnsubdir==None:
wnsubdir=string.split(relfname,'.',1)[0] # deafult is relfname up to the first .
config_out=file.untar_options.absdir_outattr
if config_out==None:
config_out="FALSE"
cond_attr=file.untar_options.cond_attr
dicts[file_list_idx].add_from_file(relfname,(cWConsts.insert_timestr(relfname),"untar",cond_attr,config_out),absfname)
dicts['untar_cfg'].add(relfname,wnsubdir)
else: # not executable nor tarball => simple file
if is_const:
val='regular'
dicts[file_list_idx].add_from_file(relfname,(cWConsts.insert_timestr(relfname),val,'TRUE','FALSE'),absfname)
else:
val='nocache'
dicts[file_list_idx].add_from_file(relfname,(relfname,val,'TRUE','FALSE'),absfname) # no timestamp if it can be modified
def add_file_unparsed(user_file, dicts, is_factory):
"""Add a user file residing in the stage area
file as described by Params.file_defaults
:param user_file: file from the config files "files" sections
:param dicts: parameters dctionaries
:param is_factory: True if invoked for the factory (cgWParamDict.py), false for the frontend (cvWParamDict.py)
:return: None (dictionaries are modified)
"""
absfname = user_file.absfname
if absfname is None:
raise RuntimeError("Found a file element without an absname: %s" % user_file)
relfname = user_file.relfname
if relfname is None:
relfname = os.path.basename(absfname) # defualt is the final part of absfname
if len(relfname) < 1:
raise RuntimeError("Found a file element with an empty relfname: %s" % user_file)
is_const = is_true(user_file.const)
is_executable = is_true(user_file.executable)
is_wrapper = is_true(user_file.wrapper)
do_untar = is_true(user_file.untar)
try:
period_value = int(user_file.period)
except (AttributeError, KeyError, ValueError):
period_value = 0
if is_factory:
# Factory (file_list, after_file_list)
file_list_idx = 'file_list'
if user_file.has_key('after_entry'):
if is_true(user_file.after_entry): # eval(user_file.after_entry,{},{}):
file_list_idx = 'after_file_list'
else:
# Frontend (preentry_file_list, file_list, aftergroup_preentry_file_list, aftergroup_file_list)
file_list_idx = 'preentry_file_list'
if user_file.has_key('after_entry'):
if is_true(user_file.after_entry):
file_list_idx = 'file_list'
if user_file.has_key('after_group'):
if is_true(user_file.after_group):
file_list_idx = 'aftergroup_%s' % file_list_idx
# period has 0 as default (in dictionary definition). Should I still protect against it not being defined?
if period_value > 0:
if not is_executable:
raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file)
if is_executable: # a script
if not is_const:
raise RuntimeError("A file cannot be executable if it is not constant: %s" % user_file)
if do_untar:
raise RuntimeError("A tar file cannot be executable: %s" % user_file)
if is_wrapper:
raise RuntimeError("A wrapper file cannot be an executable: %s" % user_file)
dicts[file_list_idx].add_from_file(relfname,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), 'exec',
user_file.period, user_file.prefix),
absfname)
elif is_wrapper: # a sourceable script for the wrapper
if not is_const:
raise RuntimeError("A file cannot be a wrapper if it is not constant: %s" % user_file)
if do_untar:
raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file)
dicts[file_list_idx].add_from_file(relfname,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), 'wrapper'),
absfname)
elif do_untar: # a tarball
if not is_const:
raise RuntimeError("A file cannot be untarred if it is not constant: %s" % user_file)
wnsubdir = user_file.untar_options.dir
if wnsubdir is None:
wnsubdir = string.split(relfname, '.', 1)[0] # deafult is relfname up to the first .
config_out = user_file.untar_options.absdir_outattr
if config_out is None:
config_out = "FALSE"
cond_attr = user_file.untar_options.cond_attr
dicts[file_list_idx].add_from_file(relfname,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname),
'untar',
cond_download=cond_attr,
config_out=config_out),
absfname)
dicts['untar_cfg'].add(relfname, wnsubdir)
else: # not executable nor tarball => simple file
if is_const:
val = 'regular'
dicts[file_list_idx].add_from_file(relfname,
cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), val),
absfname)
else:
val = 'nocache'
dicts[file_list_idx].add_from_file(relfname, cWDictFile.FileDictFile.make_val_tuple(relfname, val),
absfname) # no timestamp in the name if it can be modified
