def get_main_dicts(submit_dir,stage_dir): main_dicts=get_common_dicts(submit_dir,stage_dir) main_dicts['summary_signature']=cWDictFile.SummarySHA1DictFile(submit_dir,cWConsts.SUMMARY_SIGNATURE_FILE) main_dicts['glidein']=cWDictFile.StrDictFile(submit_dir,cgWConsts.GLIDEIN_FILE) main_dicts['frontend_descript']=cWDictFile.ReprDictFile(submit_dir,cgWConsts.FRONTEND_DESCRIPT_FILE) main_dicts['gridmap']=cWDictFile.GridMapDict(stage_dir,cWConsts.insert_timestr(cWConsts.GRIDMAP_FILE)) main_dicts['after_file_list']=cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cgWConsts.AFTER_FILE_LISTFILE),fname_idx=cgWConsts.AFTER_FILE_LISTFILE) return main_dicts
def get_main_dicts(work_dir,stage_dir,simple_work_dir,assume_groups): main_dicts=get_common_dicts(work_dir,stage_dir,simple_work_dir) main_dicts['summary_signature']=cWDictFile.SummarySHA1DictFile(work_dir,cWConsts.SUMMARY_SIGNATURE_FILE) main_dicts['frontend_descript']=cWDictFile.StrDictFile(work_dir,cvWConsts.FRONTEND_DESCRIPT_FILE) main_dicts['gridmap']=GridMapDict(stage_dir,cWConsts.insert_timestr(cvWConsts.GRIDMAP_FILE)) if assume_groups: main_dicts['aftergroup_file_list']=cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cvWConsts.AFTERGROUP_FILE_LISTFILE),fname_idx=cvWConsts.AFTERGROUP_FILE_LISTFILE) main_dicts['aftergroup_preentry_file_list']=cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cvWConsts.AFTERGROUP_PREENTRY_FILE_LISTFILE),fname_idx=cvWConsts.AFTERGROUP_PREENTRY_FILE_LISTFILE) return main_dicts
def populate(self,params=None): if params==None: params=self.params sub_params=params.entries[self.sub_name] # put default files in place first self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball # follow by the blacklist file file_name=cWConsts.BLACKLIST_FILE self.dicts['file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh',"check_blacklist.sh"): self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) #load system files self.dicts['vars'].load(params.src_dir,'condor_vars.lst.entry',change_self=False,set_not_changed=False) # put user files in stage for file in sub_params.files: add_file_unparsed(file,self.dicts) # put user attributes into config files for attr_name in sub_params.attrs.keys(): add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name) # put standard attributes into config file # override anything the user set for dtype in ('attrs','consts'): self.dicts[dtype].add("GLIDEIN_Gatekeeper",sub_params.gatekeeper,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_GridType",sub_params.gridtype,allow_overwrite=True) if sub_params.rsl!=None: self.dicts[dtype].add('GLIDEIN_GlobusRSL',sub_params.rsl,allow_overwrite=True) # populate infosys for infosys_ref in sub_params.infosys_refs: self.dicts['infosys'].add_extended(infosys_ref['type'],infosys_ref['server'],infosys_ref['ref'],allow_overwrite=True) # populate monitorgroups for monitorgroup in sub_params.monitorgroups: self.dicts['mongroup'].add_extended(monitorgroup['group_name'],allow_overwrite=True) # populate complex files populate_job_descript(self.work_dir,self.dicts['job_descript'], self.sub_name,sub_params) self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE, params.factory_name,params.glidein_name,self.sub_name, sub_params.gridtype,sub_params.gatekeeper,sub_params.rsl, params.web_url,sub_params.proxy_url,sub_params.work_dir, params.submit.base_client_log_dir)
def get_common_dicts(work_dir,stage_dir, simple_work_dir): # if True, do not create params common_dicts={'description':cWDictFile.DescriptionDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.DESCRIPTION_FILE),fname_idx=cWConsts.DESCRIPTION_FILE), 'consts':cWDictFile.StrDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.CONSTS_FILE),fname_idx=cWConsts.CONSTS_FILE), 'vars':cWDictFile.VarsDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.VARS_FILE),fname_idx=cWConsts.VARS_FILE), 'untar_cfg':cWDictFile.StrDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.UNTAR_CFG_FILE),fname_idx=cWConsts.UNTAR_CFG_FILE), 'file_list':cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.FILE_LISTFILE),fname_idx=cWConsts.FILE_LISTFILE), 'preentry_file_list':cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cvWConsts.PREENTRY_FILE_LISTFILE),fname_idx=cvWConsts.PREENTRY_FILE_LISTFILE), "signature":cWDictFile.SHA1DictFile(stage_dir,cWConsts.insert_timestr(cWConsts.SIGNATURE_FILE),fname_idx=cWConsts.SIGNATURE_FILE)} if not simple_work_dir: common_dicts['params']=ParamsDictFile(work_dir,cvWConsts.PARAMS_FILE) refresh_description(common_dicts) return common_dicts
def get_common_dicts(submit_dir,stage_dir): common_dicts={'attrs':cWDictFile.ReprDictFile(submit_dir,cgWConsts.ATTRS_FILE), 'description':cWDictFile.DescriptionDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.DESCRIPTION_FILE),fname_idx=cWConsts.DESCRIPTION_FILE), 'consts':cWDictFile.StrDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.CONSTS_FILE),fname_idx=cWConsts.CONSTS_FILE), 'params':cWDictFile.ReprDictFile(submit_dir,cgWConsts.PARAMS_FILE), 'vars':cWDictFile.VarsDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.VARS_FILE),fname_idx=cWConsts.VARS_FILE), 'untar_cfg':cWDictFile.StrDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.UNTAR_CFG_FILE),fname_idx=cWConsts.UNTAR_CFG_FILE), 'file_list':cWDictFile.FileDictFile(stage_dir,cWConsts.insert_timestr(cWConsts.FILE_LISTFILE),fname_idx=cWConsts.FILE_LISTFILE), "signature":cWDictFile.SHA1DictFile(stage_dir,cWConsts.insert_timestr(cWConsts.SIGNATURE_FILE),fname_idx=cWConsts.SIGNATURE_FILE)} refresh_description(common_dicts) return common_dicts
def populate(self,params=None): if params==None: params=self.params sub_params=params.groups[self.sub_name] # put default files in place first self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball # follow by the blacklist file file_name=cWConsts.BLACKLIST_FILE self.dicts['preentry_file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh',"check_blacklist.sh"): self.dicts['preentry_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # put user files in stage for file in sub_params.files: add_file_unparsed(file,self.dicts) # put user attributes into config files for attr_name in sub_params.attrs.keys(): add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name) # populate complex files populate_group_descript(self.work_dir,self.dicts['group_descript'], self.sub_name,sub_params) populate_common_descript(self.dicts['group_descript'],sub_params) # populate security data populate_main_security(self.client_security,params) populate_group_security(self.client_security,params,sub_params)
def populate(self,params=None): if params==None: params=self.params # put default files in place first self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball self.dicts['preentry_file_list'].add_placeholder(cWConsts.GRIDMAP_FILE,allow_overwrite=True) # this one must be loaded before factory runs setup_x509.sh # follow by the blacklist file file_name=cWConsts.BLACKLIST_FILE self.dicts['preentry_file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh',"check_blacklist.sh"): self.dicts['preentry_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # put user files in stage for file in params.files: add_file_unparsed(file,self.dicts) # put user attributes into config files for attr_name in params.attrs.keys(): add_attr_unparsed(attr_name, params,self.dicts,"main") # create GLIDEIN_Collector attribute self.dicts['params'].add_extended('GLIDEIN_Collector',False,str(calc_glidein_collectors(params.collectors))) populate_gridmap(params,self.dicts['gridmap']) if self.dicts['preentry_file_list'].is_placeholder(cWConsts.GRIDMAP_FILE): # gridmapfile is optional, so if not loaded, remove the placeholder self.dicts['preentry_file_list'].remove(cWConsts.GRIDMAP_FILE) # populate complex files populate_frontend_descript(self.work_dir,self.dicts['frontend_descript'],self.active_sub_list,params) populate_common_descript(self.dicts['frontend_descript'],params) # populate the monitor files javascriptrrd_dir = params.monitor.javascriptRRD_dir for mfarr in ((params.src_dir,'frontend_support.js'), (javascriptrrd_dir,'rrdFlot.js'), (javascriptrrd_dir,'rrdFlotMatrix.js'), (javascriptrrd_dir,'rrdFlotSupport.js'), (javascriptrrd_dir,'rrdFile.js'), (javascriptrrd_dir,'rrdFilter.js'), (javascriptrrd_dir,'binaryXHR.js'), (params.monitor.flot_dir,'jquery.flot.js'), (params.monitor.flot_dir,'jquery.flot.selection.js'), (params.monitor.flot_dir,'excanvas.js'), (params.monitor.jquery_dir,'jquery.js')): mfdir,mfname=mfarr parent_dir = self.find_parent_dir(mfdir,mfname) mfobj=cWDictFile.SimpleFile(parent_dir,mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ((params.src_dir,'frontendRRDBrowse.html'), (params.src_dir,'frontendRRDGroupMatrix.html'), (params.src_dir,'frontendGroupGraphStatusNow.html'), (params.src_dir,'frontendStatus.html')): mfdir,mfname=mfarr mfobj=cWDictFile.SimpleFile(mfdir,mfname) mfobj.load() self.monitor_htmls.append(mfobj) # populate security data populate_main_security(self.client_security,params)
def populate(self,params=None): if params is None: params=self.params # put default files in place first self.dicts['file_list'].add_placeholder('error_gen.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder('error_augment.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder('setup_script.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball #load system files for file_name in ('error_gen.sh','error_augment.sh','parse_starterlog.awk', 'advertise_failure.helper', "condor_config", "condor_config.multi_schedd.include", "condor_config.dedicated_starter.include", "condor_config.check.include", "condor_config.monitor.include"): self.dicts['file_list'].add_from_file(file_name,(cWConsts.insert_timestr(file_name),"regular","TRUE",'FALSE'),os.path.join(params.src_dir,file_name)) self.dicts['description'].add("condor_config","condor_config") self.dicts['description'].add("condor_config.multi_schedd.include","condor_config_multi_include") self.dicts['description'].add("condor_config.dedicated_starter.include","condor_config_main_include") self.dicts['description'].add("condor_config.monitor.include","condor_config_monitor_include") self.dicts['description'].add("condor_config.check.include","condor_config_check_include") self.dicts['vars'].load(params.src_dir,'condor_vars.lst',change_self=False,set_not_changed=False) # # Note: # We expect the condor platform info to be coming in as parameters # as FE provided consts file is not available at this time # # add the basic standard params self.dicts['params'].add("GLIDEIN_Report_Failed",'NEVER') self.dicts['params'].add("CONDOR_OS",'default') self.dicts['params'].add("CONDOR_ARCH",'default') self.dicts['params'].add("CONDOR_VERSION",'default') # Load initial system scripts # These should be executed before the other scripts for script_name in ('setup_script.sh','cat_consts.sh','condor_platform_select.sh'): self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) #load condor tarballs # only one will be downloaded in the end... based on what condor_platform_select.sh decides for condor_idx in range(len(params.condor_tarballs)): condor_el=params.condor_tarballs[condor_idx] # condor_el now is a combination of csv version+os+arch # Get list of valid tarballs for this condor_el # Register the tarball, but make download conditional to cond_name condor_el_valid_tarballs = get_valid_condor_tarballs([condor_el]) condor_fname = cWConsts.insert_timestr(cgWConsts.CONDOR_FILE % condor_idx) condor_tarfile = "" condor_fd = None if condor_el.tar_file is not None: # Condor tarball available. Just add it to the list of tarballs # with every possible condor_platform string condor_tarfile = condor_el.tar_file else: # Create a new tarball as usual condor_fd = cgWCreate.create_condor_tar_fd(condor_el.base_dir) condor_tarfile = os.path.join(self.dicts['file_list'].dir, condor_fname) # insert the newly created tarball fname back into the config params.subparams.data['condor_tarballs'][condor_idx]['tar_file'] = condor_tarfile for tar in condor_el_valid_tarballs: condor_platform = "%s-%s-%s" % (tar['version'], tar['os'], tar['arch']) cond_name = "CONDOR_PLATFORM_%s" % condor_platform condor_platform_fname = cgWConsts.CONDOR_FILE % condor_platform if condor_fd is None: # tar file exists. Just use it self.dicts['file_list'].add_from_file( condor_platform_fname, (condor_fname, "untar", cond_name, cgWConsts.CONDOR_ATTR), condor_el.tar_file) else: # This is addition of new tarfile # Need to rewind fd everytime condor_fd.seek(0) self.dicts['file_list'].add_from_fd( condor_platform_fname, (condor_fname,"untar",cond_name,cgWConsts.CONDOR_ATTR), condor_fd) self.dicts['untar_cfg'].add(condor_platform_fname, cgWConsts.CONDOR_DIR) # Add cond_name in the config, so that it is known # But leave it disabled by default self.dicts['consts'].add(cond_name, "0", allow_overwrite=False) if condor_fd is not None: condor_fd.close() # # Note: # We expect the collector info to be coming in as parameter # as FE consts file is not available at this time # # add the basic standard params self.dicts['params'].add("GLIDEIN_Collector",'Fake') file_list_scripts = ['collector_setup.sh', 'create_temp_mapfile.sh', 'setup_x509.sh', cgWConsts.CONDOR_STARTUP_FILE] after_file_list_scripts = ['check_proxy.sh', 'create_mapfile.sh', 'validate_node.sh', 'gcb_setup.sh', 'glexec_setup.sh', 'java_setup.sh', 'glidein_memory_setup.sh', 'glidein_cpus_setup.sh'] # Only execute scripts once duplicate_scripts = set(file_list_scripts).intersection(after_file_list_scripts) if duplicate_scripts: raise RuntimeError, "Duplicates found in the list of files to execute '%s'" % ','.join(duplicate_scripts) # Load more system scripts for script_name in file_list_scripts: self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # make sure condor_startup does not get executed ahead of time under normal circumstances # but must be loaded early, as it also works as a reporting script in case of error self.dicts['description'].add(cgWConsts.CONDOR_STARTUP_FILE,"last_script") # # At this point in the glideins, condor_advertize should be able to talk to the FE collector # # put user files in stage for file in params.files: add_file_unparsed(file,self.dicts) # put user attributes into config files for attr_name in params.attrs.keys(): add_attr_unparsed(attr_name, params,self.dicts,"main") # add additional system scripts for script_name in after_file_list_scripts: self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # populate complex files populate_factory_descript(self.work_dir,self.dicts['glidein'],self.active_sub_list,params) populate_frontend_descript(self.dicts['frontend_descript'],params) # populate the monitor files javascriptrrd_dir = params.monitor.javascriptRRD_dir for mfarr in ((params.src_dir,'factory_support.js'), (javascriptrrd_dir,'rrdFlot.js'), (javascriptrrd_dir,'rrdFlotMatrix.js'), (javascriptrrd_dir,'rrdFlotSupport.js'), (javascriptrrd_dir,'rrdFile.js'), (javascriptrrd_dir,'rrdMultiFile.js'), (javascriptrrd_dir,'rrdFilter.js'), (javascriptrrd_dir,'binaryXHR.js'), (params.monitor.flot_dir,'jquery.flot.js'), (params.monitor.flot_dir,'jquery.flot.selection.js'), (params.monitor.flot_dir,'jquery.flot.tooltip.js'), (params.monitor.flot_dir,'excanvas.js'), (params.monitor.jquery_dir,'jquery.js')): mfdir,mfname=mfarr parent_dir = self.find_parent_dir(mfdir,mfname) mfobj=cWDictFile.SimpleFile(parent_dir,mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ((params.src_dir,'factoryRRDBrowse.html'), (params.src_dir,'factoryRRDEntryMatrix.html'), (params.src_dir,'factoryStatus.html'), (params.src_dir,'factoryLogStatus.html'), (params.src_dir,'factoryCompletedStats.html'), (params.src_dir,'factoryStatusNow.html'), (params.src_dir,'factoryEntryStatusNow.html')): mfdir,mfname=mfarr mfobj=cWDictFile.SimpleFile(mfdir,mfname) mfobj.load() self.monitor_htmls.append(mfobj) # add the index page and its images mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/', 'index.html') mfobj.load() self.monitor_htmls.append(mfobj) for imgfile in ('factoryCompletedStats.png', 'factoryEntryStatusNow.png', 'factoryLogStatus.png', 'factoryRRDBrowse.png', 'factoryRRDEntryMatrix.png', 'factoryStatus.png', 'factoryStatusNow.png'): mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/images/', imgfile) mfobj.load() self.monitor_htmls.append(mfobj)
def populate(self,params=None): if params is None: params=self.params sub_params=params.entries[self.sub_name] # put default files in place first self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball # follow by the blacklist file file_name=cWConsts.BLACKLIST_FILE self.dicts['file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh',"check_blacklist.sh"): self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) #load system files self.dicts['vars'].load(params.src_dir,'condor_vars.lst.entry',change_self=False,set_not_changed=False) # put user files in stage for user_file in sub_params.files: add_file_unparsed(user_file,self.dicts) # Add attribute for voms # put user attributes into config files for attr_name in sub_params.attrs.keys(): add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name) # put standard attributes into config file # override anything the user set for dtype in ('attrs','consts'): self.dicts[dtype].add("GLIDEIN_Gatekeeper",sub_params.gatekeeper,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_GridType",sub_params.gridtype,allow_overwrite=True) # MERGENOTE: # GLIDEIN_REQUIRE_VOMS publishes an attribute so that users without VOMS proxies # can avoid sites that require VOMS proxies (using the normal Condor Requirements # string. self.dicts[dtype].add("GLIDEIN_REQUIRE_VOMS",sub_params.config.restrictions.require_voms_proxy,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_REQUIRE_GLEXEC_USE",sub_params.config.restrictions.require_glidein_glexec_use,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_TrustDomain",sub_params.trust_domain,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_SupportedAuthenticationMethod",sub_params.auth_method,allow_overwrite=True) if sub_params.rsl is not None: self.dicts[dtype].add('GLIDEIN_GlobusRSL',sub_params.rsl,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_SlotsLayout", sub_params.config.submit.slots_layout, allow_overwrite=True) self.dicts['vars'].add_extended("GLIDEIN_REQUIRE_VOMS","boolean",sub_params.config.restrictions.require_voms_proxy,None,False,True,True) self.dicts['vars'].add_extended("GLIDEIN_REQUIRE_GLEXEC_USE","boolean",sub_params.config.restrictions.require_glidein_glexec_use,None,False,True,True) # populate infosys for infosys_ref in sub_params.infosys_refs: self.dicts['infosys'].add_extended(infosys_ref['type'],infosys_ref['server'],infosys_ref['ref'],allow_overwrite=True) # populate monitorgroups for monitorgroup in sub_params.monitorgroups: self.dicts['mongroup'].add_extended(monitorgroup['group_name'],allow_overwrite=True) # populate complex files populate_job_descript(self.work_dir,self.dicts['job_descript'], self.sub_name,sub_params) self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE, params.factory_name,params.glidein_name,self.sub_name, sub_params.gridtype,sub_params.gatekeeper, sub_params.rsl, sub_params.auth_method, params.web_url,sub_params.proxy_url,sub_params.work_dir, params.submit.base_client_log_dir)
def add_file_unparsed(user_file, dicts, is_factory): absfname = user_file["absfname"] if "relfname" not in user_file: relfname = os.path.basename(absfname) # defualt is the final part of absfname else: relfname = user_file["relfname"] is_const = eval(user_file["const"]) is_executable = eval(user_file["executable"]) is_wrapper = eval(user_file["wrapper"]) do_untar = eval(user_file["untar"]) prefix = user_file["prefix"] period_value = int(user_file["period"]) file_list_idx = "file_list" if "after_entry" in user_file: if eval(user_file["after_entry"]): file_list_idx = "after_file_list" if is_executable: # a script dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "exec", period_value, prefix), absfname, ) elif is_wrapper: # a sourceable script for the wrapper dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "wrapper"), absfname ) elif do_untar: # a tarball untar_opts = user_file.get_child("untar_options") if u"dir" in untar_opts: wnsubdir = untar_opts["dir"] else: wnsubdir = string.split(relfname, ".", 1)[0] # deafult is relfname up to the first . if "absdir_outattr" in untar_opts: config_out = untar_opts["absdir_outattr"] else: config_out = "FALSE" cond_attr = untar_opts["cond_attr"] dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( cWConsts.insert_timestr(relfname), "untar", cond_download=cond_attr, config_out=config_out ), absfname, ) dicts["untar_cfg"].add(relfname, wnsubdir) else: # not executable nor tarball => simple file if is_const: val = "regular" dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), val), absfname ) else: val = "nocache" dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple(relfname, val), absfname ) # no timestamp if it can be modified
def populate(self, other=None): # put default files in place first self.dicts["file_list"].add_placeholder("error_gen.sh", allow_overwrite=True) self.dicts["file_list"].add_placeholder("error_augment.sh", allow_overwrite=True) self.dicts["file_list"].add_placeholder("setup_script.sh", allow_overwrite=True) self.dicts["file_list"].add_placeholder(cWConsts.CONSTS_FILE, allow_overwrite=True) self.dicts["file_list"].add_placeholder(cWConsts.VARS_FILE, allow_overwrite=True) self.dicts["file_list"].add_placeholder( cWConsts.UNTAR_CFG_FILE, allow_overwrite=True ) # this one must be loaded before any tarball self.dicts["file_list"].add_placeholder( cWConsts.GRIDMAP_FILE, allow_overwrite=True ) # this one must be loaded before setup_x509.sh is run # load system files for file_name in ( "error_gen.sh", "error_augment.sh", "parse_starterlog.awk", "advertise_failure.helper", "condor_config", "condor_config.multi_schedd.include", "condor_config.dedicated_starter.include", "condor_config.check.include", "condor_config.monitor.include", ): self.dicts["file_list"].add_from_file( file_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(file_name), "regular"), os.path.join(cgWConsts.WEB_BASE_DIR, file_name), ) self.dicts["description"].add("condor_config", "condor_config") self.dicts["description"].add("condor_config.multi_schedd.include", "condor_config_multi_include") self.dicts["description"].add("condor_config.dedicated_starter.include", "condor_config_main_include") self.dicts["description"].add("condor_config.monitor.include", "condor_config_monitor_include") self.dicts["description"].add("condor_config.check.include", "condor_config_check_include") self.dicts["vars"].load(cgWConsts.WEB_BASE_DIR, "condor_vars.lst", change_self=False, set_not_changed=False) # # Note: # We expect the condor platform info to be coming in as parameters # as FE provided consts file is not available at this time # # add the basic standard params self.dicts["params"].add("GLIDEIN_Report_Failed", "NEVER") self.dicts["params"].add("CONDOR_OS", "default") self.dicts["params"].add("CONDOR_ARCH", "default") self.dicts["params"].add("CONDOR_VERSION", "default") # Load initial system scripts # These should be executed before the other scripts for script_name in ("setup_script.sh", "cat_consts.sh", "condor_platform_select.sh"): self.dicts["file_list"].add_from_file( script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, script_name), ) # load condor tarballs # only one will be downloaded in the end... based on what condor_platform_select.sh decides condor_tarballs = self.conf.get_child_list(u"condor_tarballs") prev_tar_dir_map = {} if other is not None and other.main_dicts.dicts["glidein"].has_key("CondorTarballDirMap"): prev_tar_dir_map = eval(other.main_dicts.dicts["glidein"]["CondorTarballDirMap"]) tar_dir_map = {} for condor_idx in range(len(condor_tarballs)): condor_el = condor_tarballs[condor_idx] # condor_el now is a combination of csv version+os+arch # Get list of valid tarballs for this condor_el # Register the tarball, but make download conditional to cond_name condor_el_valid_tarballs = get_valid_condor_tarballs([condor_el]) condor_fname = cWConsts.insert_timestr(cgWConsts.CONDOR_FILE % condor_idx) condor_fd = None if u"tar_file" in condor_el: # Condor tarball available. Just add it to the list of tarballs # with every possible condor_platform string condor_tarfile = condor_el[u"tar_file"] # already built this tarball, just reuse it elif condor_el[u"base_dir"] in prev_tar_dir_map: condor_tarfile = prev_tar_dir_map[condor_el[u"base_dir"]] tar_dir_map[condor_el[u"base_dir"]] = condor_tarfile else: # Create a new tarball as usual condor_fd = cgWCreate.create_condor_tar_fd(condor_el[u"base_dir"]) tar_dir_map[condor_el[u"base_dir"]] = os.path.join(self.dicts["file_list"].dir, condor_fname) for tar in condor_el_valid_tarballs: condor_platform = "%s-%s-%s" % (tar["version"], tar["os"], tar["arch"]) cond_name = "CONDOR_PLATFORM_%s" % condor_platform condor_platform_fname = cgWConsts.CONDOR_FILE % condor_platform if condor_fd is None: # tar file exists. Just use it self.dicts["file_list"].add_from_file( condor_platform_fname, cWDictFile.FileDictFile.make_val_tuple( condor_fname, "untar", cond_download=cond_name, config_out=cgWConsts.CONDOR_ATTR ), condor_tarfile, ) else: # This is addition of new tarfile # Need to rewind fd every time condor_fd.seek(0) self.dicts["file_list"].add_from_fd( condor_platform_fname, cWDictFile.FileDictFile.make_val_tuple( condor_fname, "untar", cond_download=cond_name, config_out=cgWConsts.CONDOR_ATTR ), condor_fd, ) self.dicts["untar_cfg"].add(condor_platform_fname, cgWConsts.CONDOR_DIR) # Add cond_name in the config, so that it is known # But leave it disabled by default self.dicts["consts"].add(cond_name, "0", allow_overwrite=False) if condor_fd is not None: condor_fd.close() self.dicts["glidein"].add("CondorTarballDirMap", str(tar_dir_map)) # # Note: # We expect the collector info to be coming in as parameter # as FE consts file is not available at this time # # add the basic standard params self.dicts["params"].add("GLIDEIN_Collector", "Fake") # add the factory monitoring collector parameter, if any collectors are defined # this is purely a factory thing factory_monitoring_collector = calc_monitoring_collectors_string( self.conf.get_child_list(u"monitoring_collectors") ) if factory_monitoring_collector is not None: self.dicts["params"].add("GLIDEIN_Factory_Collector", str(factory_monitoring_collector)) populate_gridmap(self.conf, self.dicts["gridmap"]) file_list_scripts = [ "collector_setup.sh", "create_temp_mapfile.sh", "setup_x509.sh", cgWConsts.CONDOR_STARTUP_FILE, ] after_file_list_scripts = [ "check_proxy.sh", "create_mapfile.sh", "validate_node.sh", "setup_network.sh", "gcb_setup.sh", "glexec_setup.sh", "java_setup.sh", "glidein_memory_setup.sh", "glidein_cpus_setup.sh", "glidein_sitewms_setup.sh", "script_wrapper.sh", "smart_partitionable.sh", ] # Only execute scripts once duplicate_scripts = set(file_list_scripts).intersection(after_file_list_scripts) if duplicate_scripts: raise RuntimeError, "Duplicates found in the list of files to execute '%s'" % ",".join(duplicate_scripts) # Load more system scripts for script_name in file_list_scripts: self.dicts["file_list"].add_from_file( script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, script_name), ) # Add the drainer script drain_script = "check_wn_drainstate.sh" self.dicts["file_list"].add_from_file( drain_script, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(drain_script), "exec", 60), os.path.join(cgWConsts.WEB_BASE_DIR, drain_script), ) # make sure condor_startup does not get executed ahead of time under normal circumstances # but must be loaded early, as it also works as a reporting script in case of error self.dicts["description"].add(cgWConsts.CONDOR_STARTUP_FILE, "last_script") # # At this point in the glideins, condor_advertize should be able to talk to the FE collector # # put user files in stage for file in self.conf.get_child_list(u"files"): add_file_unparsed(file, self.dicts, True) # put user attributes into config files for attr in self.conf.get_child_list(u"attrs"): add_attr_unparsed(attr, self.dicts, "main") # add additional system scripts for script_name in after_file_list_scripts: self.dicts["after_file_list"].add_from_file( script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, script_name), ) # populate complex files populate_factory_descript( self.work_dir, self.dicts["glidein"], self.active_sub_list, self.disabled_sub_list, self.conf ) populate_frontend_descript(self.dicts["frontend_descript"], self.conf) # populate the monitor files javascriptrrd_dir = self.conf.get_child(u"monitor")[u"javascriptRRD_dir"] for mfarr in ((cgWConsts.WEB_BASE_DIR, "factory_support.js"), (javascriptrrd_dir, "javascriptrrd.wlibs.js")): mfdir, mfname = mfarr parent_dir = self.find_parent_dir(mfdir, mfname) mfobj = cWDictFile.SimpleFile(parent_dir, mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ( (cgWConsts.WEB_BASE_DIR, "factoryRRDBrowse.html"), (cgWConsts.WEB_BASE_DIR, "factoryRRDEntryMatrix.html"), (cgWConsts.WEB_BASE_DIR, "factoryStatus.html"), (cgWConsts.WEB_BASE_DIR, "factoryLogStatus.html"), (cgWConsts.WEB_BASE_DIR, "factoryCompletedStats.html"), (cgWConsts.WEB_BASE_DIR, "factoryStatusNow.html"), (cgWConsts.WEB_BASE_DIR, "factoryEntryStatusNow.html"), ): mfdir, mfname = mfarr mfobj = cWDictFile.SimpleFile(mfdir, mfname) mfobj.load() self.monitor_htmls.append(mfobj) # add the index page and its images mfobj = cWDictFile.SimpleFile(cgWConsts.WEB_BASE_DIR + "/factory/", "index.html") mfobj.load() self.monitor_htmls.append(mfobj) for imgfile in ( "factoryCompletedStats.png", "factoryEntryStatusNow.png", "factoryLogStatus.png", "factoryRRDBrowse.png", "factoryRRDEntryMatrix.png", "factoryStatus.png", "factoryStatusNow.png", ): mfobj = cWDictFile.SimpleFile(cgWConsts.WEB_BASE_DIR + "/factory/images/", imgfile) mfobj.load() self.monitor_images.append(mfobj)
def populate(self,params=None): if params is None: params=self.params # put default files in place first self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball self.dicts['preentry_file_list'].add_placeholder(cWConsts.GRIDMAP_FILE,allow_overwrite=True) # this one must be loaded before factory runs setup_x509.sh # follow by the blacklist file file_name=cWConsts.BLACKLIST_FILE self.dicts['preentry_file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh',"check_blacklist.sh"): self.dicts['preentry_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # put user files in stage for user_file in params.files: add_file_unparsed(user_file,self.dicts) # start expr is special start_expr=None # put user attributes into config files for attr_name in params.attrs.keys(): if attr_name in ('GLIDECLIENT_Start','GLIDECLIENT_Group_Start'): if start_expr is None: start_expr=params.attrs[attr_name].value elif not (params.attrs[attr_name].value in (None,'True')): start_expr="(%s)&&(%s)"%(start_expr,params.attrs[attr_name].value) # delete from the internal structure... will use it in match section del params.data['attrs'][attr_name] else: add_attr_unparsed(attr_name, params,self.dicts,"main") real_start_expr=params.match.start_expr if start_expr is not None: if real_start_expr!='True': real_start_expr="(%s)&&(%s)"%(real_start_expr,start_expr) else: real_start_expr=start_expr # since I removed the attributes, roll back into the match.start_expr params.data['match']['start_expr']=real_start_expr self.dicts['consts'].add('GLIDECLIENT_Start',real_start_expr) # create GLIDEIN_Collector attribute self.dicts['params'].add_extended('GLIDEIN_Collector',False,str(calc_glidein_collectors(params.collectors))) populate_gridmap(params,self.dicts['gridmap']) if self.dicts['preentry_file_list'].is_placeholder(cWConsts.GRIDMAP_FILE): # gridmapfile is optional, so if not loaded, remove the placeholder self.dicts['preentry_file_list'].remove(cWConsts.GRIDMAP_FILE) # populate complex files populate_frontend_descript(self.work_dir,self.dicts['frontend_descript'],self.active_sub_list,params) populate_common_descript(self.dicts['frontend_descript'],params) # populate the monitor files javascriptrrd_dir = params.monitor.javascriptRRD_dir for mfarr in ((params.src_dir,'frontend_support.js'), (javascriptrrd_dir,'rrdFlot.js'), (javascriptrrd_dir,'rrdFlotMatrix.js'), (javascriptrrd_dir,'rrdFlotSupport.js'), (javascriptrrd_dir,'rrdFile.js'), (javascriptrrd_dir,'rrdFilter.js'), (javascriptrrd_dir,'binaryXHR.js'), (params.monitor.flot_dir,'jquery.flot.js'), (params.monitor.flot_dir,'jquery.flot.selection.js'), (params.monitor.flot_dir,'jquery.flot.tooltip.js'), (params.monitor.flot_dir,'excanvas.js'), (params.monitor.jquery_dir,'jquery.js')): mfdir,mfname=mfarr parent_dir = self.find_parent_dir(mfdir,mfname) mfobj=cWDictFile.SimpleFile(parent_dir,mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ((params.src_dir,'frontendRRDBrowse.html'), (params.src_dir,'frontendRRDGroupMatrix.html'), (params.src_dir,'frontendGroupGraphStatusNow.html'), (params.src_dir,'frontendStatus.html')): mfdir,mfname=mfarr mfobj=cWDictFile.SimpleFile(mfdir,mfname) mfobj.load() self.monitor_htmls.append(mfobj) spd = self.params.data useMonitorIndexPage = True if spd.has_key('frontend_monitor_index_page'): useMonitorIndexPage = spd['frontend_monitor_index_page'] in ('True', 'true', '1') if useMonitorIndexPage: mfobj = cWDictFile.SimpleFile(params.src_dir + '/frontend', 'index.html') mfobj.load() self.monitor_htmls.append(mfobj) for imgfil in ('frontendGroupGraphsNow.small.png', 'frontendRRDBrowse.small.png', 'frontendRRDGroupMatix.small.png', 'frontendStatus.small.png'): mfobj = cWDictFile.SimpleFile(params.src_dir + '/frontend/images', imgfil) mfobj.load() self.monitor_htmls.append(mfobj) # Tell condor to advertise GLIDECLIENT_ReqNode self.dicts['vars'].add_extended('GLIDECLIENT_ReqNode','string',None,None,False,True,False) # derive attributes populate_common_attrs(self.dicts) # populate security data populate_main_security(self.client_security,params)
def populate(self,params=None): if params is None: params=self.params sub_params=params.groups[self.sub_name] # put default files in place first self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball # follow by the blacklist file file_name=cWConsts.BLACKLIST_FILE self.dicts['preentry_file_list'].add_from_file(file_name,(file_name,"nocache","TRUE",'BLACKLIST_FILE'),os.path.join(params.src_dir,file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh',"check_blacklist.sh"): self.dicts['preentry_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # put user files in stage for user_file in sub_params.files: add_file_unparsed(user_file,self.dicts) # start expr is special start_expr=None # put user attributes into config files for attr_name in sub_params.attrs.keys(): if attr_name in ('GLIDECLIENT_Group_Start','GLIDECLIENT_Start'): if start_expr is None: start_expr=sub_params.attrs[attr_name].value elif sub_params.attrs[attr_name].value is not None: start_expr="(%s)&&(%s)"%(start_expr,sub_params.attrs[attr_name].value) # delete from the internal structure... will use it in match section del sub_params.data['attrs'][attr_name] else: add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name) real_start_expr=sub_params.match.start_expr if start_expr is not None: if real_start_expr!='True': real_start_expr="(%s)&&(%s)"%(real_start_expr,start_expr) else: real_start_expr=start_expr # since I removed the attributes, roll back into the match.start_expr sub_params.data['match']['start_expr']=real_start_expr self.dicts['consts'].add('GLIDECLIENT_Group_Start',real_start_expr) # derive attributes populate_common_attrs(self.dicts) # populate complex files populate_group_descript(self.work_dir,self.dicts['group_descript'], self.sub_name,sub_params) populate_common_descript(self.dicts['group_descript'],sub_params) # populate security data populate_main_security(self.client_security,params) populate_group_security(self.client_security,params,sub_params)
def populate(self,params=None): if params==None: params=self.params # put default files in place first self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh','setup_x509.sh'): self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) #load system files for file_name in ('parse_starterlog.awk', "condor_config", "condor_config.multi_schedd.include", "condor_config.dedicated_starter.include", "condor_config.check.include", "condor_config.monitor.include"): self.dicts['file_list'].add_from_file(file_name,(cWConsts.insert_timestr(file_name),"regular","TRUE",'FALSE'),os.path.join(params.src_dir,file_name)) self.dicts['description'].add("condor_config","condor_config") self.dicts['description'].add("condor_config.multi_schedd.include","condor_config_multi_include") self.dicts['description'].add("condor_config.dedicated_starter.include","condor_config_main_include") self.dicts['description'].add("condor_config.monitor.include","condor_config_monitor_include") self.dicts['description'].add("condor_config.check.include","condor_config_check_include") self.dicts['vars'].load(params.src_dir,'condor_vars.lst',change_self=False,set_not_changed=False) # put user files in stage for file in params.files: add_file_unparsed(file,self.dicts) # put user attributes into config files for attr_name in params.attrs.keys(): add_attr_unparsed(attr_name, params,self.dicts,"main") # add the basic standard params self.dicts['params'].add("GLIDEIN_Collector",'Fake') # Prepare to load condor tarballs (after entry, as I may need data from there) for script_name in ('validate_node.sh','condor_platform_select.sh'): self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) #load condor tarballs for condor_idx in range(len(params.condor_tarballs)): condor_el=params.condor_tarballs[condor_idx] condor_platform="%s-%s-%s"%(condor_el.version,condor_el.os,condor_el.arch) cond_name="CONDOR_PLATFORM_%s"%condor_platform condor_fname=cgWConsts.CONDOR_FILE%condor_platform # register the tarball, but make the download conditional to cond_name if condor_el.tar_file!=None: # condor tarball available self.dicts['after_file_list'].add_from_file(condor_fname,(cWConsts.insert_timestr(condor_fname),"untar",cond_name,cgWConsts.CONDOR_ATTR),condor_el.tar_file) else: # create a new tarball condor_fd=cgWCreate.create_condor_tar_fd(condor_el.base_dir) condor_fname=cWConsts.insert_timestr(condor_fname) self.dicts['after_file_list'].add_from_fd(condor_fname,(condor_fname,"untar",cond_name,cgWConsts.CONDOR_ATTR),condor_fd) condor_fd.close() # insert the newly created tarball fname back into the config params.subparams.data['condor_tarballs'][condor_idx]['tar_file']=os.path.join(self.dicts['file_list'].dir,condor_fname) # add cond_name in the config, so that it is known it is there # but leave it disabled by default self.dicts['consts'].add(cond_name,"0",allow_overwrite=False) self.dicts['untar_cfg'].add(condor_fname,cgWConsts.CONDOR_DIR) # add additional system scripts for script_name in ('create_mapfile.sh','collector_setup.sh','gcb_setup.sh','glexec_setup.sh','java_setup.sh'): self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # this must be the last script in the list for script_name in (cgWConsts.CONDOR_STARTUP_FILE,): self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) self.dicts['description'].add(cgWConsts.CONDOR_STARTUP_FILE,"last_script") # populate complex files populate_factory_descript(self.work_dir,self.dicts['glidein'],self.active_sub_list,params) populate_frontend_descript(self.dicts['frontend_descript'],params) # populate the monitor files javascriptrrd_dir = params.monitor.javascriptRRD_dir for mfarr in ((params.src_dir,'factory_support.js'), (javascriptrrd_dir,'rrdFlot.js'), (javascriptrrd_dir,'rrdFlotMatrix.js'), (javascriptrrd_dir,'rrdFlotSupport.js'), (javascriptrrd_dir,'rrdFile.js'), (javascriptrrd_dir,'rrdMultiFile.js'), (javascriptrrd_dir,'rrdFilter.js'), (javascriptrrd_dir,'binaryXHR.js'), (params.monitor.flot_dir,'jquery.flot.js'), (params.monitor.flot_dir,'jquery.flot.selection.js'), (params.monitor.flot_dir,'excanvas.js'), (params.monitor.jquery_dir,'jquery.js')): mfdir,mfname=mfarr parent_dir = self.find_parent_dir(mfdir,mfname) mfobj=cWDictFile.SimpleFile(parent_dir,mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ((params.src_dir,'factoryRRDBrowse.html'), (params.src_dir,'factoryRRDEntryMatrix.html'), (params.src_dir,'factoryStatus.html'), (params.src_dir,'factoryLogStatus.html'), (params.src_dir,'factoryCompletedStats.html'), (params.src_dir,'factoryStatusNow.html'), (params.src_dir,'factoryEntryStatusNow.html')): mfdir,mfname=mfarr mfobj=cWDictFile.SimpleFile(mfdir,mfname) mfobj.load() self.monitor_htmls.append(mfobj)
def populate(self, entry, schedd): # put default files in place first self.dicts["file_list"].add_placeholder(cWConsts.CONSTS_FILE, allow_overwrite=True) self.dicts["file_list"].add_placeholder(cWConsts.VARS_FILE, allow_overwrite=True) self.dicts["file_list"].add_placeholder( cWConsts.UNTAR_CFG_FILE, allow_overwrite=True ) # this one must be loaded before any tarball # follow by the blacklist file file_name = cWConsts.BLACKLIST_FILE self.dicts["file_list"].add_from_file( file_name, cWDictFile.FileDictFile.make_val_tuple(file_name, "nocache", config_out="BLACKLIST_FILE"), os.path.join(cgWConsts.WEB_BASE_DIR, file_name), ) # Load initial system scripts # These should be executed before the other scripts for script_name in ("cat_consts.sh", "check_blacklist.sh"): self.dicts["file_list"].add_from_file( script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, script_name), ) # load system files self.dicts["vars"].load( cgWConsts.WEB_BASE_DIR, "condor_vars.lst.entry", change_self=False, set_not_changed=False ) # put user files in stage for user_file in entry.get_child_list(u"files"): add_file_unparsed(user_file, self.dicts, True) # Add attribute for voms # put user attributes into config files for attr in entry.get_child_list(u"attrs"): add_attr_unparsed(attr, self.dicts, self.sub_name) # put standard attributes into config file # override anything the user set config = entry.get_child(u"config") restrictions = config.get_child(u"restrictions") submit = config.get_child(u"submit") for dtype in ("attrs", "consts"): self.dicts[dtype].add("GLIDEIN_Gatekeeper", entry[u"gatekeeper"], allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_GridType", entry[u"gridtype"], allow_overwrite=True) # MERGENOTE: # GLIDEIN_REQUIRE_VOMS publishes an attribute so that users without VOMS proxies # can avoid sites that require VOMS proxies (using the normal Condor Requirements # string. self.dicts[dtype].add("GLIDEIN_REQUIRE_VOMS", restrictions[u"require_voms_proxy"], allow_overwrite=True) self.dicts[dtype].add( "GLIDEIN_REQUIRE_GLEXEC_USE", restrictions[u"require_glidein_glexec_use"], allow_overwrite=True ) self.dicts[dtype].add("GLIDEIN_TrustDomain", entry[u"trust_domain"], allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_SupportedAuthenticationMethod", entry[u"auth_method"], allow_overwrite=True) if u"rsl" in entry: self.dicts[dtype].add("GLIDEIN_GlobusRSL", entry[u"rsl"], allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_SlotsLayout", submit[u"slots_layout"], allow_overwrite=True) self.dicts["vars"].add_extended( "GLIDEIN_REQUIRE_VOMS", "boolean", restrictions[u"require_voms_proxy"], None, False, True, True ) self.dicts["vars"].add_extended( "GLIDEIN_REQUIRE_GLEXEC_USE", "boolean", restrictions[u"require_glidein_glexec_use"], None, False, True, True, ) # populate infosys for infosys_ref in entry.get_child_list(u"infosys_refs"): self.dicts["infosys"].add_extended( infosys_ref[u"type"], infosys_ref[u"server"], infosys_ref[u"ref"], allow_overwrite=True ) # populate monitorgroups for monitorgroup in entry.get_child_list(u"monitorgroups"): self.dicts["mongroup"].add_extended(monitorgroup[u"group_name"], allow_overwrite=True) # populate complex files populate_job_descript(self.work_dir, self.dicts["job_descript"], self.sub_name, entry, schedd) ################################################################################################################ # This is the original function call: # # self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE, # params.factory_name,params.glidein_name,self.sub_name, # sub_params.gridtype,sub_params.gatekeeper, sub_params.rsl, sub_params.auth_method, # params.web_url,sub_params.proxy_url,sub_params.work_dir, # params.submit.base_client_log_dir, sub_params.submit.submit_attrs) # # Almost all of the parameters are attributes of params and/or sub_params. Instead of maintaining an ever # increasing parameter list for this function, lets just pass params, sub_params, and the 2 other parameters # to the function and call it a day. ################################################################################################################ self.dicts["condor_jdl"].populate(cgWConsts.STARTUP_FILE, self.sub_name, self.conf, entry)
def add_file_unparsed(file,dicts): absfname=file.absfname if absfname==None: raise RuntimeError, "Found a file element without an absname: %s"%file relfname=file.relfname if relfname==None: relfname=os.path.basename(absfname) # defualt is the final part of absfname if len(relfname)<1: raise RuntimeError, "Found a file element with an empty relfname: %s"%file is_const=eval(file.const,{},{}) is_executable=eval(file.executable,{},{}) is_wrapper=eval(file.wrapper,{},{}) do_untar=eval(file.untar,{},{}) file_list_idx='file_list' if file.has_key('after_entry'): if eval(file.after_entry,{},{}): file_list_idx='after_file_list' if is_executable: # a script if not is_const: raise RuntimeError, "A file cannot be executable if it is not constant: %s"%file if do_untar: raise RuntimeError, "A tar file cannot be executable: %s"%file if is_wrapper: raise RuntimeError, "A wrapper file cannot be executable: %s"%file dicts[file_list_idx].add_from_file(relfname,(cWConsts.insert_timestr(relfname),"exec","TRUE",'FALSE'),absfname) elif is_wrapper: # a sourceable script for the wrapper if not is_const: raise RuntimeError, "A file cannot be a wrapper if it is not constant: %s"%file if do_untar: raise RuntimeError, "A tar file cannot be a wrapper: %s"%file dicts[file_list_idx].add_from_file(relfname,(cWConsts.insert_timestr(relfname),"wrapper","TRUE",'FALSE'),absfname) elif do_untar: # a tarball if not is_const: raise RuntimeError, "A file cannot be untarred if it is not constant: %s"%file wnsubdir=file.untar_options.dir if wnsubdir==None: wnsubdir=string.split(relfname,'.',1)[0] # deafult is relfname up to the first . config_out=file.untar_options.absdir_outattr if config_out==None: config_out="FALSE" cond_attr=file.untar_options.cond_attr dicts[file_list_idx].add_from_file(relfname,(cWConsts.insert_timestr(relfname),"untar",cond_attr,config_out),absfname) dicts['untar_cfg'].add(relfname,wnsubdir) else: # not executable nor tarball => simple file if is_const: val='regular' dicts[file_list_idx].add_from_file(relfname,(cWConsts.insert_timestr(relfname),val,'TRUE','FALSE'),absfname) else: val='nocache' dicts[file_list_idx].add_from_file(relfname,(relfname,val,'TRUE','FALSE'),absfname) # no timestamp if it can be modified
def add_file_unparsed(user_file, dicts, is_factory): """Add a user file residing in the stage area file as described by Params.file_defaults :param user_file: file from the config files "files" sections :param dicts: parameters dctionaries :param is_factory: True if invoked for the factory (cgWParamDict.py), false for the frontend (cvWParamDict.py) :return: None (dictionaries are modified) """ absfname = user_file.absfname if absfname is None: raise RuntimeError("Found a file element without an absname: %s" % user_file) relfname = user_file.relfname if relfname is None: relfname = os.path.basename(absfname) # defualt is the final part of absfname if len(relfname) < 1: raise RuntimeError("Found a file element with an empty relfname: %s" % user_file) is_const = is_true(user_file.const) is_executable = is_true(user_file.executable) is_wrapper = is_true(user_file.wrapper) do_untar = is_true(user_file.untar) try: period_value = int(user_file.period) except (AttributeError, KeyError, ValueError): period_value = 0 if is_factory: # Factory (file_list, after_file_list) file_list_idx = 'file_list' if user_file.has_key('after_entry'): if is_true(user_file.after_entry): # eval(user_file.after_entry,{},{}): file_list_idx = 'after_file_list' else: # Frontend (preentry_file_list, file_list, aftergroup_preentry_file_list, aftergroup_file_list) file_list_idx = 'preentry_file_list' if user_file.has_key('after_entry'): if is_true(user_file.after_entry): file_list_idx = 'file_list' if user_file.has_key('after_group'): if is_true(user_file.after_group): file_list_idx = 'aftergroup_%s' % file_list_idx # period has 0 as default (in dictionary definition). Should I still protect against it not being defined? if period_value > 0: if not is_executable: raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) if is_executable: # a script if not is_const: raise RuntimeError("A file cannot be executable if it is not constant: %s" % user_file) if do_untar: raise RuntimeError("A tar file cannot be executable: %s" % user_file) if is_wrapper: raise RuntimeError("A wrapper file cannot be an executable: %s" % user_file) dicts[file_list_idx].add_from_file(relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), 'exec', user_file.period, user_file.prefix), absfname) elif is_wrapper: # a sourceable script for the wrapper if not is_const: raise RuntimeError("A file cannot be a wrapper if it is not constant: %s" % user_file) if do_untar: raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file) dicts[file_list_idx].add_from_file(relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), 'wrapper'), absfname) elif do_untar: # a tarball if not is_const: raise RuntimeError("A file cannot be untarred if it is not constant: %s" % user_file) wnsubdir = user_file.untar_options.dir if wnsubdir is None: wnsubdir = string.split(relfname, '.', 1)[0] # deafult is relfname up to the first . config_out = user_file.untar_options.absdir_outattr if config_out is None: config_out = "FALSE" cond_attr = user_file.untar_options.cond_attr dicts[file_list_idx].add_from_file(relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), 'untar', cond_download=cond_attr, config_out=config_out), absfname) dicts['untar_cfg'].add(relfname, wnsubdir) else: # not executable nor tarball => simple file if is_const: val = 'regular' dicts[file_list_idx].add_from_file(relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), val), absfname) else: val = 'nocache' dicts[file_list_idx].add_from_file(relfname, cWDictFile.FileDictFile.make_val_tuple(relfname, val), absfname) # no timestamp in the name if it can be modified