def populate(self,params=None): if params is None: params=self.params # put default files in place first self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball self.dicts['preentry_file_list'].add_placeholder(cWConsts.GRIDMAP_FILE,allow_overwrite=True) # this one must be loaded before factory runs setup_x509.sh # follow by the blacklist file file_name = cWConsts.BLACKLIST_FILE self.dicts['preentry_file_list'].add_from_file(file_name, cWDictFile.FileDictFile.make_val_tuple(file_name, "nocache", config_out='BLACKLIST_FILE'), os.path.join(params.src_dir, file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh', 'check_blacklist.sh'): self.dicts['preentry_file_list'].add_from_file(script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), 'exec'), os.path.join(params.src_dir, script_name)) # put user files in stage for user_file in params.files: add_file_unparsed(user_file,self.dicts, False) # start expr is special start_expr=None # put user attributes into config files for attr_name in params.attrs.keys(): if attr_name in ('GLIDECLIENT_Start','GLIDECLIENT_Group_Start'): if start_expr is None: start_expr=params.attrs[attr_name].value elif not (params.attrs[attr_name].value in (None,'True')): start_expr="(%s)&&(%s)"%(start_expr,params.attrs[attr_name].value) # delete from the internal structure... will use it in match section del params.data['attrs'][attr_name] else: add_attr_unparsed(attr_name, params,self.dicts,"main") real_start_expr=params.match.start_expr if start_expr is not None: if real_start_expr!='True': real_start_expr="(%s)&&(%s)"%(real_start_expr,start_expr) else: real_start_expr=start_expr # since I removed the attributes, roll back into the match.start_expr params.data['match']['start_expr']=real_start_expr self.dicts['consts'].add('GLIDECLIENT_Start',real_start_expr) # create GLIDEIN_Collector attribute self.dicts['params'].add_extended('GLIDEIN_Collector', False, str(calc_glidein_collectors(params.collectors))) # create GLIDEIN_CCB attribute only if CCBs list is in config file tmp_glidein_ccbs_string = str(calc_glidein_ccbs(params.ccbs)) if tmp_glidein_ccbs_string: self.dicts['params'].add_extended('GLIDEIN_CCB', False, tmp_glidein_ccbs_string) populate_gridmap(params,self.dicts['gridmap']) if self.dicts['preentry_file_list'].is_placeholder(cWConsts.GRIDMAP_FILE): # gridmapfile is optional, so if not loaded, remove the placeholder self.dicts['preentry_file_list'].remove(cWConsts.GRIDMAP_FILE) # populate complex files populate_frontend_descript(self.work_dir,self.dicts['frontend_descript'],self.active_sub_list,params) populate_common_descript(self.dicts['frontend_descript'],params) # Apply multicore policy so frontend can deal with multicore # glideins and requests correctly apply_multicore_policy(self.dicts['frontend_descript']) # populate the monitor files javascriptrrd_dir = params.monitor.javascriptRRD_dir for mfarr in ((params.src_dir,'frontend_support.js'), (javascriptrrd_dir,'javascriptrrd.wlibs.js')): mfdir,mfname=mfarr parent_dir = self.find_parent_dir(mfdir,mfname) mfobj=cWDictFile.SimpleFile(parent_dir,mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ((params.src_dir,'frontendRRDBrowse.html'), (params.src_dir,'frontendRRDGroupMatrix.html'), (params.src_dir,'frontendGroupGraphStatusNow.html'), (params.src_dir,'frontendStatus.html')): mfdir,mfname=mfarr mfobj=cWDictFile.SimpleFile(mfdir,mfname) mfobj.load() self.monitor_htmls.append(mfobj) mfobj = cWDictFile.SimpleFile(params.src_dir + '/frontend', 'index.html') mfobj.load() self.monitor_htmls.append(mfobj) for imgfil in ('frontendGroupGraphsNow.small.png', 'frontendRRDBrowse.small.png', 'frontendRRDGroupMatix.small.png', 'frontendStatus.small.png'): mfobj = cWDictFile.SimpleFile(params.src_dir + '/frontend/images', imgfil) mfobj.load() self.monitor_htmls.append(mfobj) # Tell condor to advertise GLIDECLIENT_ReqNode self.dicts['vars'].add_extended('GLIDECLIENT_ReqNode','string',None,None,False,True,False) # derive attributes populate_common_attrs(self.dicts) # populate security data populate_main_security(self.client_security,params)
def populate(self,params=None): if params is None: params=self.params # put default files in place first self.dicts['file_list'].add_placeholder('error_gen.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder('error_augment.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder('setup_script.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball self.dicts['file_list'].add_placeholder(cWConsts.GRIDMAP_FILE,allow_overwrite=True) # this one must be loaded before setup_x509.sh is run #load system files for file_name in ('error_gen.sh', 'error_augment.sh', 'parse_starterlog.awk', 'advertise_failure.helper', 'condor_config', 'condor_config.multi_schedd.include', 'condor_config.dedicated_starter.include', 'condor_config.check.include', 'condor_config.monitor.include'): self.dicts['file_list'].add_from_file(file_name, (cWConsts.insert_timestr(file_name), 'regular', 0, 'TRUE', 'FALSE'), os.path.join(params.src_dir, file_name)) self.dicts['description'].add("condor_config","condor_config") self.dicts['description'].add("condor_config.multi_schedd.include","condor_config_multi_include") self.dicts['description'].add("condor_config.dedicated_starter.include","condor_config_main_include") self.dicts['description'].add("condor_config.monitor.include","condor_config_monitor_include") self.dicts['description'].add("condor_config.check.include","condor_config_check_include") self.dicts['vars'].load(params.src_dir,'condor_vars.lst',change_self=False,set_not_changed=False) # # Note: # We expect the condor platform info to be coming in as parameters # as FE provided consts file is not available at this time # # add the basic standard params self.dicts['params'].add("GLIDEIN_Report_Failed",'NEVER') self.dicts['params'].add("CONDOR_OS",'default') self.dicts['params'].add("CONDOR_ARCH",'default') self.dicts['params'].add("CONDOR_VERSION",'default') # Load initial system scripts # These should be executed before the other scripts for script_name in ('setup_script.sh', 'cat_consts.sh', 'condor_platform_select.sh'): self.dicts['file_list'].add_from_file(script_name, (cWConsts.insert_timestr(script_name), 'exec', 0, 'TRUE', 'FALSE'), os.path.join(params.src_dir, script_name)) #load condor tarballs # only one will be downloaded in the end... based on what condor_platform_select.sh decides for condor_idx in range(len(params.condor_tarballs)): condor_el=params.condor_tarballs[condor_idx] # condor_el now is a combination of csv version+os+arch # Get list of valid tarballs for this condor_el # Register the tarball, but make download conditional to cond_name condor_el_valid_tarballs = get_valid_condor_tarballs([condor_el]) condor_fname = cWConsts.insert_timestr(cgWConsts.CONDOR_FILE % condor_idx) condor_tarfile = "" condor_fd = None if condor_el.tar_file is not None: # Condor tarball available. Just add it to the list of tarballs # with every possible condor_platform string condor_tarfile = condor_el.tar_file else: # Create a new tarball as usual condor_fd = cgWCreate.create_condor_tar_fd(condor_el.base_dir) condor_tarfile = os.path.join(self.dicts['file_list'].dir, condor_fname) # insert the newly created tarball fname back into the config params.subparams.data['condor_tarballs'][condor_idx]['tar_file'] = condor_tarfile for tar in condor_el_valid_tarballs: condor_platform = "%s-%s-%s" % (tar['version'], tar['os'], tar['arch']) cond_name = "CONDOR_PLATFORM_%s" % condor_platform condor_platform_fname = cgWConsts.CONDOR_FILE % condor_platform if condor_fd is None: # tar file exists. Just use it self.dicts['file_list'].add_from_file( condor_platform_fname, (condor_fname, 'untar', 0, cond_name, cgWConsts.CONDOR_ATTR), condor_el.tar_file ) else: # This is addition of new tarfile # Need to rewind fd everytime condor_fd.seek(0) self.dicts['file_list'].add_from_fd( condor_platform_fname, (condor_fname, 'untar', 0, cond_name, cgWConsts.CONDOR_ATTR), condor_fd ) self.dicts['untar_cfg'].add(condor_platform_fname, cgWConsts.CONDOR_DIR) # Add cond_name in the config, so that it is known # But leave it disabled by default self.dicts['consts'].add(cond_name, "0", allow_overwrite=False) if condor_fd is not None: condor_fd.close() # # Note: # We expect the collector info to be coming in as parameter # as FE consts file is not available at this time # # add the basic standard params self.dicts['params'].add("GLIDEIN_Collector",'Fake') # add the factory monitoring collector parameter, if any collectors are defined # this is purely a factory thing factory_monitoring_collector=calc_monitoring_collectors_string(params.monitoring_collectors) if factory_monitoring_collector is not None: self.dicts['params'].add('GLIDEIN_Factory_Collector',str(factory_monitoring_collector)) populate_gridmap(params,self.dicts['gridmap']) file_list_scripts = ['collector_setup.sh', 'create_temp_mapfile.sh', 'setup_x509.sh', cgWConsts.CONDOR_STARTUP_FILE] after_file_list_scripts = ['check_proxy.sh', 'create_mapfile.sh', 'validate_node.sh', 'setup_network.sh', 'gcb_setup.sh', 'glexec_setup.sh', 'java_setup.sh', 'glidein_memory_setup.sh', 'glidein_cpus_setup.sh', 'glidein_sitewms_setup.sh', 'script_wrapper.sh', 'smart_partitionable.sh'] # Only execute scripts once duplicate_scripts = set(file_list_scripts).intersection(after_file_list_scripts) if duplicate_scripts: raise RuntimeError, "Duplicates found in the list of files to execute '%s'" % ','.join(duplicate_scripts) # Load more system scripts for script_name in file_list_scripts: self.dicts['file_list'].add_from_file(script_name, (cWConsts.insert_timestr(script_name), 'exec', 0, 'TRUE', 'FALSE'), os.path.join(params.src_dir, script_name)) # make sure condor_startup does not get executed ahead of time under normal circumstances # but must be loaded early, as it also works as a reporting script in case of error self.dicts['description'].add(cgWConsts.CONDOR_STARTUP_FILE,"last_script") # # At this point in the glideins, condor_advertize should be able to talk to the FE collector # # put user files in stage for file in params.files: add_file_unparsed(file, self.dicts, True) # put user attributes into config files for attr_name in params.attrs.keys(): add_attr_unparsed(attr_name, params,self.dicts,"main") # add additional system scripts for script_name in after_file_list_scripts: self.dicts['after_file_list'].add_from_file(script_name, (cWConsts.insert_timestr(script_name), 'exec', 0, 'TRUE', 'FALSE'), os.path.join(params.src_dir, script_name)) # populate complex files populate_factory_descript(self.work_dir,self.dicts['glidein'],self.active_sub_list,params) populate_frontend_descript(self.dicts['frontend_descript'],params) # populate the monitor files javascriptrrd_dir = params.monitor.javascriptRRD_dir for mfarr in ((params.src_dir,'factory_support.js'), (javascriptrrd_dir,'javascriptrrd.wlibs.js')): mfdir,mfname=mfarr parent_dir = self.find_parent_dir(mfdir,mfname) mfobj=cWDictFile.SimpleFile(parent_dir,mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ((params.src_dir,'factoryRRDBrowse.html'), (params.src_dir,'factoryRRDEntryMatrix.html'), (params.src_dir,'factoryStatus.html'), (params.src_dir,'factoryLogStatus.html'), (params.src_dir,'factoryCompletedStats.html'), (params.src_dir,'factoryStatusNow.html'), (params.src_dir,'factoryEntryStatusNow.html')): mfdir,mfname=mfarr mfobj=cWDictFile.SimpleFile(mfdir,mfname) mfobj.load() self.monitor_htmls.append(mfobj) # add the index page and its images mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/', 'index.html') mfobj.load() self.monitor_htmls.append(mfobj) for imgfile in ('factoryCompletedStats.png', 'factoryEntryStatusNow.png', 'factoryLogStatus.png', 'factoryRRDBrowse.png', 'factoryRRDEntryMatrix.png', 'factoryStatus.png', 'factoryStatusNow.png'): mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/images/', imgfile) mfobj.load() self.monitor_images.append(mfobj)
def populate(self,params=None): if params is None: params=self.params sub_params=params.groups[self.sub_name] # put default files in place first self.dicts['preentry_file_list'].add_placeholder(cWConsts.CONSTS_FILE, allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.VARS_FILE, allow_overwrite=True) self.dicts['preentry_file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE, allow_overwrite=True) # this one must be loaded before any tarball # follow by the blacklist file file_name = cWConsts.BLACKLIST_FILE self.dicts['preentry_file_list'].add_from_file(file_name, cWDictFile.FileDictFile.make_val_tuple(file_name, "nocache", config_out='BLACKLIST_FILE'), os.path.join(params.src_dir, file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh', "check_blacklist.sh"): self.dicts['preentry_file_list'].add_from_file(script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), 'exec'), os.path.join(params.src_dir, script_name)) # put user files in stage for user_file in sub_params.files: add_file_unparsed(user_file, self.dicts, False) # start expr is special start_expr=None # put user attributes into config files for attr_name in sub_params.attrs.keys(): if attr_name in ('GLIDECLIENT_Group_Start','GLIDECLIENT_Start'): if start_expr is None: start_expr=sub_params.attrs[attr_name].value elif sub_params.attrs[attr_name].value is not None: start_expr="(%s)&&(%s)"%(start_expr,sub_params.attrs[attr_name].value) # delete from the internal structure... will use it in match section del sub_params.data['attrs'][attr_name] else: add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name) real_start_expr=sub_params.match.start_expr if start_expr is not None: if real_start_expr!='True': real_start_expr="(%s)&&(%s)"%(real_start_expr,start_expr) else: real_start_expr=start_expr # since I removed the attributes, roll back into the match.start_expr sub_params.data['match']['start_expr']=real_start_expr self.dicts['consts'].add('GLIDECLIENT_Group_Start',real_start_expr) # derive attributes populate_common_attrs(self.dicts) # populate complex files populate_group_descript(self.work_dir,self.dicts['group_descript'], self.sub_name,sub_params) populate_common_descript(self.dicts['group_descript'],sub_params) # Apply group specific glexec policy apply_group_glexec_policy(self.dicts['group_descript'], sub_params, params) # populate security data populate_main_security(self.client_security,params) populate_group_security(self.client_security,params,sub_params)
def populate(self,params=None): if params is None: params=self.params sub_params=params.entries[self.sub_name] # put default files in place first self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball # follow by the blacklist file file_name=cWConsts.BLACKLIST_FILE self.dicts['file_list'].add_from_file(file_name, (file_name, 'nocache', 0, 'TRUE', 'BLACKLIST_FILE'), os.path.join(params.src_dir, file_name)) # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh', 'check_blacklist.sh'): self.dicts['file_list'].add_from_file(script_name, (cWConsts.insert_timestr(script_name), 'exec', 0, 'TRUE', 'FALSE'), os.path.join(params.src_dir, script_name)) #load system files self.dicts['vars'].load(params.src_dir,'condor_vars.lst.entry',change_self=False,set_not_changed=False) # put user files in stage for user_file in sub_params.files: add_file_unparsed(user_file, self.dicts, True) # Add attribute for voms # put user attributes into config files for attr_name in sub_params.attrs.keys(): add_attr_unparsed(attr_name, sub_params,self.dicts,self.sub_name) # put standard attributes into config file # override anything the user set for dtype in ('attrs','consts'): self.dicts[dtype].add("GLIDEIN_Gatekeeper",sub_params.gatekeeper,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_GridType",sub_params.gridtype,allow_overwrite=True) # MERGENOTE: # GLIDEIN_REQUIRE_VOMS publishes an attribute so that users without VOMS proxies # can avoid sites that require VOMS proxies (using the normal Condor Requirements # string. self.dicts[dtype].add("GLIDEIN_REQUIRE_VOMS",sub_params.config.restrictions.require_voms_proxy,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_REQUIRE_GLEXEC_USE",sub_params.config.restrictions.require_glidein_glexec_use,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_TrustDomain",sub_params.trust_domain,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_SupportedAuthenticationMethod",sub_params.auth_method,allow_overwrite=True) if sub_params.rsl is not None: self.dicts[dtype].add('GLIDEIN_GlobusRSL',sub_params.rsl,allow_overwrite=True) self.dicts[dtype].add("GLIDEIN_SlotsLayout", sub_params.config.submit.slots_layout, allow_overwrite=True) self.dicts['vars'].add_extended("GLIDEIN_REQUIRE_VOMS","boolean",sub_params.config.restrictions.require_voms_proxy,None,False,True,True) self.dicts['vars'].add_extended("GLIDEIN_REQUIRE_GLEXEC_USE","boolean",sub_params.config.restrictions.require_glidein_glexec_use,None,False,True,True) # populate infosys for infosys_ref in sub_params.infosys_refs: self.dicts['infosys'].add_extended(infosys_ref['type'],infosys_ref['server'],infosys_ref['ref'],allow_overwrite=True) # populate monitorgroups for monitorgroup in sub_params.monitorgroups: self.dicts['mongroup'].add_extended(monitorgroup['group_name'],allow_overwrite=True) # populate complex files populate_job_descript(self.work_dir,self.dicts['job_descript'], self.sub_name,sub_params) ################################################################################################################ # This is the original function call: # # self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE, # params.factory_name,params.glidein_name,self.sub_name, # sub_params.gridtype,sub_params.gatekeeper, sub_params.rsl, sub_params.auth_method, # params.web_url,sub_params.proxy_url,sub_params.work_dir, # params.submit.base_client_log_dir, sub_params.submit.submit_attrs) # # Almost all of the parameters are attributes of params and/or sub_params. Instead of maintaining an ever # increasing parameter list for this function, lets just pass params, sub_params, and the 2 other parameters # to the function and call it a day. ################################################################################################################ self.dicts['condor_jdl'].populate(cgWConsts.STARTUP_FILE, self.sub_name, params, sub_params)