def populate(self,params=None): if params is None: params=self.params # put default files in place first self.dicts['file_list'].add_placeholder('error_gen.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder('error_augment.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder('setup_script.sh',allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball #load system files for file_name in ('error_gen.sh','error_augment.sh','parse_starterlog.awk', 'advertise_failure.helper', "condor_config", "condor_config.multi_schedd.include", "condor_config.dedicated_starter.include", "condor_config.check.include", "condor_config.monitor.include"): self.dicts['file_list'].add_from_file(file_name,(cWConsts.insert_timestr(file_name),"regular","TRUE",'FALSE'),os.path.join(params.src_dir,file_name)) self.dicts['description'].add("condor_config","condor_config") self.dicts['description'].add("condor_config.multi_schedd.include","condor_config_multi_include") self.dicts['description'].add("condor_config.dedicated_starter.include","condor_config_main_include") self.dicts['description'].add("condor_config.monitor.include","condor_config_monitor_include") self.dicts['description'].add("condor_config.check.include","condor_config_check_include") self.dicts['vars'].load(params.src_dir,'condor_vars.lst',change_self=False,set_not_changed=False) # # Note: # We expect the condor platform info to be coming in as parameters # as FE provided consts file is not available at this time # # add the basic standard params self.dicts['params'].add("GLIDEIN_Report_Failed",'NEVER') self.dicts['params'].add("CONDOR_OS",'default') self.dicts['params'].add("CONDOR_ARCH",'default') self.dicts['params'].add("CONDOR_VERSION",'default') # Load initial system scripts # These should be executed before the other scripts for script_name in ('setup_script.sh','cat_consts.sh','condor_platform_select.sh'): self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) #load condor tarballs # only one will be downloaded in the end... based on what condor_platform_select.sh decides for condor_idx in range(len(params.condor_tarballs)): condor_el=params.condor_tarballs[condor_idx] # condor_el now is a combination of csv version+os+arch # Get list of valid tarballs for this condor_el # Register the tarball, but make download conditional to cond_name condor_el_valid_tarballs = get_valid_condor_tarballs([condor_el]) condor_fname = cWConsts.insert_timestr(cgWConsts.CONDOR_FILE % condor_idx) condor_tarfile = "" condor_fd = None if condor_el.tar_file is not None: # Condor tarball available. Just add it to the list of tarballs # with every possible condor_platform string condor_tarfile = condor_el.tar_file else: # Create a new tarball as usual condor_fd = cgWCreate.create_condor_tar_fd(condor_el.base_dir) condor_tarfile = os.path.join(self.dicts['file_list'].dir, condor_fname) # insert the newly created tarball fname back into the config params.subparams.data['condor_tarballs'][condor_idx]['tar_file'] = condor_tarfile for tar in condor_el_valid_tarballs: condor_platform = "%s-%s-%s" % (tar['version'], tar['os'], tar['arch']) cond_name = "CONDOR_PLATFORM_%s" % condor_platform condor_platform_fname = cgWConsts.CONDOR_FILE % condor_platform if condor_fd is None: # tar file exists. Just use it self.dicts['file_list'].add_from_file( condor_platform_fname, (condor_fname, "untar", cond_name, cgWConsts.CONDOR_ATTR), condor_el.tar_file) else: # This is addition of new tarfile # Need to rewind fd everytime condor_fd.seek(0) self.dicts['file_list'].add_from_fd( condor_platform_fname, (condor_fname,"untar",cond_name,cgWConsts.CONDOR_ATTR), condor_fd) self.dicts['untar_cfg'].add(condor_platform_fname, cgWConsts.CONDOR_DIR) # Add cond_name in the config, so that it is known # But leave it disabled by default self.dicts['consts'].add(cond_name, "0", allow_overwrite=False) if condor_fd is not None: condor_fd.close() # # Note: # We expect the collector info to be coming in as parameter # as FE consts file is not available at this time # # add the basic standard params self.dicts['params'].add("GLIDEIN_Collector",'Fake') file_list_scripts = ['collector_setup.sh', 'create_temp_mapfile.sh', 'setup_x509.sh', cgWConsts.CONDOR_STARTUP_FILE] after_file_list_scripts = ['check_proxy.sh', 'create_mapfile.sh', 'validate_node.sh', 'gcb_setup.sh', 'glexec_setup.sh', 'java_setup.sh', 'glidein_memory_setup.sh', 'glidein_cpus_setup.sh'] # Only execute scripts once duplicate_scripts = set(file_list_scripts).intersection(after_file_list_scripts) if duplicate_scripts: raise RuntimeError, "Duplicates found in the list of files to execute '%s'" % ','.join(duplicate_scripts) # Load more system scripts for script_name in file_list_scripts: self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # make sure condor_startup does not get executed ahead of time under normal circumstances # but must be loaded early, as it also works as a reporting script in case of error self.dicts['description'].add(cgWConsts.CONDOR_STARTUP_FILE,"last_script") # # At this point in the glideins, condor_advertize should be able to talk to the FE collector # # put user files in stage for file in params.files: add_file_unparsed(file,self.dicts) # put user attributes into config files for attr_name in params.attrs.keys(): add_attr_unparsed(attr_name, params,self.dicts,"main") # add additional system scripts for script_name in after_file_list_scripts: self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # populate complex files populate_factory_descript(self.work_dir,self.dicts['glidein'],self.active_sub_list,params) populate_frontend_descript(self.dicts['frontend_descript'],params) # populate the monitor files javascriptrrd_dir = params.monitor.javascriptRRD_dir for mfarr in ((params.src_dir,'factory_support.js'), (javascriptrrd_dir,'rrdFlot.js'), (javascriptrrd_dir,'rrdFlotMatrix.js'), (javascriptrrd_dir,'rrdFlotSupport.js'), (javascriptrrd_dir,'rrdFile.js'), (javascriptrrd_dir,'rrdMultiFile.js'), (javascriptrrd_dir,'rrdFilter.js'), (javascriptrrd_dir,'binaryXHR.js'), (params.monitor.flot_dir,'jquery.flot.js'), (params.monitor.flot_dir,'jquery.flot.selection.js'), (params.monitor.flot_dir,'jquery.flot.tooltip.js'), (params.monitor.flot_dir,'excanvas.js'), (params.monitor.jquery_dir,'jquery.js')): mfdir,mfname=mfarr parent_dir = self.find_parent_dir(mfdir,mfname) mfobj=cWDictFile.SimpleFile(parent_dir,mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ((params.src_dir,'factoryRRDBrowse.html'), (params.src_dir,'factoryRRDEntryMatrix.html'), (params.src_dir,'factoryStatus.html'), (params.src_dir,'factoryLogStatus.html'), (params.src_dir,'factoryCompletedStats.html'), (params.src_dir,'factoryStatusNow.html'), (params.src_dir,'factoryEntryStatusNow.html')): mfdir,mfname=mfarr mfobj=cWDictFile.SimpleFile(mfdir,mfname) mfobj.load() self.monitor_htmls.append(mfobj) # add the index page and its images mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/', 'index.html') mfobj.load() self.monitor_htmls.append(mfobj) for imgfile in ('factoryCompletedStats.png', 'factoryEntryStatusNow.png', 'factoryLogStatus.png', 'factoryRRDBrowse.png', 'factoryRRDEntryMatrix.png', 'factoryStatus.png', 'factoryStatusNow.png'): mfobj=cWDictFile.SimpleFile(params.src_dir + '/factory/images/', imgfile) mfobj.load() self.monitor_htmls.append(mfobj)
def populate(self,params=None): if params==None: params=self.params # put default files in place first self.dicts['file_list'].add_placeholder(cWConsts.CONSTS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.VARS_FILE,allow_overwrite=True) self.dicts['file_list'].add_placeholder(cWConsts.UNTAR_CFG_FILE,allow_overwrite=True) # this one must be loaded before any tarball # Load initial system scripts # These should be executed before the other scripts for script_name in ('cat_consts.sh','setup_x509.sh'): self.dicts['file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) #load system files for file_name in ('parse_starterlog.awk', "condor_config", "condor_config.multi_schedd.include", "condor_config.dedicated_starter.include", "condor_config.check.include", "condor_config.monitor.include"): self.dicts['file_list'].add_from_file(file_name,(cWConsts.insert_timestr(file_name),"regular","TRUE",'FALSE'),os.path.join(params.src_dir,file_name)) self.dicts['description'].add("condor_config","condor_config") self.dicts['description'].add("condor_config.multi_schedd.include","condor_config_multi_include") self.dicts['description'].add("condor_config.dedicated_starter.include","condor_config_main_include") self.dicts['description'].add("condor_config.monitor.include","condor_config_monitor_include") self.dicts['description'].add("condor_config.check.include","condor_config_check_include") self.dicts['vars'].load(params.src_dir,'condor_vars.lst',change_self=False,set_not_changed=False) # put user files in stage for file in params.files: add_file_unparsed(file,self.dicts) # put user attributes into config files for attr_name in params.attrs.keys(): add_attr_unparsed(attr_name, params,self.dicts,"main") # add the basic standard params self.dicts['params'].add("GLIDEIN_Collector",'Fake') # Prepare to load condor tarballs (after entry, as I may need data from there) for script_name in ('validate_node.sh','condor_platform_select.sh'): self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) #load condor tarballs for condor_idx in range(len(params.condor_tarballs)): condor_el=params.condor_tarballs[condor_idx] condor_platform="%s-%s-%s"%(condor_el.version,condor_el.os,condor_el.arch) cond_name="CONDOR_PLATFORM_%s"%condor_platform condor_fname=cgWConsts.CONDOR_FILE%condor_platform # register the tarball, but make the download conditional to cond_name if condor_el.tar_file!=None: # condor tarball available self.dicts['after_file_list'].add_from_file(condor_fname,(cWConsts.insert_timestr(condor_fname),"untar",cond_name,cgWConsts.CONDOR_ATTR),condor_el.tar_file) else: # create a new tarball condor_fd=cgWCreate.create_condor_tar_fd(condor_el.base_dir) condor_fname=cWConsts.insert_timestr(condor_fname) self.dicts['after_file_list'].add_from_fd(condor_fname,(condor_fname,"untar",cond_name,cgWConsts.CONDOR_ATTR),condor_fd) condor_fd.close() # insert the newly created tarball fname back into the config params.subparams.data['condor_tarballs'][condor_idx]['tar_file']=os.path.join(self.dicts['file_list'].dir,condor_fname) # add cond_name in the config, so that it is known it is there # but leave it disabled by default self.dicts['consts'].add(cond_name,"0",allow_overwrite=False) self.dicts['untar_cfg'].add(condor_fname,cgWConsts.CONDOR_DIR) # add additional system scripts for script_name in ('create_mapfile.sh','collector_setup.sh','gcb_setup.sh','glexec_setup.sh','java_setup.sh'): self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) # this must be the last script in the list for script_name in (cgWConsts.CONDOR_STARTUP_FILE,): self.dicts['after_file_list'].add_from_file(script_name,(cWConsts.insert_timestr(script_name),'exec','TRUE','FALSE'),os.path.join(params.src_dir,script_name)) self.dicts['description'].add(cgWConsts.CONDOR_STARTUP_FILE,"last_script") # populate complex files populate_factory_descript(self.work_dir,self.dicts['glidein'],self.active_sub_list,params) populate_frontend_descript(self.dicts['frontend_descript'],params) # populate the monitor files javascriptrrd_dir = params.monitor.javascriptRRD_dir for mfarr in ((params.src_dir,'factory_support.js'), (javascriptrrd_dir,'rrdFlot.js'), (javascriptrrd_dir,'rrdFlotMatrix.js'), (javascriptrrd_dir,'rrdFlotSupport.js'), (javascriptrrd_dir,'rrdFile.js'), (javascriptrrd_dir,'rrdMultiFile.js'), (javascriptrrd_dir,'rrdFilter.js'), (javascriptrrd_dir,'binaryXHR.js'), (params.monitor.flot_dir,'jquery.flot.js'), (params.monitor.flot_dir,'jquery.flot.selection.js'), (params.monitor.flot_dir,'excanvas.js'), (params.monitor.jquery_dir,'jquery.js')): mfdir,mfname=mfarr parent_dir = self.find_parent_dir(mfdir,mfname) mfobj=cWDictFile.SimpleFile(parent_dir,mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ((params.src_dir,'factoryRRDBrowse.html'), (params.src_dir,'factoryRRDEntryMatrix.html'), (params.src_dir,'factoryStatus.html'), (params.src_dir,'factoryLogStatus.html'), (params.src_dir,'factoryCompletedStats.html'), (params.src_dir,'factoryStatusNow.html'), (params.src_dir,'factoryEntryStatusNow.html')): mfdir,mfname=mfarr mfobj=cWDictFile.SimpleFile(mfdir,mfname) mfobj.load() self.monitor_htmls.append(mfobj)
def populate(self, other=None): # put default files in place first self.dicts["file_list"].add_placeholder("error_gen.sh", allow_overwrite=True) self.dicts["file_list"].add_placeholder("error_augment.sh", allow_overwrite=True) self.dicts["file_list"].add_placeholder("setup_script.sh", allow_overwrite=True) self.dicts["file_list"].add_placeholder(cWConsts.CONSTS_FILE, allow_overwrite=True) self.dicts["file_list"].add_placeholder(cWConsts.VARS_FILE, allow_overwrite=True) self.dicts["file_list"].add_placeholder( cWConsts.UNTAR_CFG_FILE, allow_overwrite=True ) # this one must be loaded before any tarball self.dicts["file_list"].add_placeholder( cWConsts.GRIDMAP_FILE, allow_overwrite=True ) # this one must be loaded before setup_x509.sh is run # load system files for file_name in ( "error_gen.sh", "error_augment.sh", "parse_starterlog.awk", "advertise_failure.helper", "condor_config", "condor_config.multi_schedd.include", "condor_config.dedicated_starter.include", "condor_config.check.include", "condor_config.monitor.include", ): self.dicts["file_list"].add_from_file( file_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(file_name), "regular"), os.path.join(cgWConsts.WEB_BASE_DIR, file_name), ) self.dicts["description"].add("condor_config", "condor_config") self.dicts["description"].add("condor_config.multi_schedd.include", "condor_config_multi_include") self.dicts["description"].add("condor_config.dedicated_starter.include", "condor_config_main_include") self.dicts["description"].add("condor_config.monitor.include", "condor_config_monitor_include") self.dicts["description"].add("condor_config.check.include", "condor_config_check_include") self.dicts["vars"].load(cgWConsts.WEB_BASE_DIR, "condor_vars.lst", change_self=False, set_not_changed=False) # # Note: # We expect the condor platform info to be coming in as parameters # as FE provided consts file is not available at this time # # add the basic standard params self.dicts["params"].add("GLIDEIN_Report_Failed", "NEVER") self.dicts["params"].add("CONDOR_OS", "default") self.dicts["params"].add("CONDOR_ARCH", "default") self.dicts["params"].add("CONDOR_VERSION", "default") # Load initial system scripts # These should be executed before the other scripts for script_name in ("setup_script.sh", "cat_consts.sh", "condor_platform_select.sh"): self.dicts["file_list"].add_from_file( script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, script_name), ) # load condor tarballs # only one will be downloaded in the end... based on what condor_platform_select.sh decides condor_tarballs = self.conf.get_child_list(u"condor_tarballs") prev_tar_dir_map = {} if other is not None and other.main_dicts.dicts["glidein"].has_key("CondorTarballDirMap"): prev_tar_dir_map = eval(other.main_dicts.dicts["glidein"]["CondorTarballDirMap"]) tar_dir_map = {} for condor_idx in range(len(condor_tarballs)): condor_el = condor_tarballs[condor_idx] # condor_el now is a combination of csv version+os+arch # Get list of valid tarballs for this condor_el # Register the tarball, but make download conditional to cond_name condor_el_valid_tarballs = get_valid_condor_tarballs([condor_el]) condor_fname = cWConsts.insert_timestr(cgWConsts.CONDOR_FILE % condor_idx) condor_fd = None if u"tar_file" in condor_el: # Condor tarball available. Just add it to the list of tarballs # with every possible condor_platform string condor_tarfile = condor_el[u"tar_file"] # already built this tarball, just reuse it elif condor_el[u"base_dir"] in prev_tar_dir_map: condor_tarfile = prev_tar_dir_map[condor_el[u"base_dir"]] tar_dir_map[condor_el[u"base_dir"]] = condor_tarfile else: # Create a new tarball as usual condor_fd = cgWCreate.create_condor_tar_fd(condor_el[u"base_dir"]) tar_dir_map[condor_el[u"base_dir"]] = os.path.join(self.dicts["file_list"].dir, condor_fname) for tar in condor_el_valid_tarballs: condor_platform = "%s-%s-%s" % (tar["version"], tar["os"], tar["arch"]) cond_name = "CONDOR_PLATFORM_%s" % condor_platform condor_platform_fname = cgWConsts.CONDOR_FILE % condor_platform if condor_fd is None: # tar file exists. Just use it self.dicts["file_list"].add_from_file( condor_platform_fname, cWDictFile.FileDictFile.make_val_tuple( condor_fname, "untar", cond_download=cond_name, config_out=cgWConsts.CONDOR_ATTR ), condor_tarfile, ) else: # This is addition of new tarfile # Need to rewind fd every time condor_fd.seek(0) self.dicts["file_list"].add_from_fd( condor_platform_fname, cWDictFile.FileDictFile.make_val_tuple( condor_fname, "untar", cond_download=cond_name, config_out=cgWConsts.CONDOR_ATTR ), condor_fd, ) self.dicts["untar_cfg"].add(condor_platform_fname, cgWConsts.CONDOR_DIR) # Add cond_name in the config, so that it is known # But leave it disabled by default self.dicts["consts"].add(cond_name, "0", allow_overwrite=False) if condor_fd is not None: condor_fd.close() self.dicts["glidein"].add("CondorTarballDirMap", str(tar_dir_map)) # # Note: # We expect the collector info to be coming in as parameter # as FE consts file is not available at this time # # add the basic standard params self.dicts["params"].add("GLIDEIN_Collector", "Fake") # add the factory monitoring collector parameter, if any collectors are defined # this is purely a factory thing factory_monitoring_collector = calc_monitoring_collectors_string( self.conf.get_child_list(u"monitoring_collectors") ) if factory_monitoring_collector is not None: self.dicts["params"].add("GLIDEIN_Factory_Collector", str(factory_monitoring_collector)) populate_gridmap(self.conf, self.dicts["gridmap"]) file_list_scripts = [ "collector_setup.sh", "create_temp_mapfile.sh", "setup_x509.sh", cgWConsts.CONDOR_STARTUP_FILE, ] after_file_list_scripts = [ "check_proxy.sh", "create_mapfile.sh", "validate_node.sh", "setup_network.sh", "gcb_setup.sh", "glexec_setup.sh", "java_setup.sh", "glidein_memory_setup.sh", "glidein_cpus_setup.sh", "glidein_sitewms_setup.sh", "script_wrapper.sh", "smart_partitionable.sh", ] # Only execute scripts once duplicate_scripts = set(file_list_scripts).intersection(after_file_list_scripts) if duplicate_scripts: raise RuntimeError, "Duplicates found in the list of files to execute '%s'" % ",".join(duplicate_scripts) # Load more system scripts for script_name in file_list_scripts: self.dicts["file_list"].add_from_file( script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, script_name), ) # Add the drainer script drain_script = "check_wn_drainstate.sh" self.dicts["file_list"].add_from_file( drain_script, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(drain_script), "exec", 60), os.path.join(cgWConsts.WEB_BASE_DIR, drain_script), ) # make sure condor_startup does not get executed ahead of time under normal circumstances # but must be loaded early, as it also works as a reporting script in case of error self.dicts["description"].add(cgWConsts.CONDOR_STARTUP_FILE, "last_script") # # At this point in the glideins, condor_advertize should be able to talk to the FE collector # # put user files in stage for file in self.conf.get_child_list(u"files"): add_file_unparsed(file, self.dicts, True) # put user attributes into config files for attr in self.conf.get_child_list(u"attrs"): add_attr_unparsed(attr, self.dicts, "main") # add additional system scripts for script_name in after_file_list_scripts: self.dicts["after_file_list"].add_from_file( script_name, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, script_name), ) # populate complex files populate_factory_descript( self.work_dir, self.dicts["glidein"], self.active_sub_list, self.disabled_sub_list, self.conf ) populate_frontend_descript(self.dicts["frontend_descript"], self.conf) # populate the monitor files javascriptrrd_dir = self.conf.get_child(u"monitor")[u"javascriptRRD_dir"] for mfarr in ((cgWConsts.WEB_BASE_DIR, "factory_support.js"), (javascriptrrd_dir, "javascriptrrd.wlibs.js")): mfdir, mfname = mfarr parent_dir = self.find_parent_dir(mfdir, mfname) mfobj = cWDictFile.SimpleFile(parent_dir, mfname) mfobj.load() self.monitor_jslibs.append(mfobj) for mfarr in ( (cgWConsts.WEB_BASE_DIR, "factoryRRDBrowse.html"), (cgWConsts.WEB_BASE_DIR, "factoryRRDEntryMatrix.html"), (cgWConsts.WEB_BASE_DIR, "factoryStatus.html"), (cgWConsts.WEB_BASE_DIR, "factoryLogStatus.html"), (cgWConsts.WEB_BASE_DIR, "factoryCompletedStats.html"), (cgWConsts.WEB_BASE_DIR, "factoryStatusNow.html"), (cgWConsts.WEB_BASE_DIR, "factoryEntryStatusNow.html"), ): mfdir, mfname = mfarr mfobj = cWDictFile.SimpleFile(mfdir, mfname) mfobj.load() self.monitor_htmls.append(mfobj) # add the index page and its images mfobj = cWDictFile.SimpleFile(cgWConsts.WEB_BASE_DIR + "/factory/", "index.html") mfobj.load() self.monitor_htmls.append(mfobj) for imgfile in ( "factoryCompletedStats.png", "factoryEntryStatusNow.png", "factoryLogStatus.png", "factoryRRDBrowse.png", "factoryRRDEntryMatrix.png", "factoryStatus.png", "factoryStatusNow.png", ): mfobj = cWDictFile.SimpleFile(cgWConsts.WEB_BASE_DIR + "/factory/images/", imgfile) mfobj.load() self.monitor_images.append(mfobj)