def initialize(self): """ Initialize the agent. """ self.am_setOption("PollingTime", 86400) #Once a day is enough self.swtc = SoftwareTagClient() self.submitjobs = self.am_getOption('SubmitJobs', False) if self.submitjobs: self.log.info( "Will submit probe jobs to validate the software tags") else: self.log.info("Will mark as Valid all 'New' tags directly.") self.delay = self.am_getOption("Delay", 86400) self.log.info( "Will reset to 'New' the tasks that have been 'Probing' for %s seconds" % self.delay) self.script = self.am_getOption("SoftwareManagementScript", "ProbeSoftwareArea.py") self.am_setOption('shifterProxy', 'SoftwareManager') #Needs to be able to submit job for that VO return S_OK()
def ProbeSoftwareArea(): """ Look into the shared area and report back to the SoftwareTag service """ from DIRAC import gLogger, gConfig #site = siteName() ce = gConfig.getValue('/LocalSite/GridCE', '') if not ce: return S_ERROR("CE undefined, cannot proceed") from GlastDIRAC.ResourceStatusSystem.Client.SoftwareTagClient import SoftwareTagClient swtc = SoftwareTagClient() if not 'VO_GLAST_ORG_SW_DIR' in os.environ: res = swtc.updateCEStatus("", ce, "Bad") if not res['OK']: return S_ERROR("Failed to report Bad site, missing software area.") return S_ERROR("Missing VO_GLAST_ORG_SW_DIR environment variable") base_sw_dir = os.environ['VO_GLAST_ORG_SW_DIR'] gLogger.notice("Found the following software directory:", base_sw_dir) message = None directory_list = [] for root, dirnames, files in os.walk( os.path.join(base_sw_dir, "glast/ground/releases")): if "bin" in dirnames: directory_list.append(root) for directory in directory_list: gLogger.notice("Decoding %s and tries to make a tag out of it" % directory) #Need mapping between Tag name and local software directory name res = getMappingTagFromDirectory(directory) if not res['OK']: gLogger.error("Failed finding relation between directory and Tag") continue tag = res['Value'] gLogger.notice("Found tag ", tag) res = swtc.updateCEStatus(tag, ce, 'Valid') if not res['OK']: gLogger.error("Failed to report back: %s" % res['Message']) message = res['Message'] else: gLogger.notice("Tag now Valid!") if message: return S_ERROR(message) return S_OK()
j.setName(opts.name) if not opts.group is "user": j.setJobGroup(str(opts.type)) j.setInputSandbox(input_sandbox_files) # all input files in the sandbox j.setOutputSandbox(output_sandbox_files) j.setCPUTime(opts.cpu) if not opts.site is None: j.setDestination(opts.site.split(",")) #can also be a list if not opts.bannedSites is None: j.setBannedSites(opts.bannedSites.split(",")) if not opts.release is None: tag = opts.release cl = SoftwareTagClient() result = cl.getSitesForTag(tag, 'Valid') # keyword doesn't work there. if not result['OK']: gLogger.error("*ERROR* Could not get sites for Tag %s" % tag, result['Message']) dexit(1) sites = result['Value'] j.setDestination(sites) # new feature: add xrootd-keytab file to input list. this one resides on SE input_stage_files = [] if pipeline: xrd_keytab = op.getValue("Pipeline/XrdKey", None) if not xrd_keytab: gLogger.notice("*DEBUG* adding XrdKey file %s to input" % xrd_keytab)
def InstallSoftware(tag, verbose=True, forceValidate=False): """ Look into the shared area and report back to the SoftwareTag service """ from DIRAC import gLogger, gConfig from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations op = Operations('glast.org') if not 'VO_GLAST_ORG_SW_DIR' in os.environ: return S_ERROR("Missing VO_GLAST_ORG_SW_DIR environment variable") base_sw_dir = os.environ['VO_GLAST_ORG_SW_DIR'] # get site info site = gConfig.getValue('/LocalSite/Site', '') if site == '': return S_ERROR("Fail to retrieve the site name") from GlastDIRAC.ResourceStatusSystem.Client.SoftwareTagClient import SoftwareTagClient swtc = SoftwareTagClient() status = "Valid" # check if tag is not already present res = swtc.getTagsAtSite(site, status) if not res['OK']: return S_ERROR( "Failed to retrieve tags registered for site %s, message: %s" % (site, res["Message"])) else: gLogger.notice("Found following tags on site %s:\n%s" % (site, str(res["Value"]))) if tag in res["Value"]: # we should break here if the tag is already there. gLogger.notice("Tag already found on site, doing nothing!") return S_OK("Tag found already - nothing to do") gLogger.notice("Found the following software directory:", base_sw_dir) rsync_server = op.getValue("Pipeline/RsyncServer", "ccglast02.in2p3.fr::VO_GLAST_ORG_SW_DIR") #rsync_server = "ccglast02.in2p3.fr::VO_GLAST_ORG_SW_DIR" rsync_cmd = "/usr/bin/rsync -azvv" # tag parsing res = getArrayFromTag(tag) if not res['OK']: return S_ERROR(res['Message']) infosoft = res['Value'] OS_GLEAM = infosoft['os'] VERSION_GLEAM = infosoft['version'] VARIANT_GLEAM = infosoft['variant'] SW_SHARED_DIR = base_sw_dir + "/glast/" # Directories to install ########################################################################################### dir_to_install = {} # name => ['src','destination'] dir_to_install['setup script'] = ['/setup.sh', "/"] dir_to_install['Moot files'] = ['/moot/', "/moot/"] dir_to_install['Missing librairies'] = [ '/lib/' + OS_GLEAM + "/", "/lib/" + OS_GLEAM + "/" ] dir_to_install['Calibrations files'] = [ '/ground/releases/calibrations/', "/ground/releases/calibrations/" ] dir_to_install['GLAST_EXT'] = [ "/ground/GLAST_EXT/" + OS_GLEAM + "/", "/ground/GLAST_EXT/" + OS_GLEAM + "/" ] dir_to_install['Gleam'] = [ "/ground/releases/" + OS_GLEAM + "/" + VARIANT_GLEAM + "/GlastRelease/" + VERSION_GLEAM + "/", "/ground/releases/" + OS_GLEAM + "/" + VARIANT_GLEAM + "/GlastRelease/" + VERSION_GLEAM + "/" ] dir_to_install['GPL librairie files'] = [ "/ground/PipelineConfig/", "/ground/PipelineConfig/" ] dir_to_install['Overlay data files'] = ["/overlay-data/", "/overlay-data/"] dir_to_install['Overlay XML files'] = ["/overlay/", "/overlay/"] dir_to_install['Transfer wilko files'] = [ "/transferwilko/", "/transferwilko/" ] if verbose: rsync_cmd = rsync_cmd + "v" # add the parameter "verbose" to the command line rsync_cmd = rsync_cmd + " " + rsync_server for name, array in dir_to_install.iteritems(): if not os.path.isdir(SW_SHARED_DIR + array[1]): os.makedirs(SW_SHARED_DIR + array[1]) gLogger.notice(" '" + name + "' retrieval with '" + rsync_cmd + array[0] + " " + SW_SHARED_DIR + array[1] + "'") gLogger.notice(" ... ") if os.system(rsync_cmd + array[0] + " " + SW_SHARED_DIR + array[1]) != 0: gLogger.notice(" -> FAILED !") return S_ERROR("Error 2 during the retrieval of '" + array[1] + "' from '" + rsync_server + "'") gLogger.notice(" -> OK !") # since we install a tag, we should register a new one: res = swtc.addTagAtSite(tag, site) # optional allow enforced validation! if forceValidate: res = swtc.updateStatus(tag, site, "Valid") if not res['OK']: return S_ERROR('Message: %s' % res['Message']) elif res['Value']['Failed']: return S_ERROR('Failed to update %s' % res['Value']['Failed']) else: return S_ERROR('Successfully updated %i CEs' % len(res['Value']['Successful'])) return return S_OK()