def initialize(self):
""" Initialize the agent.
"""
self.am_setOption("PollingTime", 86400) #Once a day is enough
self.swtc = SoftwareTagClient()
self.submitjobs = self.am_getOption('SubmitJobs', False)
if self.submitjobs:
self.log.info(
"Will submit probe jobs to validate the software tags")
else:
self.log.info("Will mark as Valid all 'New' tags directly.")
self.delay = self.am_getOption("Delay", 86400)
self.log.info(
"Will reset to 'New' the tasks that have been 'Probing' for %s seconds"
% self.delay)
self.script = self.am_getOption("SoftwareManagementScript",
"ProbeSoftwareArea.py")
self.am_setOption('shifterProxy', 'SoftwareManager')
#Needs to be able to submit job for that VO
return S_OK()
def ProbeSoftwareArea():
""" Look into the shared area and report back to the SoftwareTag service
"""
from DIRAC import gLogger, gConfig
#site = siteName()
ce = gConfig.getValue('/LocalSite/GridCE', '')
if not ce:
return S_ERROR("CE undefined, cannot proceed")
from GlastDIRAC.ResourceStatusSystem.Client.SoftwareTagClient import SoftwareTagClient
swtc = SoftwareTagClient()
if not 'VO_GLAST_ORG_SW_DIR' in os.environ:
res = swtc.updateCEStatus("", ce, "Bad")
if not res['OK']:
return S_ERROR("Failed to report Bad site, missing software area.")
return S_ERROR("Missing VO_GLAST_ORG_SW_DIR environment variable")
base_sw_dir = os.environ['VO_GLAST_ORG_SW_DIR']
gLogger.notice("Found the following software directory:", base_sw_dir)
message = None
directory_list = []
for root, dirnames, files in os.walk(
os.path.join(base_sw_dir, "glast/ground/releases")):
if "bin" in dirnames:
directory_list.append(root)
for directory in directory_list:
gLogger.notice("Decoding %s and tries to make a tag out of it" %
directory)
#Need mapping between Tag name and local software directory name
res = getMappingTagFromDirectory(directory)
if not res['OK']:
gLogger.error("Failed finding relation between directory and Tag")
continue
tag = res['Value']
gLogger.notice("Found tag ", tag)
res = swtc.updateCEStatus(tag, ce, 'Valid')
if not res['OK']:
gLogger.error("Failed to report back: %s" % res['Message'])
message = res['Message']
else:
gLogger.notice("Tag now Valid!")
if message:
return S_ERROR(message)
return S_OK()
j.setName(opts.name)
if not opts.group is "user":
j.setJobGroup(str(opts.type))
j.setInputSandbox(input_sandbox_files) # all input files in the sandbox
j.setOutputSandbox(output_sandbox_files)
j.setCPUTime(opts.cpu)
if not opts.site is None:
j.setDestination(opts.site.split(",")) #can also be a list
if not opts.bannedSites is None:
j.setBannedSites(opts.bannedSites.split(","))
if not opts.release is None:
tag = opts.release
cl = SoftwareTagClient()
result = cl.getSitesForTag(tag, 'Valid') # keyword doesn't work there.
if not result['OK']:
gLogger.error("*ERROR* Could not get sites for Tag %s" % tag,
result['Message'])
dexit(1)
sites = result['Value']
j.setDestination(sites)
# new feature: add xrootd-keytab file to input list. this one resides on SE
input_stage_files = []
if pipeline:
xrd_keytab = op.getValue("Pipeline/XrdKey", None)
if not xrd_keytab:
gLogger.notice("*DEBUG* adding XrdKey file %s to input" %
xrd_keytab)
def InstallSoftware(tag, verbose=True, forceValidate=False):
""" Look into the shared area and report back to the SoftwareTag service
"""
from DIRAC import gLogger, gConfig
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
op = Operations('glast.org')
if not 'VO_GLAST_ORG_SW_DIR' in os.environ:
return S_ERROR("Missing VO_GLAST_ORG_SW_DIR environment variable")
base_sw_dir = os.environ['VO_GLAST_ORG_SW_DIR']
# get site info
site = gConfig.getValue('/LocalSite/Site', '')
if site == '':
return S_ERROR("Fail to retrieve the site name")
from GlastDIRAC.ResourceStatusSystem.Client.SoftwareTagClient import SoftwareTagClient
swtc = SoftwareTagClient()
status = "Valid"
# check if tag is not already present
res = swtc.getTagsAtSite(site, status)
if not res['OK']:
return S_ERROR(
"Failed to retrieve tags registered for site %s, message: %s" %
(site, res["Message"]))
else:
gLogger.notice("Found following tags on site %s:\n%s" %
(site, str(res["Value"])))
if tag in res["Value"]:
# we should break here if the tag is already there.
gLogger.notice("Tag already found on site, doing nothing!")
return S_OK("Tag found already - nothing to do")
gLogger.notice("Found the following software directory:", base_sw_dir)
rsync_server = op.getValue("Pipeline/RsyncServer",
"ccglast02.in2p3.fr::VO_GLAST_ORG_SW_DIR")
#rsync_server = "ccglast02.in2p3.fr::VO_GLAST_ORG_SW_DIR"
rsync_cmd = "/usr/bin/rsync -azvv"
# tag parsing
res = getArrayFromTag(tag)
if not res['OK']:
return S_ERROR(res['Message'])
infosoft = res['Value']
OS_GLEAM = infosoft['os']
VERSION_GLEAM = infosoft['version']
VARIANT_GLEAM = infosoft['variant']
SW_SHARED_DIR = base_sw_dir + "/glast/"
# Directories to install ###########################################################################################
dir_to_install = {} # name => ['src','destination']
dir_to_install['setup script'] = ['/setup.sh', "/"]
dir_to_install['Moot files'] = ['/moot/', "/moot/"]
dir_to_install['Missing librairies'] = [
'/lib/' + OS_GLEAM + "/", "/lib/" + OS_GLEAM + "/"
]
dir_to_install['Calibrations files'] = [
'/ground/releases/calibrations/', "/ground/releases/calibrations/"
]
dir_to_install['GLAST_EXT'] = [
"/ground/GLAST_EXT/" + OS_GLEAM + "/",
"/ground/GLAST_EXT/" + OS_GLEAM + "/"
]
dir_to_install['Gleam'] = [
"/ground/releases/" + OS_GLEAM + "/" + VARIANT_GLEAM +
"/GlastRelease/" + VERSION_GLEAM + "/", "/ground/releases/" +
OS_GLEAM + "/" + VARIANT_GLEAM + "/GlastRelease/" + VERSION_GLEAM + "/"
]
dir_to_install['GPL librairie files'] = [
"/ground/PipelineConfig/", "/ground/PipelineConfig/"
]
dir_to_install['Overlay data files'] = ["/overlay-data/", "/overlay-data/"]
dir_to_install['Overlay XML files'] = ["/overlay/", "/overlay/"]
dir_to_install['Transfer wilko files'] = [
"/transferwilko/", "/transferwilko/"
]
if verbose:
rsync_cmd = rsync_cmd + "v" # add the parameter "verbose" to the command line
rsync_cmd = rsync_cmd + " " + rsync_server
for name, array in dir_to_install.iteritems():
if not os.path.isdir(SW_SHARED_DIR + array[1]):
os.makedirs(SW_SHARED_DIR + array[1])
gLogger.notice(" '" + name + "' retrieval with '" + rsync_cmd +
array[0] + " " + SW_SHARED_DIR + array[1] + "'")
gLogger.notice(" ... ")
if os.system(rsync_cmd + array[0] + " " + SW_SHARED_DIR +
array[1]) != 0:
gLogger.notice(" -> FAILED !")
return S_ERROR("Error 2 during the retrieval of '" + array[1] +
"' from '" + rsync_server + "'")
gLogger.notice(" -> OK !")
# since we install a tag, we should register a new one:
res = swtc.addTagAtSite(tag, site)
# optional allow enforced validation!
if forceValidate:
res = swtc.updateStatus(tag, site, "Valid")
if not res['OK']:
return S_ERROR('Message: %s' % res['Message'])
elif res['Value']['Failed']:
return S_ERROR('Failed to update %s' % res['Value']['Failed'])
else:
return S_ERROR('Successfully updated %i CEs' %
len(res['Value']['Successful']))
return
return S_OK()
