def __init__(self, bkClient=None, dm=None):
"""Module initialization.
"""
self.log = gLogger.getSubLogger("UserJobFinalization")
super(UserJobFinalization, self).__init__(self.log,
bkClientIn=bkClient,
dm=dm)
self.version = __RCSID__
self.enable = True
self.defaultOutputSE = resolveSEGroup(
gConfig.getValue('/Resources/StorageElementGroups/Tier1-USER', []))
self.failoverSEs = resolveSEGroup(
gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover',
[]))
# List all parameters here
self.request = None
# Always allow any files specified by users
self.outputDataFileMask = ''
self.userOutputData = []
self.userOutputSE = ''
self.userOutputPath = ''
self.failoverTransfer = None
self.replicateUserOutputData = False
self.userPrependString = ''
def storageType(seList):
""" Return the storage type: disk, tape or archive """
if not (set(se for se in seList if not dmsHelper.isSEArchive(se)) -
set(['CERN-SWTEST', 'CERN-FREEZER-EOS', 'CERN-FREEZER'])) and \
set(se for se in seList if dmsHelper.isSEArchive(se)):
# Only -ARCHIVE
return 'Archived'
tapeSEs = resolveSEGroup('Tier1-RAW') + resolveSEGroup('Tier1-RDST')
if [se for se in seList if se in tapeSEs]:
# Any file on Tape
return 'Tape'
# Others
return 'Disk'
def getOption(self, switch, default=None):
"""
Get a specific items set by the setters
"""
if switch == 'SEs':
# SEs have to be resolved recursively using StorageElementGroups
return resolveSEGroup(self.options.get(switch, default))
value = self.options.get(switch, default)
if switch in ('LFNs', 'Directory'):
# Special case for getting LFNs or directories: parse the option
if value == default and switch == 'Directory':
value = self.options.get('LFNs', default)
if not value:
if not sys.stdin.isatty():
# If the input file is a pipe, no need to specify it
self.setLFNsFromTerm()
value = self.options.get('LFNs', default)
if value:
# Parse the LFNs out of the "LFNs" option list
value = self.getLFNsFromList(value,
directories=switch == 'Directory')
if value and self.setLastFile and switch == 'LFNs':
# Storethe list of LFNs in a temporary file
gLogger.always("Got %d LFNs" % len(value))
if self.setLastFile != self.lastFile:
self.setLastFile = False
with open(self.lastFile, 'w') as tmpFile:
tmpFile.write('\n'.join(sorted(value)))
if isinstance(value, set):
# Return a sorted list from a set
value = sorted(value)
return value
def _Transformation__checkSEs(self, seList):
# This test allows to set some parameters empty
if seList == []:
return S_OK()
if resolveSEGroup(seList):
return S_OK()
gLogger.error("Some SEs are unknown in %s" % ','.join(seList))
return S_ERROR("Some StorageElements not known")
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch('', 'Path=', ' Path to search for')
Script.registerSwitch(
'', 'SE=',
' (comma-separated list of) SEs/SE-groups to be searched')
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
path = '/'
seList = None
for opt, val in Script.getUnprocessedSwitches():
if opt == 'Path':
path = val
elif opt == 'SE':
seList = resolveSEGroup(val.split(','))
if seList:
args.append("SE=%s" % ','.join(seList))
fc = FileCatalog()
result = fc.getMetadataFields()
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
typeDict = result['Value']['FileMetaFields']
typeDict.update(result['Value']['DirectoryMetaFields'])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
if len(args) < 1:
print("Error: No argument provided\n%s:" % Script.scriptName)
gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict))
Script.showHelp(exitCode=1)
mq = MetaQuery(typeDict=typeDict)
result = mq.setMetaQuery(args)
if not result['OK']:
gLogger.error("Illegal metaQuery:", result['Message'])
DIRAC.exit(-1)
metaDict = result['Value']
path = metaDict.pop('Path', path)
result = fc.findFilesByMetadata(metaDict, path)
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
lfnList = sorted(result['Value'])
gLogger.notice('\n'.join(lfn for lfn in lfnList))
pass
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
site = switch[1]
# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
DIRAC.exit(2)
res = getProxyInfo()
if not res['OK']:
gLogger.error('Failed to get proxy information', res['Message'])
DIRAC.exit(2)
userName = res['Value'].get('username')
if not userName:
gLogger.error('Failed to get username for proxy')
def main():
read = True
write = True
check = True
remove = True
site = ''
mute = False
Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element")
Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element")
Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element")
Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element")
Script.registerSwitch("a", "All", " Ban all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch(
"S:",
"Site=",
" Ban all SEs associate to site (note that if writing is allowed, check is always allowed)")
Script.parseCommandLine(ignoreErrors=True)
ses = Script.getPositionalArgs()
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() in ("r", "banread"):
write = False
check = False
remove = False
if switch[0].lower() in ("w", "banwrite"):
read = False
check = False
remove = False
if switch[0].lower() in ("k", "bancheck"):
read = False
write = False
remove = False
if switch[0].lower() in ("v", "banremove"):
read = False
write = False
check = False
if switch[0].lower() in ("a", "all"):
pass
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
site = switch[1]
# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
DIRAC.exit(2)
res = getProxyInfo()
if not res['OK']:
gLogger.error('Failed to get proxy information', res['Message'])
DIRAC.exit(2)
userName = res['Value'].get('username')
if not userName:
gLogger.error('Failed to get username for proxy')
DIRAC.exit(2)
if site:
res = getSites()
if not res['OK']:
gLogger.error(res['Message'])
DIRAC.exit(-1)
if site not in res['Value']:
gLogger.error('The provided site (%s) is not known.' % site)
DIRAC.exit(-1)
ses.extend(res['Value']['SE'].replace(' ', '').split(','))
if not ses:
gLogger.error('There were no SEs provided')
DIRAC.exit(-1)
readBanned = []
writeBanned = []
checkBanned = []
removeBanned = []
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res['OK']:
gLogger.error("Storage Element %s does not exist" % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-ban-se by %s' % userName
for se, seOptions in res['Value'].items():
resW = resC = resR = {'OK': False}
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if read and 'ReadAccess' in seOptions:
if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Read option for %s is %s, instead of %s' %
(se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" )
if not resR['OK']:
gLogger.error('Failed to update %s read access to Banned' % se)
else:
gLogger.notice('Successfully updated %s read access to Banned' % se)
readBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if write and 'WriteAccess' in seOptions:
if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Write option for %s is %s, instead of %s' %
(se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" )
if not resW['OK']:
gLogger.error("Failed to update %s write access to Banned" % se)
else:
gLogger.notice("Successfully updated %s write access to Banned" % se)
writeBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if check and 'CheckAccess' in seOptions:
if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Check option for %s is %s, instead of %s' %
(se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC['OK']:
gLogger.error("Failed to update %s check access to Banned" % se)
else:
gLogger.notice("Successfully updated %s check access to Banned" % se)
checkBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if remove and 'RemoveAccess' in seOptions:
if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Remove option for %s is %s, instead of %s' %
(se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC['OK']:
gLogger.error("Failed to update %s remove access to Banned" % se)
else:
gLogger.notice("Successfully updated %s remove access to Banned" % se)
removeBanned.append(se)
if not(resR['OK'] or resW['OK'] or resC['OK']):
DIRAC.exit(-1)
if not (writeBanned or readBanned or checkBanned or removeBanned):
gLogger.notice("No storage elements were banned")
DIRAC.exit(-1)
if mute:
gLogger.notice('Email is muted by script switch')
DIRAC.exit(0)
subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned)
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
body = ''
if read:
body = "%s\n\nThe following storage elements were banned for reading:" % body
for se in readBanned:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were banned for writing:" % body
for se in writeBanned:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were banned for check access:" % body
for se in checkBanned:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were banned for remove access:" % body
for se in removeBanned:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice('Notifying %s' % address)
if res['OK']:
gLogger.notice(res['Value'])
else:
gLogger.notice(res['Message'])
DIRAC.exit(0)
write = False
check = False
if switch[0].lower() == "m" or switch[0].lower() == "mute":
mute = True
if switch[0] == "S" or switch[0].lower() == "site":
site = switch[1]
# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC import gConfig, gLogger
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
ses = resolveSEGroup( ses )
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
setup = gConfig.getValue( '/DIRAC/Setup', '' )
if not setup:
print 'ERROR: Could not contact Configuration Service'
exitCode = 2
DIRAC.exit( exitCode )
res = getProxyInfo()
if not res[ 'OK' ]:
gLogger.error( 'Failed to get proxy information', res[ 'Message' ] )
DIRAC.exit( 2 )
userName = res['Value'].get( 'username' )
def main():
Script.registerSwitch("", "Path=", " Path to search for")
Script.registerSwitch(
"", "SE=",
" (comma-separated list of) SEs/SE-groups to be searched")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
[
"metaspec: metadata index specification (of the form: "
'"meta=value" or "meta<value", "meta!=value", etc.)'
],
mandatory=False,
)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
path = "/"
seList = None
for opt, val in Script.getUnprocessedSwitches():
if opt == "Path":
path = val
elif opt == "SE":
seList = resolveSEGroup(val.split(","))
if seList:
args.append("SE=%s" % ",".join(seList))
fc = FileCatalog()
result = fc.getMetadataFields()
if not result["OK"]:
gLogger.error("Can not access File Catalog:", result["Message"])
DIRAC.exit(-1)
typeDict = result["Value"]["FileMetaFields"]
typeDict.update(result["Value"]["DirectoryMetaFields"])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
if len(args) < 1:
print("Error: No argument provided\n%s:" % Script.scriptName)
gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict))
Script.showHelp(exitCode=1)
mq = MetaQuery(typeDict=typeDict)
result = mq.setMetaQuery(args)
if not result["OK"]:
gLogger.error("Illegal metaQuery:", result["Message"])
DIRAC.exit(-1)
metaDict = result["Value"]
path = metaDict.pop("Path", path)
result = fc.findFilesByMetadata(metaDict, path)
if not result["OK"]:
gLogger.error("Can not access File Catalog:", result["Message"])
DIRAC.exit(-1)
lfnList = sorted(result["Value"])
gLogger.notice("\n".join(lfn for lfn in lfnList))
def main():
read = True
write = True
check = True
remove = True
sites = []
mute = False
Script.registerSwitch("r", "BanRead",
" Ban only reading from the storage element")
Script.registerSwitch("w", "BanWrite",
" Ban writing to the storage element")
Script.registerSwitch("k", "BanCheck",
" Ban check access to the storage element")
Script.registerSwitch("v", "BanRemove",
" Ban remove access to the storage element")
Script.registerSwitch("a", "All",
" Ban all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch(
"S:", "Site=",
" Ban all SEs associate to site (note that if writing is allowed, check is always allowed)"
)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
["seGroupList: list of SEs or comma-separated SEs"])
switches, ses = Script.parseCommandLine(ignoreErrors=True)
for switch in switches:
if switch[0].lower() in ("r", "banread"):
write = False
check = False
remove = False
if switch[0].lower() in ("w", "banwrite"):
read = False
check = False
remove = False
if switch[0].lower() in ("k", "bancheck"):
read = False
write = False
remove = False
if switch[0].lower() in ("v", "banremove"):
read = False
write = False
check = False
if switch[0].lower() in ("a", "all"):
pass
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
sites = switch[1].split(",")
# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup, DMSHelpers
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
print("ERROR: Could not contact Configuration Service")
DIRAC.exit(2)
res = getProxyInfo()
if not res["OK"]:
gLogger.error("Failed to get proxy information", res["Message"])
DIRAC.exit(2)
userName = res["Value"].get("username")
if not userName:
gLogger.error("Failed to get username for proxy")
DIRAC.exit(2)
for site in sites:
res = DMSHelpers().getSEsForSite(site)
if not res["OK"]:
gLogger.error(res["Message"], site)
DIRAC.exit(-1)
ses.extend(res["Value"])
if not ses:
gLogger.error("There were no SEs provided")
DIRAC.exit(-1)
readBanned = []
writeBanned = []
checkBanned = []
removeBanned = []
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res["OK"]:
gLogger.error("Storage Element %s does not exist" % ses)
DIRAC.exit(-1)
reason = "Forced with dirac-admin-ban-se by %s" % userName
for se, seOptions in res["Value"].items():
resW = resC = resR = {"OK": False}
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if read and "ReadAccess" in seOptions:
if seOptions["ReadAccess"] == "Banned":
gLogger.notice("Read access already banned", se)
resR["OK"] = True
elif not seOptions["ReadAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Read option for %s is %s, instead of %s" %
(se, seOptions["ReadAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resR = resourceStatus.setElementStatus(se, "StorageElement",
"ReadAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" )
if not resR["OK"]:
gLogger.error("Failed to update %s read access to Banned" %
se)
else:
gLogger.notice(
"Successfully updated %s read access to Banned" % se)
readBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if write and "WriteAccess" in seOptions:
if seOptions["WriteAccess"] == "Banned":
gLogger.notice("Write access already banned", se)
resW["OK"] = True
elif not seOptions["WriteAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Write option for %s is %s, instead of %s" %
(se, seOptions["WriteAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resW = resourceStatus.setElementStatus(se, "StorageElement",
"WriteAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" )
if not resW["OK"]:
gLogger.error(
"Failed to update %s write access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s write access to Banned" % se)
writeBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if check and "CheckAccess" in seOptions:
if seOptions["CheckAccess"] == "Banned":
gLogger.notice("Check access already banned", se)
resC["OK"] = True
elif not seOptions["CheckAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Check option for %s is %s, instead of %s" %
(se, seOptions["CheckAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resC = resourceStatus.setElementStatus(se, "StorageElement",
"CheckAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC["OK"]:
gLogger.error(
"Failed to update %s check access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s check access to Banned" % se)
checkBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if remove and "RemoveAccess" in seOptions:
if seOptions["RemoveAccess"] == "Banned":
gLogger.notice("Remove access already banned", se)
resC["OK"] = True
elif not seOptions["RemoveAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Remove option for %s is %s, instead of %s" %
(se, seOptions["RemoveAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resC = resourceStatus.setElementStatus(se, "StorageElement",
"RemoveAccess",
"Banned", reason,
userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC["OK"]:
gLogger.error(
"Failed to update %s remove access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s remove access to Banned" % se)
removeBanned.append(se)
if not (resR["OK"] or resW["OK"] or resC["OK"]):
DIRAC.exit(-1)
if not (writeBanned or readBanned or checkBanned or removeBanned):
gLogger.notice("No storage elements were banned")
DIRAC.exit(-1)
if mute:
gLogger.notice("Email is muted by script switch")
DIRAC.exit(0)
subject = "%s storage elements banned for use" % len(
writeBanned + readBanned + checkBanned + removeBanned)
addressPath = "EMail/Production"
address = Operations().getValue(addressPath, "")
body = ""
if read:
body = "%s\n\nThe following storage elements were banned for reading:" % body
for se in readBanned:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were banned for writing:" % body
for se in writeBanned:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were banned for check access:" % body
for se in checkBanned:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were banned for remove access:" % body
for se in removeBanned:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice("Notifying %s" % address)
if res["OK"]:
gLogger.notice(res["Value"])
else:
gLogger.notice(res["Message"])
DIRAC.exit(0)
def main():
read = False
write = False
check = False
remove = False
site = ''
mute = False
Script.registerSwitch("r", "AllowRead",
" Allow only reading from the storage element")
Script.registerSwitch("w", "AllowWrite",
" Allow only writing to the storage element")
Script.registerSwitch(
"k", "AllowCheck",
" Allow only check access to the storage element")
Script.registerSwitch(
"v", "AllowRemove",
" Allow only remove access to the storage element")
Script.registerSwitch("a", "All",
" Allow all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch("S:", "Site=",
" Allow all SEs associated to site")
Script.parseCommandLine(ignoreErrors=True)
ses = Script.getPositionalArgs()
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() in ("r", "allowread"):
read = True
if switch[0].lower() in ("w", "allowwrite"):
write = True
if switch[0].lower() in ("k", "allowcheck"):
check = True
if switch[0].lower() in ("v", "allowremove"):
remove = True
if switch[0].lower() in ("a", "all"):
read = True
write = True
check = True
remove = True
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
site = switch[1]
# imports
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
if not (read or write or check or remove):
# No switch was specified, means we need all of them
gLogger.notice(
"No option given, all accesses will be allowed if they were not")
read = True
write = True
check = True
remove = True
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
DIRAC.exit(2)
res = getProxyInfo()
if not res['OK']:
gLogger.error('Failed to get proxy information', res['Message'])
DIRAC.exit(2)
userName = res['Value'].get('username')
if not userName:
gLogger.error('Failed to get username for proxy')
DIRAC.exit(2)
if site:
res = getSites()
if not res['OK']:
gLogger.error(res['Message'])
DIRAC.exit(-1)
if site not in res['Value']:
gLogger.error('The provided site (%s) is not known.' % site)
DIRAC.exit(-1)
ses.extend(res['Value']['SE'].replace(' ', '').split(','))
if not ses:
gLogger.error('There were no SEs provided')
DIRAC.exit()
STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"]
ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"]
statusAllowedDict = {}
for statusType in STATUS_TYPES:
statusAllowedDict[statusType] = []
statusFlagDict = {}
statusFlagDict['ReadAccess'] = read
statusFlagDict['WriteAccess'] = write
statusFlagDict['CheckAccess'] = check
statusFlagDict['RemoveAccess'] = remove
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res['OK']:
gLogger.error('Storage Element %s does not exist' % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-allow-se by %s' % userName
for se, seOptions in res['Value'].items():
# InActive is used on the CS model, Banned is the equivalent in RSS
for statusType in STATUS_TYPES:
if statusFlagDict[statusType]:
if seOptions.get(statusType) == "Active":
gLogger.notice('%s status of %s is already Active' %
(statusType, se))
continue
if statusType in seOptions:
if not seOptions[statusType] in ALLOWED_STATUSES:
gLogger.notice(
'%s option for %s is %s, instead of %s' %
(statusType, se, seOptions['ReadAccess'],
ALLOWED_STATUSES))
gLogger.notice('Try specifying the command switches')
else:
resR = resourceStatus.setElementStatus(
se, "StorageElement", statusType, 'Active', reason,
userName)
if not resR['OK']:
gLogger.fatal(
"Failed to update %s %s to Active, exit -" %
(se, statusType), resR['Message'])
DIRAC.exit(-1)
else:
gLogger.notice(
"Successfully updated %s %s to Active" %
(se, statusType))
statusAllowedDict[statusType].append(se)
totalAllowed = 0
totalAllowedSEs = []
for statusType in STATUS_TYPES:
totalAllowed += len(statusAllowedDict[statusType])
totalAllowedSEs += statusAllowedDict[statusType]
totalAllowedSEs = list(set(totalAllowedSEs))
if not totalAllowed:
gLogger.info("No storage elements were allowed")
DIRAC.exit(-1)
if mute:
gLogger.notice('Email is muted by script switch')
DIRAC.exit(0)
subject = '%s storage elements allowed for use' % len(totalAllowedSEs)
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
body = ''
if read:
body = "%s\n\nThe following storage elements were allowed for reading:" % body
for se in statusAllowedDict['ReadAccess']:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were allowed for writing:" % body
for se in statusAllowedDict['WriteAccess']:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were allowed for checking:" % body
for se in statusAllowedDict['CheckAccess']:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were allowed for removing:" % body
for se in statusAllowedDict['RemoveAccess']:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice('Notifying %s' % address)
if res['OK']:
gLogger.notice(res['Value'])
else:
gLogger.notice(res['Message'])
DIRAC.exit(0)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC import gLogger
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
path = '/'
seList = None
for opt, val in Script.getUnprocessedSwitches():
if opt == 'Path':
path = val
elif opt == 'SE':
seList = resolveSEGroup(val.split(','))
if seList:
args.append("SE=%s" % ','.join(seList))
fc = FileCatalog()
result = fc.getMetadataFields()
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
typeDict = result['Value']['FileMetaFields']
typeDict.update(result['Value']['DirectoryMetaFields'])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
if len(args) < 1:
print "Error: No argument provided\n%s:" % Script.scriptName
def getDestinationSEList(outputSE, site, outputmode='Any', run=None):
""" Evaluate the output SE list from a workflow and return the concrete list
of SEs to upload output data.
"""
if outputmode.lower() not in ('any', 'local', 'run'):
raise RuntimeError("Unexpected outputmode")
if outputmode.lower() == 'run':
gLogger.verbose(
"Output mode set to 'run', thus ignoring site parameter")
if not run:
raise RuntimeError("Expected runNumber")
try:
run = long(run)
except ValueError as ve:
raise RuntimeError("Expected runNumber as a number: %s" % ve)
gLogger.debug("RunNumber = %d" % run)
from LHCbDIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
runDestination = TransformationClient().getDestinationForRun(run)
if not runDestination['OK'] or run not in runDestination['Value']:
raise RuntimeError("Issue getting destinationForRun (%d): " % run +
runDestination.get('Message', 'unknown run'))
site = runDestination['Value'][run]
gLogger.verbose("Site set to %s for run %d" % (site, run))
outputmode = 'Local'
# Add output SE defined in the job description
gLogger.info('Resolving workflow output SE description: %s' % outputSE)
# Check if the SE is defined explicitly for the site
prefix = site.split('.')[0]
country = site.split('.')[-1]
# Concrete SE name
result = gConfig.getOptions('/Resources/StorageElements/' + outputSE)
if result['OK']:
gLogger.info('Found concrete SE %s' % outputSE)
return [outputSE]
# Get local SEs
localSEs = getSEsForSite(site)
if not localSEs['OK']:
raise RuntimeError(localSEs['Message'])
localSEs = localSEs['Value']
gLogger.verbose("Local SE list is: %s" % (localSEs))
# There is an alias defined for this Site
associatedSEs = gConfig.getValue(
'/Resources/Sites/%s/%s/AssociatedSEs/%s' % (prefix, site, outputSE),
[])
if associatedSEs:
associatedSEs = _setLocalFirst(associatedSEs, localSEs)
gLogger.info("Found associated SE %s for site %s" %
(associatedSEs, site))
return associatedSEs
groupSEs = resolveSEGroup(outputSE)
if not groupSEs:
raise RuntimeError("Failed to resolve SE " + outputSE)
gLogger.verbose("Group SE list is: %s" % (groupSEs))
# Find a local SE or an SE considered as local because the country is associated to it
if outputmode.lower() == "local":
# First, check if one SE in the group is local
for se in localSEs:
if se in groupSEs:
gLogger.info("Found eligible local SE: %s" % (se))
return [se]
# Final check for country associated SE
assignedCountry = country
while True:
# check if country is already one with associated SEs
section = '/Resources/Countries/%s/AssociatedSEs/%s' % (
assignedCountry, outputSE)
associatedSEs = gConfig.getValue(section, [])
if associatedSEs:
associatedSEs = _setLocalFirst(associatedSEs, localSEs)
gLogger.info('Found associated SEs %s in %s' %
(associatedSEs, section))
return associatedSEs
gLogger.verbose("/Resources/Countries/%s/AssignedTo" %
assignedCountry)
opt = gConfig.getOption("/Resources/Countries/%s/AssignedTo" %
assignedCountry)
if opt['OK'] and opt['Value']:
assignedCountry = opt['Value']
else:
# No associated SE and no assigned country, give up
raise RuntimeError(
"Could not establish associated SE nor assigned country for country %s"
% assignedCountry)
# For collective Any and All modes return the whole group
# Make sure that local SEs are passing first
orderedSEs = _setLocalFirst(groupSEs, localSEs)
gLogger.info('Found SEs, local first: %s' % orderedSEs)
return orderedSEs
def executeForVO(self, vo):
"""
Execute one SE and user synchronisation cycle for a VO.
:param str vo: Virtual organisation name.
:return: S_OK or S_ERROR
:rtype: dict
"""
valid_protocols = ["srm", "gsiftp", "davs", "https", "root"]
default_email = None
try:
try:
client = Client(account="root", auth_type="userpass")
except Exception as err:
self.log.info(
"Login to Rucio as root with password failed. Will try host cert/key",
str(err))
certKeyTuple = Locations.getHostCertificateAndKeyLocation()
if not certKeyTuple:
self.log.error("Hostcert/key location not set")
return S_ERROR("Hostcert/key location not set")
hostcert, hostkey = certKeyTuple
self.log.info("Logging in with a host cert/key pair:")
self.log.debug("account: ",
self.clientConfig[vo]["privilegedAccount"])
self.log.debug("rucio host: ",
self.clientConfig[vo]["rucioHost"])
self.log.debug("auth host: ",
self.clientConfig[vo]["authHost"])
self.log.debug("CA cert path: ", self.caCertPath)
self.log.debug("Cert location: ", hostcert)
self.log.debug("Key location: ", hostkey)
self.log.debug("VO: ", vo)
client = Client(
account=self.clientConfig[vo]["privilegedAccount"],
rucio_host=self.clientConfig[vo]["rucioHost"],
auth_host=self.clientConfig[vo]["authHost"],
ca_cert=self.caCertPath,
auth_type="x509",
creds={
"client_cert": hostcert,
"client_key": hostkey
},
timeout=600,
user_agent="rucio-clients",
vo=vo,
)
self.log.info("Rucio client instantiated for VO:", vo)
# Get the storage elements from Dirac Configuration and create them in Rucio
newRSE = False
self.log.info("Synchronizing SEs for VO ", vo)
result = getStorageElements(vo)
if result["OK"]:
rses = [rse["rse"] for rse in client.list_rses()]
for se in result["Value"]:
if se not in rses:
# The SE doesn't exist. Will create it
newRSE = True
self.log.info(
"Rucio Storage Element does not exist and will be created:",
se)
try:
client.add_rse(rse=se,
deterministic=True,
volatile=False)
except Exception as err:
self.log.error(
"Cannot create RSE",
"[RSE: %s, Error: %s]" % (se, str(err)))
continue
# Add RSE attributes for the new RSE
ret = gConfig.getOptionsDict(
"Resources/FTSEndpoints/FTS3")
ftsList = ""
if ret["OK"]:
ftsList = ",".join(ret["Value"].values())
dictRSEAttributes = {
"naming_convention": "BelleII",
"ANY": True,
"fts": ftsList
}
for key in dictRSEAttributes:
self.log.info(
"Setting RSE attributes",
"[RSE: %s, Attr. name: %s, Value: %s]" %
(se, key, dictRSEAttributes[key]),
)
client.add_rse_attribute(
se, key, value=dictRSEAttributes[key])
client.set_local_account_limit("root", se,
100000000000000000)
# Create the protocols
try:
protocols = client.get_protocols(se)
except RSEProtocolNotSupported as err:
self.log.info("Cannot get protocols for",
"[RSE %s, Error: %s]" % (se, str(err)))
protocols = []
existing_protocols = []
for prot in protocols:
existing_protocols.append(
(str(prot["scheme"]), str(prot["hostname"]),
str(prot["port"]), str(prot["prefix"])))
protocols_to_create = []
for params in result["Value"][se]:
prot = (
str(params["scheme"]),
str(params["hostname"]),
str(params["port"]),
str(params["prefix"]),
)
protocols_to_create.append(prot)
if prot not in existing_protocols and prot[
0] in valid_protocols:
# The protocol defined in Dirac does not exist in Rucio. Will be created
self.log.info(
"Will create new protocol:",
"%s://%s:%s%s on %s" %
(params["scheme"], params["hostname"],
params["port"], params["prefix"], se),
)
try:
client.add_protocol(rse=se, params=params)
except Duplicate as err:
self.log.info(
"Protocol already exists on",
"[RSE: %s, schema:%s]" %
(se, params["scheme"]))
except Exception as err:
self.log.error(
"Cannot create protocol on RSE",
"[RSE: %s, Error: %s]" % (se, str(err)))
else:
update = False
for protocol in protocols:
if prot == (
str(protocol["scheme"]),
str(protocol["hostname"]),
str(protocol["port"]),
str(protocol["prefix"]),
):
# Check if the protocol defined in Dirac has the same priority as the one defined in Rucio
for domain in ["lan", "wan"]:
for activity in [
"read", "write", "delete"
]:
if (params["domains"][domain]
[activity] !=
protocol["domains"][domain]
[activity]):
update = True
break
if (params["domains"]["wan"]
["third_party_copy"] !=
protocol["domains"]["wan"]
["third_party_copy"]):
update = True
if update:
data = {
"prefix":
params["prefix"],
"read_lan":
params["domains"]["lan"]["read"],
"read_wan":
params["domains"]["wan"]["read"],
"write_lan":
params["domains"]["lan"]["write"],
"write_wan":
params["domains"]["wan"]["write"],
"delete_lan":
params["domains"]["lan"]["delete"],
"delete_wan":
params["domains"]["wan"]["delete"],
"third_party_copy":
params["domains"]["wan"]["write"],
}
self.log.info(
"Will update protocol:",
"%s://%s:%s%s on %s" % (
params["scheme"],
params["hostname"],
params["port"],
params["prefix"],
se,
),
)
client.update_protocols(
rse=se,
scheme=params["scheme"],
data=data,
hostname=params["hostname"],
port=params["port"],
)
for prot in existing_protocols:
if prot not in protocols_to_create:
self.log.info(
"Will delete protocol:", "%s://%s:%s%s on %s" %
(prot[0], prot[1], prot[2], prot[3], se))
client.delete_protocols(se,
scheme=prot[0],
hostname=prot[1],
port=prot[2])
else:
self.log.error("Cannot get SEs:", result["Message"])
# If new RSE added, add distances
rses = [rse["rse"] for rse in client.list_rses()]
if newRSE:
self.log.info("Adding distances")
for src_rse, dest_rse in permutations(rses, r=2):
try:
client.add_distance(src_rse, dest_rse, {
"ranking": 1,
"distance": 10
})
except Exception as err:
self.log.error(
"Cannot add distance for",
"Source RSE: %s, Dest RSE: %s, Error:%s" %
(src_rse, dest_rse, str(err)),
)
# Collect the shares from Dirac Configuration and create them in Rucio
self.log.info("Synchronizing shares")
result = Operations().getOptionsDict("Production/SEshares")
if result["OK"]:
rseDict = result["Value"]
for rse in rses:
try:
self.log.info("Setting productionSEshare for",
"[RSE: %s : Share: %s", rse,
rseDict.get(rse, 0))
client.add_rse_attribute(rse, "productionSEshare",
rseDict.get(rse, 0))
except Exception as err:
self.log.error(
"Cannot create productionSEshare for RSE:", rse)
else:
self.log.error("Cannot get SEs", result["Message"])
result = Operations().getSections("Shares")
if result["OK"]:
for dataLevel in result["Value"]:
result = Operations().getOptionsDict("Shares/%s" %
dataLevel)
if not result["OK"]:
self.log.error("Cannot get SEs:" % result["Message"])
continue
rseDict = result["Value"]
for rse in rses:
try:
self.log.info(
"Setting", "%sShare for %s : %s" %
(dataLevel, rse, rseDict.get(rse, 0)))
client.add_rse_attribute(rse,
"%sShare" % dataLevel,
rseDict.get(rse, 0))
except Exception as err:
self.log.error("Cannot create share:",
"%sShare for %s", dataLevel, rse)
else:
self.log.error("Cannot get shares:", result["Message"])
# Create the RSE attribute PrimaryDataSE and OccupancyLFN
result = gConfig.getValue(
"Resources/StorageElementGroups/PrimarySEs")
result = getStorageElements(vo)
if result["OK"]:
allSEs = result["Value"]
primarySEs = resolveSEGroup("PrimarySEs", allSEs)
self.log.info("Will set primarySEs flag to:", str(primarySEs))
for rse in rses:
if rse in allSEs:
storage = StorageElement(rse)
if not storage.valid:
self.log.warn(
"Storage element is not valid. Skipped RSE:",
rse)
continue
occupancyLFN = storage.options.get("OccupancyLFN")
try:
client.add_rse_attribute(rse, "OccupancyLFN",
occupancyLFN)
except Exception as err:
self.log.error(
"Cannot create RSE attribute OccupancyLFN for",
"[RSE: %s, Error: %s]" % (rse, str(err)))
if rse in primarySEs:
try:
client.add_rse_attribute(rse, "PrimaryDataSE",
True)
except Exception as err:
self.log.error(
"Cannot create RSE attribute PrimaryDataSE for",
"[RSE: %s, Error: %s]" % (rse, str(err)),
)
else:
try:
client.delete_rse_attribute(rse, "PrimaryDataSE")
except RSEAttributeNotFound:
pass
except Exception as err:
self.log.error(
"Cannot remove RSE attribute PrimaryDataSE for",
"[RSE: %s, Error: %s]" % (rse, str(err)),
)
self.log.info("RSEs synchronized for VO: ", vo)
# Collect the user accounts from Dirac Configuration and create user accounts in Rucio
self.log.info("Synchronizing accounts for VO", vo)
listAccounts = [
str(acc["account"]) for acc in client.list_accounts()
]
listScopes = [str(scope) for scope in client.list_scopes()]
dnMapping = {}
diracUsers = getUsersInVO(vo)
self.log.debug(" Will consider following Dirac users for",
"[VO: %s, Dirac users: %s]" % (vo, diracUsers))
for account in diracUsers:
dn = getUserOption(account, "DN")
email = getUserOption(account, "Email")
dnMapping[dn] = email
if account not in listAccounts:
self.log.info("Will create account with associated DN ",
"[account: %s, DN: %s]" % (account, dn))
try:
client.add_account(account, "USER", email)
listAccounts.append(account)
except Exception as err:
self.log.error(
"Cannot create account",
"[account: %s, Error: %s]" % (account, str(err)))
try:
client.add_identity(account=account,
identity=dn,
authtype="X509",
email=email,
default=True)
except Exception as err:
self.log.error(
"Cannot add identity for account",
"[Identity: dn=%s, account:%s, Error: %s]" %
(dn, account, str(err)),
)
self.log.error(
" Account/identity skipped (it will not be created in Rucio)",
"[%s/%s]" % (account, dn))
continue
for rse in rses:
client.set_local_account_limit(account, rse,
1000000000000000)
else:
try:
client.add_identity(account=account,
identity=dn,
authtype="X509",
email=email,
default=True)
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account",
"[DN: %s, account: %s, Error: %s]" %
(dn, account, str(err)),
)
scope = "user." + account
if scope not in listScopes:
try:
self.log.info("Will create a scope",
"[Scope: %s]" % scope)
client.add_scope(account, scope)
self.log.info("Scope successfully added",
"[Scope: %s]" % scope)
except Exception as err:
self.log.error(
"Cannot create a scope",
"[Scope: %s, Error: %s]" % (scope, str(err)))
# Collect the group accounts from Dirac Configuration and create service accounts in Rucio
result = getGroupsForVO(vo)
if result["OK"]:
groups = result["Value"]
self.log.debug(" Will consider following Dirac groups for",
"[%s VO: %s]" % (vo, groups))
else:
groups = []
self.log.debug("No Dirac groups for", "%s VO " % vo)
self.log.debug("No Rucio service accounts will be created")
for group in groups:
if group not in listAccounts:
self.log.info(
"Will create SERVICE account for Dirac group:",
str(group))
try:
client.add_account(group, "SERVICE", None)
listAccounts.append(group)
except Exception as err:
self.log.error(
"Cannot create SERVICE account for",
"[group: %s, Error: %s]" % (group, str(err)))
for rse in rses:
client.set_local_account_limit(account, rse,
1000000000000000)
for dn in getDNsInGroup(group):
try:
client.add_identity(account=group,
identity=dn,
authtype="X509",
email=dnMapping.get(
dn, default_email))
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account",
"[identity %s, account %s, Error: %s]" %
(dn, group, str(err)),
)
self.log.error(format_exc())
# Collect the group accounts from Dirac Configuration and create service accounts in Rucio
result = getHosts()
if not result["OK"]:
self.log.error("Cannot get host accounts:",
"%s" % result["Message"])
else:
hosts = result["Value"]
for host in hosts:
dn = getHostOption(host, "DN")
email = dnMapping.get(dn, default_email)
try:
client.add_identity(account="dirac_srv",
identity=dn,
authtype="X509",
email=email)
except Duplicate:
pass
except Exception as err:
self.log.error(
"Cannot create identity for account dirac_srv:",
"[DN: %s, Error: %s]" % (dn, str(err)))
self.log.error(format_exc())
return S_OK()
except Exception as exc:
self.log.exception("Synchronisation for VO failed. VO skipped ",
"VO=%s" % vo,
lException=exc)
return S_ERROR(str(format_exc()))
def execute():
"""
Parse the options and execute the script
"""
bkQuery = dmScript.getBKQuery()
fileType = bkQuery.getFileTypeList()
if not set(fileType) & {'FULL.DST', 'RDST', 'SDST'}:
gLogger.error("Please provide a reconstruction BK path")
DIRAC.exit(1)
from LHCbDIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC.Core.Utilities.List import breakListIntoChunks
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers, resolveSEGroup
bk = BookkeepingClient()
tr = TransformationClient()
dm = DataManager()
dmsHelper = DMSHelpers()
bkQueryDict = bkQuery.getQueryDict()
gLogger.notice("For BK Query:", str(bkQueryDict))
progressBar = ProgressBar(1, title="Running BK query...", step=1)
res = bk.getFilesWithMetadata(bkQueryDict)
if not res['OK']:
gLogger.error("Error getting files from BK", res['Message'])
DIRAC.exit(2)
if 'ParameterNames' in res.get('Value', {}):
parameterNames = res['Value']['ParameterNames']
info = res['Value']['Records']
progressBar.endLoop("Obtained %d files" % len(info))
else:
gLogger.error('\nNo metadata found')
DIRAC.exit(3)
lfns = []
runLFNs = {}
for item in info:
metadata = dict(zip(parameterNames, item))
lfn = metadata['FileName']
lfns.append(lfn)
runLFNs.setdefault(metadata['RunNumber'], []).append(lfn)
chunkSize = 1000
progressBar = ProgressBar(len(lfns),
title='Getting replicas of %d files' % len(lfns),
chunk=chunkSize)
replicas = {}
errors = {}
for lfnChunk in breakListIntoChunks(lfns, chunkSize):
progressBar.loop()
res = dm.getReplicas(lfnChunk, getUrl=False)
if not res['OK']:
errors.setdefault(res['Message'], []).extend(lfnChunk)
else:
replicas.update(res['Value']['Successful'])
for lfn, error in res['Value']['Failed'].iteritems():
errors.setdefault(error, []).append(lfn)
progressBar.endLoop()
for error, lfns in errors.iteritems():
gLogger.error(error, 'for %d files' % len(lfns))
tier1RDST = set(resolveSEGroup('Tier1-RDST'))
setOK = 0
errors = {}
progressBar = ProgressBar(len(runLFNs),
title='Defining destination for %d runs' %
len(runLFNs),
step=10)
for run, lfns in runLFNs.iteritems():
progressBar.loop()
res = tr.getDestinationForRun(run)
if res.get('Value'):
errors.setdefault('Destination already set', []).append(str(run))
continue
# print 'Run', run, len( lfns ), 'Files', lfns[:3]
seCounts = {}
for lfn in lfns:
for se in tier1RDST.intersection(replicas.get(lfn, [])):
seCounts[se] = seCounts.setdefault(se, 0) + 1
# print seCounts
maxi = 0
seMax = None
for se, count in seCounts.iteritems():
if count > maxi:
seMax = se
maxi = count
if not seMax:
errors.setdefault('No SE found, use CERN-RDST',
[]).append(str(run))
seMax = 'CERN-RDST'
# SE found, get its site
res = dmsHelper.getLocalSiteForSE(seMax)
if res['OK']:
site = res['Value']
res = tr.setDestinationForRun(run, site)
if not res['OK']:
errors.setdefault(res['Message'], []).append(str(run))
else:
setOK += 1
progressBar.endLoop('Successfully set destination for %d runs' % setOK)
for error, runs in errors.iteritems():
gLogger.error(error, 'for runs %s' % ','.join(runs))
