def checkOrGetGroupProxy( group ):
"""Check if current proxy corresponds to the given group, and if not call the
:any:`getNewProxy` method to obtain a proxy for this group
:param string group: dirac group of the desired proxy
:returns: :func:`S_OK() <DIRAC:DIRAC.Core.Utilities.ReturnValues.S_OK>`, :func:`~DIRAC:DIRAC.Core.Utilities.ReturnValues.S_ERROR`
"""
result = getProxyInfo()
if result['OK'] and 'group' in result['Value']:
if result['Value']['group'] == group:
return S_OK()
else:
print "You don't have an %s proxy, trying to get one..." % group
else:
print "Error to get proxy information, trying to get proxy"
newProxyRetVal = getNewProxy( group=group )
if newProxyRetVal: ## != 0
return S_ERROR("dirac-proxy-init failed")
result = getProxyInfo()
if result['OK'] and 'group' in result['Value']:
if result['Value']['group'] == group:
return S_OK()
else:
print 'You do not have a valid group'
return S_ERROR( "Could not obtain valid group" )
elif result['OK'] and 'group' not in result['Value']:
return S_ERROR( "Could not obtain group information from proxy" )
print 'Could not obtain proxy information: %s' % result['Message']
return result
def __setupProxy(self, ownerDN, ownerGroup):
"""
Retrieve a proxy for the execution of the job
"""
if gConfig.getValue("/DIRAC/Security/UseServerCertificate", False):
proxyResult = self.__requestProxyFromProxyManager(ownerDN, ownerGroup)
if not proxyResult["OK"]:
self.log.error("Invalid Proxy", proxyResult["Message"])
return S_ERROR("Failed to setup proxy: %s" % proxyResult["Message"])
return S_OK(proxyResult["Value"])
else:
ret = getProxyInfo(disableVOMS=True)
if not ret["OK"]:
self.log.error("Invalid Proxy", ret["Message"])
return S_ERROR("Invalid Proxy")
proxyChain = ret["Value"]["chain"]
if not "groupProperties" in ret["Value"]:
print ret["Value"]
print proxyChain.dumpAllToString()
self.log.error("Invalid Proxy", "Group has no properties defined")
return S_ERROR("Proxy has no group properties defined")
groupProps = ret["Value"]["groupProperties"]
if Properties.GENERIC_PILOT in groupProps or Properties.PILOT in groupProps:
proxyResult = self.__requestProxyFromProxyManager(ownerDN, ownerGroup)
if not proxyResult["OK"]:
self.log.error("Invalid Proxy", proxyResult["Message"])
return S_ERROR("Failed to setup proxy: %s" % proxyResult["Message"])
proxyChain = proxyResult["Value"]
return S_OK(proxyChain)
def monitorUninstallation( system, component, cpu = None, hostname = None ):
"""
Register the uninstallation of a component in the ComponentMonitoringDB
"""
monitoringClient = ComponentMonitoringClient()
# Retrieve user uninstalling the component
result = getProxyInfo()
if result[ 'OK' ]:
user = result[ 'Value' ][ 'username' ]
else:
return result
if not user:
user = '******'
if not cpu:
cpu = 'Not available'
for line in open( '/proc/cpuinfo' ):
if line.startswith( 'model name' ):
cpu = line.split( ':' )[1][0:64]
cpu = cpu.replace( '\n', '' ).lstrip().rstrip()
if not hostname:
hostname = socket.getfqdn()
instance = component[ 0 : 32 ]
result = monitoringClient.updateInstallations \
( { 'Instance': instance, 'UnInstallationTime': None },
{ 'System': system },
{ 'HostName': hostname, 'CPU': cpu },
{ 'UnInstallationTime': datetime.datetime.utcnow(),
'UnInstalledBy': user } )
return result
def getToken(key):
"""
Function that gets the userName from the proxy
"""
proxyInfo = getProxyInfo()
if not proxyInfo["OK"]:
error(str(proxyInfo))
if key.lower() == "owner":
userName = proxyInfo["Value"]["username"]
tokenOwner = S_OK(userName)
if not tokenOwner["OK"]:
error(tokenOwner["Message"])
return tokenOwner["Value"]
elif key.lower() == "expiration":
expiration = proxyInfo["Value"]["secondsLeft"]
tokenExpiration = S_OK(expiration)
if not tokenExpiration["OK"]:
error(tokenExpiration["Message"])
now = Time.dateTime()
# datetime.datetime.utcnow()
expirationDate = now + datetime.timedelta(seconds=tokenExpiration["Value"])
expirationDate = Time.toString(expirationDate)
expirationDate = expirationDate.split(".")[0]
return expirationDate
def setSiteStatus(self, site, status, comment='No comment'):
"""
Set the status of a site in the 'SiteStatus' table of RSS
examples
>>> siteStatus.banSite( 'site1.test.test' )
S_OK()
>>> siteStatus.banSite( None )
S_ERROR( ... )
:Parameters:
**site** - `String`
the site that is going to be banned
**comment** - `String`
reason for banning
:return: S_OK() || S_ERROR()
"""
if not status:
return S_ERROR(DErrno.ERESUNK, 'status parameter is empty')
# fix case sensitive string
status = status.capitalize()
allowedStateList = ['Active', 'Banned', 'Degraded', 'Probing', 'Error', 'Unknown']
if status not in allowedStateList:
return S_ERROR(errno.EINVAL, 'Not a valid status, parameter rejected')
if self.rssFlag:
result = getProxyInfo()
if result['OK']:
tokenOwner = result['Value']['username']
else:
return S_ERROR("Unable to get user proxy info %s " % result['Message'])
tokenExpiration = datetime.utcnow() + timedelta(days=1)
self.rssCache.acquireLock()
try:
result = self.rsClient.modifyStatusElement('Site', 'Status', status=status, name=site,
tokenExpiration=tokenExpiration, reason=comment,
tokenOwner=tokenOwner)
if result['OK']:
self.rssCache.refreshCache()
else:
_msg = 'Error updating status of site %s to %s' % (site, status)
gLogger.warn('RSS: %s' % _msg)
# Release lock, no matter what.
finally:
self.rssCache.releaseLock()
else:
if status in ['Active', 'Degraded']:
result = RPCClient('WorkloadManagement/WMSAdministrator').allowSite()
else:
result = RPCClient('WorkloadManagement/WMSAdministrator').banSite()
return result
def __setupProxy( self, ownerDN, ownerGroup ):
"""
Retrieve a proxy for the execution of the job
"""
if gConfig.getValue( '/DIRAC/Security/UseServerCertificate' , False ):
proxyResult = self.__requestProxyFromProxyManager( ownerDN, ownerGroup )
if not proxyResult['OK']:
self.log.error( 'Invalid Proxy', proxyResult['Message'] )
return S_ERROR( 'Failed to setup proxy: %s' % proxyResult[ 'Message' ] )
return S_OK( proxyResult['Value'] )
else:
ret = getProxyInfo( disableVOMS = True )
if not ret['OK']:
self.log.error( 'Invalid Proxy', ret['Message'] )
return S_ERROR( 'Invalid Proxy' )
proxyChain = ret['Value']['chain']
if not 'groupProperties' in ret['Value']:
print ret['Value']
print proxyChain.dumpAllToString()
self.log.error( 'Invalid Proxy', 'Group has no properties defined' )
return S_ERROR( 'Proxy has no group properties defined' )
groupProps = ret['Value']['groupProperties']
if Properties.GENERIC_PILOT in groupProps or Properties.PILOT in groupProps:
proxyResult = self.__requestProxyFromProxyManager( ownerDN, ownerGroup )
if not proxyResult['OK']:
self.log.error( 'Invalid Proxy', proxyResult['Message'] )
return S_ERROR( 'Failed to setup proxy: %s' % proxyResult[ 'Message' ] )
proxyChain = proxyResult['Value']
return S_OK( proxyChain )
def initialize( self, request ):
""" Set default values to attributes and parameters
"""
if type( request ) == NoneType:
# Set some defaults
for name in self.attributeNames:
self.attributes[name] = 'Unknown'
self.attributes['CreationTime'] = str( Time.dateTime() )
self.attributes['Status'] = "New"
result = getProxyInfo()
if result['OK']:
proxyDict = result[ 'Value' ]
self.attributes['OwnerDN'] = proxyDict[ 'identity' ]
if 'group' in proxyDict:
self.attributes['OwnerGroup'] = proxyDict[ 'group' ]
self.attributes['DIRACSetup'] = gConfig.getValue( '/DIRAC/Setup', 'Unknown' )
elif isinstance( request, RequestContainer ):
for attr in self.attributeNames:
self.attributes[attr] = request.attributes[attr]
# initialize request from an XML string
if type( request ) in StringTypes:
for name in self.attributeNames:
self.attributes[name] = 'Unknown'
self.parseRequest( request )
# Initialize request from another request
elif isinstance( request, RequestContainer ):
self.subRequests = copy.deepcopy( request.subrequests )
def getToken( key ):
'''
Function that gets the userName from the proxy
'''
proxyInfo = getProxyInfo()
if not proxyInfo[ 'OK' ]:
error( str( proxyInfo ) )
if key.lower() == 'owner':
userName = proxyInfo[ 'Value' ][ 'username' ]
tokenOwner = S_OK( userName )
if not tokenOwner[ 'OK' ]:
error( tokenOwner[ 'Message' ] )
return tokenOwner[ 'Value' ]
elif key.lower() == 'expiration':
expiration = proxyInfo[ 'Value' ][ 'secondsLeft' ]
tokenExpiration = S_OK( expiration )
if not tokenExpiration[ 'OK' ]:
error( tokenExpiration[ 'Message' ] )
now = Time.dateTime()
#datetime.datetime.utcnow()
expirationDate = now + datetime.timedelta( seconds=tokenExpiration['Value'] )
expirationDate = Time.toString( expirationDate )
expirationDate = expirationDate.split('.')[0]
return expirationDate
def _banStorageElement( self, storageElement ):
endpoints = getFTS3Servers()[ 'Value' ]
blacklist = {}
for endpoint in endpoints:
# endpoint = 'https://fts3-pilot.cern.ch:8446'
# TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API
proxyPath = getProxyInfo()
if not proxyPath['OK']:
return proxyPath
try:
proxyPath = proxyPath['Value']['path']
except Exception as e:
return S_ERROR( repr( e ).replace( ',)', ')' ) )
context = fts3.Context( endpoint, proxyPath )
timeout = 3600 # or...?
status = 'wait' # or...?
allow_submit = False # or...?
# TODO: ban_se returns the list of jobIDs interrupted by the banning
pausedJobIDs = fts3.ban_se( context, storageElement, status, timeout, allow_submit )
self.log.info( "fts3.ban_se: %s" % pausedJobIDs )
blacklist[endpoint] = json.loads( context.get( "ban/se" ) )
return S_OK( blacklist )
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN='',
bulkSubmissionFlag=False):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername(owner)
if not res['OK']:
return res
ownerDN = res['Value'][0]
try:
transJson = json.loads(transBody)
self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup)
except ValueError: # #json couldn't load
self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup)
return S_OK(taskDict)
def _unbanStorageElement(self, storageElement):
endpoints = getFTS3Servers()
if not endpoints["OK"]:
return endpoints
endpoints = endpoints["Value"]
blacklist = {}
for endpoint in endpoints:
# endpoint = 'https://fts3-pilot.cern.ch:8446'
# TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API
proxyPath = getProxyInfo()
if not proxyPath["OK"]:
return proxyPath
try:
proxyPath = proxyPath["Value"]["path"]
except Exception as e:
return S_ERROR(repr(e).replace(",)", ")"))
context = fts3.Context(endpoint, proxyPath)
fts3.unban_se(context, storageElement)
blacklist[endpoint] = json.loads(context.get("ban/se"))
return S_OK(blacklist)
def __getPrefix(self):
op = Operations("glast.org")
self.userprefix = None
res = getProxyInfo()
if res["OK"]:
if "username" in res["Value"]:
user = res["Value"]["username"]
self.userprefix = "/glast.org/user/%s/%s/" % (user[0], user)
else:
self.log.error("Proxy could not be found")
return 1
task_category = os.environ["GPL_TASKCATEGORY"]
if not task_category:
task_category = op.getValue("Pipeline/TaskCategory", None)
if not task_category:
self.log.error("Could not find task category")
return 1
self.prefixDest = op.getValue("Pipeline/StorageElementBasePath", self.userprefix)
self.stagingDest = (
self.prefixDest
+ "/"
+ task_category
+ "/"
+ os.environ["PIPELINE_TASK"]
+ "/"
+ os.environ["PIPELINE_STREAM"]
)
return 0
def _banStorageElement( self, storageElement ):
endpoints = getFTS3Servers()[ 'Value' ]
blacklist = {}
for endpoint in endpoints:
#endpoint = 'https://fts3-pilot.cern.ch:8446'
#TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API
proxyPath = getProxyInfo()
if not proxyPath.get('OK'):
return S_ERROR("Proxy not found!")
try:
proxyPath = proxyPath.get('Value').get('path')
except Exception as e:
return S_ERROR(e.message)
context = fts3.Context(endpoint, proxyPath)
timeout = 3600 #or...?
status = 'wait' #or...?
allow_submit = False #or...?
#TODO: ban_se returns the list of jobIDs interrupted by the banning
pausedJobIDs = fts3.ban_se(context, storageElement, status, timeout, allow_submit)
blacklist[endpoint] = json.loads(context.get("ban/se"))
return S_OK( blacklist )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __init__(self, script = None):
super(ProductionJob, self).__init__( script )
self.prodVersion = __RCSID__
self.dryrun = False
self.created = False
self.checked = False
self.call_finalization = False
self.finalsdict = {}
self.transfid = 0
self.type = 'Production'
self.csSection = '/Production/Defaults'
self.ops = Operations()
self.fc = FileCatalogClient()
self.trc = TransformationClient()
self.defaultProdID = '12345'
self.defaultProdJobID = '12345'
self.jobFileGroupSize = 1
self.nbtasks = 1
self.slicesize =0
self.basename = ''
self.basepath = self.ops.getValue('/Production/CLIC/BasePath','/ilc/prod/clic/')
self.evttype = ''
self.datatype = ''
self.energycat = ''
self.detector = ''
self.currtrans = None
self.description = ''
self.finalpaths = []
self.finalMetaDict = defaultdict( dict )
self.prodMetaDict = {}
self.finalMetaDictNonSearch = {}
self.metadict_external = {}
self.outputStorage = ''
self.proxyinfo = getProxyInfo()
self.inputdataquery = False
self.inputBKSelection = {}
self.plugin = 'Standard'
self.prodGroup = ''
self.prodTypes = ['MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction',
'MCReconstruction_Overlay', 'Merge', 'Split',
'MCGeneration_ILD',
'MCSimulation_ILD',
'MCReconstruction_ILD',
'MCReconstruction_Overlay_ILD',
'Split_ILD'
]
self.prodparameters = {}
self.prodparameters['NbInputFiles'] = 1
self.prodparameters['nbevts'] = 0
#self.prodparameters["SWPackages"] = ''
self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job")
if not script:
self.__setDefaults()
self._recBasePaths = {}
self.maxFCFoldersToCheck = 100000
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '',
ownerDN = '', bulkSubmissionFlag = False ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
jobClass is by default "DIRAC.Interfaces.API.Job.Job". An extension of it also works.
:param transBody: transformation job template
:param taskDict: dictionary of per task parameters
:param owner: owner of the transformation
:param ownerGroup: group of the owner of the transformation
:param ownerDN: DN of the owner of the transformation
:return: S_OK/S_ERROR with updated taskDict
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
if bulkSubmissionFlag:
return self.__prepareTransformationTasksBulk( transBody, taskDict, owner, ownerGroup, ownerDN )
else:
return self.__prepareTransformationTasks( transBody, taskDict, owner, ownerGroup, ownerDN )
def __changeSiteStatus( self, site, comment, statusType, status, printOutput = False ):
"""
Change the RSS status of the given site
"""
result = self.__checkSiteIsValid( site )
if not result['OK']:
return result
wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
result = wmsAdmin.allowSite( site, comment )
if not result['OK']:
return result
rsc = ResourceStatusClient()
proxyInfo = getProxyInfo()
if not proxyInfo[ 'OK' ]:
return proxyInfo
userName = proxyInfo[ 'Value' ][ 'username' ]
tomorrow = datetime.utcnow().replace( microsecond = 0 ) + timedelta( days = 1 )
result = rsc.modifyStatusElement( 'Site', 'Status',
name = site,
statusType = statusType,
status = status,
reason = comment,
tokenOwner = userName,
tokenExpiration = tomorrow )
return result
def notifyAboutNewSoftware(self):
"""Send an email to the mailing list if a new software version was defined"""
#Only send email when something was actually added
if not self.modifiedCS:
return
subject = '%s %s added to DIRAC CS' % (self.appName, self.appVersion)
msg = 'New application %s %s declared into Configuration service\n %s' % (self.appName,
self.appVersion,
self.comment)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUserOption
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
notifyClient = NotificationClient()
gLogger.notice('Sending mail for software installation to %s' % (self.mailadress))
res = getProxyInfo()
if not res['OK']:
sender = '*****@*****.**'
else:
if 'username' in res['Value']:
sender = getUserOption(res['Value']['username'],'Email')
else:
sender = '*****@*****.**'
gLogger.info('*'*80)# surround email with stars
res = notifyClient.sendMail(self.mailadress, subject, msg, sender, localAttempt = False)
gLogger.info('*'*80)
if not res[ 'OK' ]:
gLogger.error('The mail could not be sent: %s' % res['Message'])
def initialize(self, request):
""" Set default values to attributes and parameters
"""
if type(request) == NoneType:
# Set some defaults
for name in self.attributeNames:
self.attributes[name] = "Unknown"
self.attributes["CreationTime"] = str(Time.dateTime())
self.attributes["Status"] = "New"
result = getProxyInfo()
if result["OK"]:
proxyDict = result["Value"]
self.attributes["OwnerDN"] = proxyDict["identity"]
if "group" in proxyDict:
self.attributes["OwnerGroup"] = proxyDict["group"]
self.attributes["DIRACSetup"] = gConfig.getValue("/DIRAC/Setup", "Unknown")
elif isinstance(request, RequestContainer):
for attr in self.attributeNames:
self.attributes[attr] = request.attributes[attr]
# initialize request from an XML string
if type(request) in StringTypes:
for name in self.attributeNames:
self.attributes[name] = "Unknown"
self.parseRequest(request)
# Initialize request from another request
elif isinstance(request, RequestContainer):
self.subRequests = copy.deepcopy(request.subrequests)
def uploadProxyToMyProxy( params, DNAsUsername ):
""" Upload proxy to the MyProxy server
"""
myProxy = MyProxy()
if DNAsUsername:
gLogger.verbose( "Uploading pilot proxy with group %s to %s..." % ( params.getDIRACGroup(), myProxy.getMyProxyServer() ) )
else:
gLogger.verbose( "Uploading user proxy with group %s to %s..." % ( params.getDIRACGroup(), myProxy.getMyProxyServer() ) )
retVal = myProxy.getInfo( proxyInfo[ 'path' ], useDNAsUserName = DNAsUsername )
if retVal[ 'OK' ]:
remainingSecs = ( int( params.getProxyRemainingSecs() / 3600 ) * 3600 ) - 7200
myProxyInfo = retVal[ 'Value' ]
if 'timeLeft' in myProxyInfo and remainingSecs < myProxyInfo[ 'timeLeft' ]:
gLogger.verbose( " Already uploaded" )
return True
retVal = generateProxy( params )
if not retVal[ 'OK' ]:
gLogger.error( " There was a problem generating proxy to be uploaded to myproxy: %s" % retVal[ 'Message' ] )
return False
retVal = getProxyInfo( retVal[ 'Value' ] )
if not retVal[ 'OK' ]:
gLogger.error( " There was a problem generating proxy to be uploaded to myproxy: %s" % retVal[ 'Message' ] )
return False
generatedProxyInfo = retVal[ 'Value' ]
retVal = myProxy.uploadProxy( generatedProxyInfo[ 'path' ], useDNAsUserName = DNAsUsername )
if not retVal[ 'OK' ]:
gLogger.error( " Can't upload to myproxy: %s" % retVal[ 'Message' ] )
return False
gLogger.verbose( " Uploaded" )
return True
def _banStorageElement( self, storageElement ):
endpoints = getFTS3Servers()
if not endpoints['OK']:
return endpoints
endpoints = endpoints['Value']
blacklist = {}
for endpoint in endpoints:
# endpoint = 'https://fts3-pilot.cern.ch:8446'
# TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API
proxyPath = getProxyInfo()
if not proxyPath['OK']:
return proxyPath
try:
proxyPath = proxyPath['Value']['path']
except Exception as e:
return S_ERROR( repr( e ).replace( ',)', ')' ) )
context = fts3.Context( endpoint, proxyPath )
status = 'wait' # This status leaves the jobs queued. The only alternative is "cancel"
pausedJobIDs = fts3.ban_se( context, storageElement, status, timeout = 3600, allow_submit = False )
self.log.info( "fts3.ban_se: paused jobs: %s" % ','.join(pausedJobIDs) )
blacklist[endpoint] = json.loads( context.get( "ban/se" ) )
return S_OK( blacklist )
def _unbanStorageElement( self, storageElement ):
endpoints = getFTS3Servers()[ 'Value' ]
blacklist = {}
for endpoint in endpoints:
#endpoint = 'https://fts3-pilot.cern.ch:8446'
#TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API
proxyPath = getProxyInfo()
if not proxyPath.get('OK'):
return S_ERROR("Proxy not found!")
try:
proxyPath = proxyPath.get('Value').get('path')
except Exception as e:
return S_ERROR(e.message)
context = fts3.Context(endpoint, proxyPath)
fts3.unban_se(context, storageElement)
blacklist[endpoint] = json.loads(context.get("ban/se"))
return S_OK( blacklist )
def __init__(self, script=None): """Initialize UserJob, including proxy and splitmixin.""" super(UserJob, self).__init__(script) self.type = 'User' self.diracinstance = None self.usergroup = ['ilc_user', 'calice_user'] self.proxyinfo = getProxyInfo() SplitMixin._initialize(self)
def execute( self ):
res = getProxyInfo( disableVOMS = True )
if not res['OK']:
return res
self.proxyInfoDict = res['Value']
return self.monitorStageRequests()
def __init__( self, fromDict = None ):
"""c'tor
:param self: self reference
:param fromDict : if false, new request. Can be json string that represents the object, or the dictionary directly
"""
self.__waiting = None
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self._CreationTime = now
self._SubmitTime = now
self._LastUpdate = now
# the time before which the request should not be executed
# If None, no delay
self._NotBefore = now
self._Status = "Done"
self.JobID = 0
self.Error = None
self.DIRACSetup = None
self.OwnerDN = None
self.RequestName = None
self.OwnerGroup = None
self.SourceComponent = None
self.dmsHelper = DMSHelpers()
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__operations__ = []
fromDict = fromDict if isinstance( fromDict, dict )\
else json.loads( fromDict ) if isinstance( fromDict, StringTypes )\
else {}
if "Operations" in fromDict:
for opDict in fromDict.get( "Operations", [] ):
self +=Operation( opDict )
del fromDict["Operations"]
for key, value in fromDict.items():
# The JSON module forces the use of UTF-8, which is not properly
# taken into account in DIRAC.
# One would need to replace all the '== str' with 'in StringTypes'
if type( value ) in StringTypes:
value = value.encode()
if value:
setattr( self, key, value )
self._notify()
def setUp( self ):
gLogger.setLevel( 'DEBUG' )
self.wrapperFile = None
# get proxy
proxyInfo = getProxyInfo( disableVOMS = True )
proxyChain = proxyInfo['Value']['chain']
proxyDumped = proxyChain.dumpAllToString()
self.payloadProxy = proxyDumped['Value']
def monitorInstallation( componentType, system, component, module = None, cpu = None, hostname = None ):
"""
Register the installation of a component in the ComponentMonitoringDB
"""
monitoringClient = ComponentMonitoringClient()
if not module:
module = component
# Retrieve user installing the component
user = None
result = getProxyInfo()
if result[ 'OK' ]:
proxyInfo = result[ 'Value' ]
if 'username' in proxyInfo:
user = proxyInfo[ 'username' ]
else:
return result
if not user:
user = '******'
if not cpu:
cpu = 'Not available'
for line in open( '/proc/cpuinfo' ):
if line.startswith( 'model name' ):
cpu = line.split( ':' )[1][ 0 : 64 ]
cpu = cpu.replace( '\n', '' ).lstrip().rstrip()
if not hostname:
hostname = socket.getfqdn()
instance = component[ 0 : 32 ]
result = monitoringClient.installationExists \
( { 'Instance': instance,
'UnInstallationTime': None },
{ 'Type': componentType,
'System': system,
'Module': module },
{ 'HostName': hostname,
'CPU': cpu } )
if not result[ 'OK' ]:
return result
if result[ 'Value' ]:
return S_OK( 'Monitoring of %s is already enabled' % component )
result = monitoringClient.addInstallation \
( { 'InstallationTime': datetime.datetime.utcnow(),
'InstalledBy': user,
'Instance': instance },
{ 'Type': componentType,
'System': system,
'Module': module },
{ 'HostName': hostname,
'CPU': cpu },
True )
return result
def loadCredentials( self ):
retVal = getProxyInfo()
if retVal[ 'OK' ]:
credDict = retVal[ 'Value' ]
self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ],
credDict[ 'group' ],
Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) )
return retVal
return retVal
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
if (not owner) or (not ownerGroup):
res = getProxyInfo(False,False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
oJob = Job(transBody)
for taskNumber in sortList(taskDict.keys()):
paramsDict = taskDict[taskNumber]
transID = paramsDict['TransformationID']
self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup))
oJob.setOwner(owner)
oJob.setOwnerGroup(ownerGroup)
transGroup = str(transID).zfill(8)
self.log.verbose('Adding default transformation group of %s' % (transGroup))
oJob.setJobGroup(transGroup)
constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8)
self.log.verbose('Setting task name to %s' % constructedName)
oJob.setName(constructedName)
oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8))
oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8))
inputData = None
for paramName,paramValue in paramsDict.items():
self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue))
if paramName=='InputData':
if paramValue:
self.log.verbose('Setting input data to %s' %paramValue)
oJob.setInputData(paramValue)
elif paramName=='Site':
if paramValue:
self.log.verbose('Setting allocated site to: %s' %(paramValue))
oJob.setDestination(paramValue)
elif paramValue:
self.log.verbose('Setting %s to %s' % (paramName,paramValue))
oJob._addJDLParameter(paramName,paramValue)
hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])]
if int(transID) in hospitalTrans:
hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch')
hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[])
oJob.setType('Hospital')
oJob.setDestination(hospitalSite)
oJob.setInputDataPolicy('download',dataScheduling=False)
if hospitalCEs:
oJob._addJDLParameter('GridRequiredCEs',hospitalCEs)
taskDict[taskNumber]['TaskObject'] = ''
res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData})
if not res ['OK']:
self.log.error("Failed to generate output data",res['Message'])
continue
for name,output in res['Value'].items():
oJob._addJDLParameter(name,string.join(output,';'))
taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML())
return S_OK(taskDict)
def submitJob(self, executableFile, proxy, **kwargs):
""" Method to submit job.
"""
if self.pPool is None:
self.pPool = ProcessPool(minSize=self.processors,
maxSize=self.processors,
poolCallback=self.finalizeJob)
self.pPool.processResults()
processorsInUse = self.getProcessorsInUse()
if kwargs.get('wholeNode'):
if processorsInUse > 0:
return S_ERROR('Can not take WholeNode job') # , %d/%d slots used' % (self.slotsInUse,self.slots) )
else:
requestedProcessors = self.processors
elif "numberOfProcessors" in kwargs:
requestedProcessors = int(kwargs['numberOfProcessors'])
if requestedProcessors > 0:
if (processorsInUse + requestedProcessors) > self.processors:
return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
self.processors - processorsInUse))
else:
requestedProcessors = 1
if self.processors - processorsInUse < requestedProcessors:
return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
self.processors - processorsInUse))
ret = getProxyInfo()
if not ret['OK']:
pilotProxy = None
else:
pilotProxy = ret['Value']['path']
self.log.notice('Pilot Proxy:', pilotProxy)
kwargs = {'UseSudo': False}
if self.useSudo:
for nUser in range(MAX_NUMBER_OF_SUDO_UNIX_USERS):
if nUser not in self.userNumberPerTask.values():
break
kwargs['NUser'] = nUser
kwargs['PayloadUser'] = os.environ['USER'] + 'p%s' % str(nUser).zfill(2)
kwargs['UseSudo'] = True
result = self.pPool.createAndQueueTask(executeJob,
args=(executableFile, proxy, self.taskID),
kwargs=kwargs,
taskID=self.taskID,
usePoolCallbacks=True)
self.processorsPerTask[self.taskID] = requestedProcessors
self.taskID += 1
self.pPool.processResults()
return result
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def proxyUser():
"""
Read proxy to get username.
"""
res = getProxyInfo()
if not res["OK"]:
return res
return S_OK(res["Value"]["username"])
def getTokenOwner():
'''
Function that gets the userName from the proxy
'''
proxyInfo = getProxyInfo()
if not proxyInfo['OK']:
return proxyInfo
userName = proxyInfo['Value']['username']
return S_OK(userName)
def proxyUser():
"""
Read proxy to get username.
"""
res = getProxyInfo()
if not res[ 'OK' ]:
return res
return S_OK( res[ 'Value' ][ 'username' ] )
def execute( self ):
res = getProxyInfo( disableVOMS = True )
if not res['OK']:
return res
self.proxyInfoDict = res['Value']
res = self.monitorStageRequests()
return res
def loadCredentials(self):
retVal = getProxyInfo()
if retVal["OK"]:
credDict = retVal["Value"]
self.commiterId = "%s@%s - %s" % (
credDict["username"],
credDict["group"],
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
)
return retVal
return retVal
def __ownerAndGroup(self):
res = getProxyInfo(False, False)
if not res['OK']:
raise Exception('GetProxyInfo error: {0}'.format(res['Message']))
self.__owner = res['Value']['username']
self.__ownerGroup = res['Value']['group']
self.__vo = Registry.getVOMSVOForGroup(self.__ownerGroup)
self.__voHome = '/{0}'.format(self.__vo)
self.__prodHome = '/{0}/reconstruction'.format(self.__vo)
self.__userHome = '/{0}/user/{1:.1}/{1}'.format(
self.__vo, self.__owner)
def getCurrentOwner():
"""Simple function to return current DIRAC username.
"""
result = getProxyInfo()
if not result['OK']:
return S_ERROR('Could not obtain proxy information')
if 'username' not in result['Value']:
return S_ERROR('Could not get username from proxy')
username = result['Value']['username']
return S_OK(username)
def setSiteStatus(self, site, status, comment='No comment'):
"""
Set the status of a site in the 'SiteStatus' table of RSS
examples
>>> siteStatus.banSite( 'site1.test.test' )
S_OK()
>>> siteStatus.banSite( None )
S_ERROR( ... )
:Parameters:
**site** - `String`
the site that is going to be banned
**comment** - `String`
reason for banning
:return: S_OK() || S_ERROR()
"""
if not status:
return S_ERROR(DErrno.ERESUNK, 'status parameter is empty')
# fix case sensitive string
status = status.capitalize()
allowedStateList = [
'Active', 'Banned', 'Degraded', 'Probing', 'Error', 'Unknown'
]
if status not in allowedStateList:
return S_ERROR(errno.EINVAL,
'Not a valid status, parameter rejected')
result = getProxyInfo()
if result['OK']:
tokenOwner = result['Value']['username']
else:
return S_ERROR("Unable to get user proxy info %s " %
result['Message'])
tokenExpiration = datetime.utcnow() + timedelta(days=1)
result = self.rsClient.modifyStatusElement(
'Site',
'Status',
status=status,
name=site,
tokenExpiration=tokenExpiration,
reason=comment,
tokenOwner=tokenOwner)
if not result['OK']:
return result
return S_OK()
def _getCurrentUser( self ):
res = getProxyInfo( False, False )
if not res['OK']:
return self._errorReport( 'No proxy found in local environment', res['Message'] )
proxyInfo = res['Value']
gLogger.debug( formatProxyInfoAsString( proxyInfo ) )
if 'group' not in proxyInfo:
return self._errorReport( 'Proxy information does not contain the group', res['Message'] )
res = getDNForUsername( proxyInfo['username'] )
if not res['OK']:
return self._errorReport( 'Failed to get proxies for user', res['Message'] )
return S_OK( proxyInfo['username'] )
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs",
"LFN: Logical File Names"))
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
allowUsers = Operations().getValue(
"DataManagement/AllowUserReplicaManagement", False)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res["OK"]:
gLogger.fatal("Can't get proxy info", res["Message"])
dexit(1)
properties = res["Value"].get("groupProperties", [])
if not allowUsers:
if "FileCatalogManagement" not in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement"
)
dexit(5)
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
fc = FileCatalog()
import os
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = fc.removeFile(lfns)
if not res["OK"]:
print("Error:", res["Message"])
dexit(1)
for lfn in sorted(res["Value"]["Failed"].keys()):
message = res["Value"]["Failed"][lfn]
print("Error: failed to remove %s: %s" % (lfn, message))
print("Successfully removed %d catalog files." %
(len(res["Value"]["Successful"])))
def __init__(self, fromDict=None):
"""c'tor
:param self: self reference
:param fromDict: if false, new request. Can be json string that represents the object, or the dictionary directly
"""
self.__waiting = None
now = datetime.datetime.utcnow().replace(microsecond=0)
self._CreationTime = now
self._SubmitTime = now
self._LastUpdate = now
# the time before which the request should not be executed
# If None, no delay
self._NotBefore = now
self._Status = "Done"
self.JobID = 0
self.Error = None
self.DIRACSetup = None
self.OwnerDN = None
self.RequestName = None
self.OwnerGroup = None
self._SourceComponent = None
self.dmsHelper = DMSHelpers()
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__operations__ = []
if isinstance(fromDict, str):
fromDict = json.loads(fromDict)
elif not isinstance(fromDict, dict):
fromDict = {}
if "Operations" in fromDict:
for opDict in fromDict.get("Operations", []):
self += Operation(opDict)
del fromDict["Operations"]
for key, value in fromDict.items():
if value:
setattr(self, key, value)
self._notify()
def getDatasetNamePrefix(self):
"""descide the prefix of a datasetName"""
prefix = ''
result = getProxyInfo(False, False)
if result['OK']:
userGroup = result['Value']['group']
if userGroup == 'bes_user':
prefix = 'User_'
elif userGroup == 'production':
prefix = 'Proc_'
return prefix
else:
return prefix
def execute(self):
self.log.info("Start Execution")
result = getProxyInfo()
if not result['OK']:
return result
infoDict = result['Value']
self.log.info(formatProxyInfoAsString(infoDict))
self.__lookForCE()
self.__infoFromCE()
self.log.info("End Execution")
return S_OK()
def getCurrentVO():
"""Simple function to return current DIRAC username.
"""
result = getProxyInfo()
if not result['OK']:
return S_ERROR('Could not obtain proxy information')
if 'group' not in result['Value']:
return S_ERROR('Could not get group from proxy')
group = result['Value']['group']
vo = group.split("_")[0]
return S_OK(vo)
def submitJob( self, executableFile, proxy, dummy = None ):
""" Method to submit job, should be overridden in sub-class.
"""
ret = getProxyInfo()
if not ret['OK']:
pilotProxy = None
else:
pilotProxy = ret['Value']['path']
self.log.notice( 'Pilot Proxy:', pilotProxy )
payloadEnv = dict( os.environ )
payloadProxy = ''
if proxy:
self.log.verbose( 'Setting up proxy for payload' )
result = self.writeProxyToFile( proxy )
if not result['OK']:
return result
payloadProxy = result['Value']
# pilotProxy = os.environ['X509_USER_PROXY']
payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy
self.log.verbose( 'Starting process for monitoring payload proxy' )
gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy, taskArgs = ( pilotProxy, payloadProxy ), executions = 0, elapsedTime = 0 )
if not os.access( executableFile, 5 ):
os.chmod( executableFile, 0755 )
cmd = os.path.abspath( executableFile )
self.log.verbose( 'CE submission command: %s' % ( cmd ) )
result = systemCall( 0, cmd, callbackFunction = self.sendOutput, env = payloadEnv )
if payloadProxy:
os.unlink( payloadProxy )
ret = S_OK()
if not result['OK']:
self.log.error( 'Fail to run InProcess', result['Message'] )
elif result['Value'][0] < 0:
self.log.error( 'InProcess Job Execution Failed' )
self.log.info( 'Exit status:', result['Value'][0] )
return S_ERROR( 'InProcess Job Execution Failed' )
elif result['Value'][0] > 0:
self.log.error( 'Fail in payload execution' )
self.log.info( 'Exit status:', result['Value'][0] )
ret['PayloadFailed'] = result['Value'][0]
else:
self.log.debug( 'InProcess CE result OK' )
self.submittedJobs += 1
return ret
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
self.log = gLogger
self.section = COMPONENT_NAME
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
self.systemConfig = 'ANY'
self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE',
'MinCPUTime': 'other.NAME<=VALUE',
'Site': 'other.NAME=="VALUE"',
'Platform': 'other.NAME=="VALUE"',
#'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately
'BannedSites': 'other.Site!="VALUE"',
'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'}
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
#Global error dictionary
self.errorDict = {}
def main():
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
allowUsers = Operations().getValue(
"DataManagement/AllowUserReplicaManagement", False)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res['OK']:
gLogger.fatal("Can't get proxy info", res['Message'])
dexit(1)
properties = res['Value'].get('groupProperties', [])
if not allowUsers:
if 'FileCatalogManagement' not in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement"
)
dexit(5)
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
fc = FileCatalog()
import os
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp(exitCode=1)
else:
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = fc.removeFile(lfns)
if not res['OK']:
print("Error:", res['Message'])
dexit(1)
for lfn in sorted(res['Value']['Failed'].keys()):
message = res['Value']['Failed'][lfn]
print('Error: failed to remove %s: %s' % (lfn, message))
print('Successfully removed %d catalog files.' %
(len(res['Value']['Successful'])))
def getAuthorDNfromProxy(self):
""" gets the AuthorDN and username of the transformation from the uploaded proxy
"""
username = ""
author = ""
res = getProxyInfo()
if res['OK']:
author = res['Value']['identity']
username = res['Value']['username']
else:
gLogger.error("Unable to get uploaded proxy Info %s " % res['Message'])
return S_ERROR(res['Message'])
res = {'username': username, 'authorDN': author}
return S_OK(res)
def getAuthorDNfromProxy(self):
"""gets the AuthorDN and username of the transformation from the uploaded proxy"""
username = ""
author = ""
res = getProxyInfo()
if res["OK"]:
author = res["Value"]["identity"]
username = res["Value"]["username"]
else:
gLogger.error("Unable to get uploaded proxy Info %s " %
res["Message"])
return S_ERROR(res["Message"])
res = {"username": username, "authorDN": author}
return S_OK(res)
def __init__(self, script=None):
super(UserJob, self).__init__(script)
self.type = 'User'
self.diracinstance = None
self.usergroup = ['ilc_user', 'calice_user']
self.proxyinfo = getProxyInfo()
########## SPLITTING STUFF: ATTRIBUTES ##########
self._data = []
self.splittingOption = None
self._switch = {}
self.numberOfJobs = None
self.totalNumberOfEvents = None
self.eventsPerJob = None
self.numberOfFilesPerJob = 1
def checkCredentials(self, requested_group):
""" Utility method to check the credentials and that the use belongs
to a specific group """
# First locate the DIRAC code
try:
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
except:
raise DiracNotFoundException("DIRAC not found")
# Now check the credentials themselves
# Now using dirac to lookup the proxy info
try:
info = getProxyInfo()
except Exception, e:
raise InvalidGridCredentialsException(str(e))
def main():
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
allowUsers = Operations().getValue(
"DataManagement/AllowUserReplicaManagement", False)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res['OK']:
gLogger.fatal("Can't get proxy info", res['Message'])
dexit(1)
properties = res['Value'].get('groupProperties', [])
if not allowUsers:
if 'FileCatalogManagement' not in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement"
)
dexit(5)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp(exitCode=1)
else:
inputFileName = args[0]
storageElementName = args[1]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = dm.removeReplicaFromCatalog(storageElementName, lfns)
if not res['OK']:
print(res['Message'])
dexit(0)
for lfn in sorted(res['Value']['Failed']):
message = res['Value']['Failed'][lfn]
print('Failed to remove %s replica of %s: %s' %
(storageElementName, lfn, message))
print('Successfully remove %d catalog replicas at %s' %
(len(res['Value']['Successful']), storageElementName))
def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"):
# Warm up local CS
CSHelpers.warmUp()
if rStatus is None:
self.rStatus = ResourceStatusClient()
if rManagement is None:
self.rManagement = ResourceManagementClient()
self.defaultStatus = defaultStatus
self.rssConfig = RssConfiguration()
self.tokenOwner = "rs_svc"
result = getProxyInfo()
if result['OK']:
self.tokenOwner = result['Value']['username']
def _getCurrentUser(self):
"""Get current user
:return: S_OK(dict)/S_ERROR()
"""
res = getProxyInfo(False, False)
if not res["OK"]:
return self._errorReport("No proxy found in local environment", res["Message"])
proxyInfo = res["Value"]
gLogger.debug(formatProxyInfoAsString(proxyInfo))
if "group" not in proxyInfo:
return self._errorReport("Proxy information does not contain the group", res["Message"])
res = getDNForUsername(proxyInfo["username"])
if not res["OK"]:
return self._errorReport("Failed to get proxies for user", res["Message"])
return S_OK(proxyInfo["username"])
def submitTasks( self ):
""" Submit the tasks to an external system, using the taskManager provided
"""
gLogger.info( "submitTasks: Submitting tasks for transformations" )
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "submitTasks: Failed to determine credentials for submission", res['Message'] )
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
gLogger.info( "submitTasks: Tasks will be submitted with the credentials %s:%s" % ( owner, ownerGroup ) )
# Get the transformations which should be submitted
tasksPerLoop = self.am_getOption( 'TasksPerLoop', 50 )
status = self.am_getOption( 'SubmitStatus', ['Active', 'Completing'] )
res = self._selectTransformations( transType = self.transType, status = status )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
transBody = transformation['Body']
res = self.transClient.getTasksToSubmit( transID, tasksPerLoop )
if not res['OK']:
gLogger.error( "submitTasks: Failed to obtain tasks for transformation", "%s %s" % ( transID, res['Message'] ) )
continue
tasks = res['Value']['JobDictionary']
if not tasks:
gLogger.verbose( "submitTasks: No tasks found for submission for transformation %s" % transID )
continue
gLogger.info( "submitTasks: Obtained %d tasks for submission for transformation %s" % ( len( tasks ), transID ) )
res = self.taskManager.prepareTransformationTasks( transBody, tasks, owner, ownerGroup )
if not res['OK']:
gLogger.error( "submitTasks: Failed to prepare tasks for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
res = self.taskManager.submitTransformationTasks( res['Value'] )
if not res['OK']:
gLogger.error( "submitTasks: Failed to submit prepared tasks for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
res = self.taskManager.updateDBAfterTaskSubmission( res['Value'] )
if not res['OK']:
gLogger.error( "submitTasks: Failed to update DB after task submission for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
gLogger.info( "submitTasks: Submission of transformation tasks complete" )
return S_OK()
def _prepareProxy(self):
"""Set the environment variable X509_USER_PROXY"""
if not self.proxy:
result = getProxyInfo()
if not result["OK"]:
return S_ERROR("No proxy available")
if "path" in result["Value"]:
os.environ["X509_USER_PROXY"] = result["Value"]["path"]
return S_OK()
else:
result = gProxyManager.dumpProxyToFile(self.proxy, requiredTimeLeft=self.minProxyTime)
if not result["OK"]:
return result
os.environ["X509_USER_PROXY"] = result["Value"]
self.log.debug("Set proxy variable X509_USER_PROXY to %s" % os.environ["X509_USER_PROXY"])
return S_OK()
def _getCurrentOwner(self):
"""Simple function to return current DIRAC username.
"""
if 'OwnerName' in self.workflow_commons:
return self.workflow_commons['OwnerName']
result = getProxyInfo()
if not result['OK']:
if not self._enableModule():
return 'testUser'
raise RuntimeError('Could not obtain proxy information')
if 'username' not in result['Value']:
raise RuntimeError('Could not get username from proxy')
return result['Value']['username']
def __init__(self, script=None, stdout='std.out', stderr='std.err'):
"""Instantiates the Workflow object and some default parameters.
"""
super(Job, self).__init__()
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup(ret['Value']['group'])
self.group = vo
self.site = None
# self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
# $DIRACROOT is set by the JobWrapper at execution time
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
# #Add member to handle Parametric jobs
self.numberOfParameters = 0
self.parameterSeqs = {}
self.wfArguments = {}
self.parametricWFArguments = {}
# loading the function that will be used to determine the platform (it can be VO specific)
res = ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", 'getDIRACPlatforms')
if not res['OK']:
self.log.fatal(res['Message'])
self.getDIRACPlatforms = res['Value']
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow(script)
self._siteSet = set(getSites().get('Value', []))
def monitorUninstallation(system, component, cpu=None, hostname=None):
"""
Register the uninstallation of a component in the ComponentMonitoringDB
"""
monitoringClient = ComponentMonitoringClient()
# Retrieve user uninstalling the component
user = None
result = getProxyInfo()
if result["OK"]:
proxyInfo = result["Value"]
if "username" in proxyInfo:
user = proxyInfo["username"]
else:
return result
if not user:
user = "******"
if not cpu:
cpu = "Not available"
for line in open("/proc/cpuinfo"):
if line.startswith("model name"):
cpu = line.split(":")[1][0:64]
cpu = cpu.replace("\n", "").lstrip().rstrip()
if not hostname:
hostname = socket.getfqdn()
instance = component[0:32]
result = monitoringClient.updateInstallations(
{
"Instance": instance,
"UnInstallationTime": None
},
{"DIRACSystem": system},
{
"HostName": hostname,
"CPU": cpu
},
{
"UnInstallationTime": datetime.datetime.utcnow(),
"UnInstalledBy": user
},
)
return result
def submitJob(self, executableFile, proxy, **kwargs):
""" Method to submit job.
"""
self.pPool.processResults()
coresInUse = self.getCoresInUse()
if "WholeNode" in kwargs and kwargs['WholeNode']:
if coresInUse > 0:
return S_ERROR(
'Can not take WholeNode job'
) #, %d/%d slots used' % (self.slotsInUse,self.slots) )
else:
requestedCores = self.cores
elif "NumberOfCores" in kwargs:
requestedCores = int(kwargs['NumberOfCores'])
if requestedCores > 0:
if (coresInUse + requestedCores) > self.cores:
return S_ERROR(
'Not enough slots: requested %d, available %d' %
(requestedCores, self.cores - coresInUse))
else:
requestedCores = 1
if self.cores - coresInUse < requestedCores:
return S_ERROR('Not enough slots: requested %d, available %d' %
(requestedCores, self.cores - coresInUse))
ret = getProxyInfo()
if not ret['OK']:
pilotProxy = None
else:
pilotProxy = ret['Value']['path']
self.log.notice('Pilot Proxy:', pilotProxy)
result = self.pPool.createAndQueueTask(
executeJob, [executableFile, proxy, self.taskID],
None,
self.taskID,
usePoolCallbacks=True)
self.taskID += 1
self.coresPerTask[self.taskID] = requestedCores
self.pPool.processResults()
return result
