def addVOMSExtIfNeeded(self):
addVOMS = self.__piParams.addVOMSExt or Registry.getGroupOption(
self.__piParams.diracGroup, "AutoAddVOMS", False
)
if not addVOMS:
return S_OK()
vomsAttr = Registry.getVOMSAttributeForGroup(self.__piParams.diracGroup)
if not vomsAttr:
return S_ERROR(
"Requested adding a VOMS extension but no VOMS attribute defined for group %s"
% self.__piParams.diracGroup
)
result = VOMS.VOMS().setVOMSAttributes(
self.__proxyGenerated, attribute=vomsAttr, vo=Registry.getVOForGroup(self.__piParams.diracGroup)
)
if not result["OK"]:
return S_ERROR(
"Could not add VOMS extensions to the proxy\nFailed adding VOMS attribute: %s" % result["Message"]
)
gLogger.notice("Added VOMS attribute %s" % vomsAttr)
chain = result["Value"]
chain.dumpAllToFile(self.__proxyGenerated)
return S_OK()
def resetFailedRequest(self, requestID, allR=False):
""" Reset a failed request to "Waiting" status
"""
# # we can safely only peek the request as it is Failed and therefore not owned by an agent
res = self.peekRequest(requestID)
if not res['OK']:
return res
req = res['Value']
if allR or recoverableRequest(req):
# Only reset requests that can be recovered
if req.Status != 'Failed':
gLogger.notice("Reset NotBefore time, was %s" % str(req.NotBefore))
else:
for i, op in enumerate(req):
op.Error = ''
if op.Status == 'Failed':
printOperation((i, op), onlyFailed=True)
for fi in op:
if fi.Status == 'Failed':
fi.Attempt = 1
fi.Error = ''
fi.Status = 'Waiting'
if op.Status == 'Failed':
op.Status = 'Waiting'
# Reset also NotBefore
req.NotBefore = datetime.datetime.utcnow().replace(microsecond=0)
return self.putRequest(req)
return S_OK("Not reset")
def __generateReleaseNotes( self ):
result = self.__loadReleaseNotesFile()
if not result[ 'OK' ]:
return result
releaseData = result[ 'Value' ]
if not releaseData:
gLogger.info( "release.notes not found. Trying to find releasenotes.rst" )
for rstFileName in ( "releasenotes.rst", "releasehistory.rst" ):
result = self.__compileReleaseNotes( rstFileName )
if result[ 'OK' ]:
gLogger.notice( "Compiled %s file!" % rstFileName )
else:
gLogger.warn( result[ 'Message' ] )
return S_OK()
gLogger.info( "Loaded release.notes" )
for rstFileName, singleVersion in ( ( "releasenotes.rst", True ),
( "releasehistory.rst", False ) ):
result = self.__generateRSTFile( releaseData, rstFileName, self.params.version,
singleVersion )
if not result[ 'OK' ]:
gLogger.error( "Could not generate %s: %s" % ( rstFileName, result[ 'Message' ] ) )
continue
result = self.__compileReleaseNotes( rstFileName )
if not result[ 'OK' ]:
gLogger.error( "Could not compile %s: %s" % ( rstFileName, result[ 'Message' ] ) )
continue
gLogger.notice( "Compiled %s file!" % rstFileName )
return S_OK()
def printInfo(self):
result = ProxyInfo.getProxyInfoAsString(self.__proxyGenerated)
if not result["OK"]:
gLogger.error("Failed to get the new proxy info: %s" % result["Message"])
else:
gLogger.notice("Proxy generated:")
gLogger.notice(result["Value"])
if self.__uploadedInfo:
gLogger.notice("\nProxies uploaded:")
maxDNLen = 0
maxGroupLen = 0
for userDN in self.__uploadedInfo:
maxDNLen = max(maxDNLen, len(userDN))
for group in self.__uploadedInfo[userDN]:
maxGroupLen = max(maxGroupLen, len(group))
gLogger.notice(" %s | %s | Until (GMT)" % ("DN".ljust(maxDNLen), "Group".ljust(maxGroupLen)))
for userDN in self.__uploadedInfo:
for group in self.__uploadedInfo[userDN]:
gLogger.notice(
" %s | %s | %s"
% (
userDN.ljust(maxDNLen),
group.ljust(maxGroupLen),
self.__uploadedInfo[userDN][group].strftime("%Y/%m/%d %H:%M"),
)
)
def __checkDestination( self ):
if not self.params.destination:
self.params.destination = tempfile.mkdtemp( 'DIRACTarball' )
gLogger.notice( "Will generate tarball in %s" % self.params.destination )
mkDir(self.params.destination)
return S_OK()
def do_update( self, args ):
""" Update the software on the target host to a given version
usage:
update <version> [ -r <rootPath> ] [ -g <lcgVersion> ]
where rootPath - path to the DIRAC installation
lcgVersion - version of the LCG bindings to install
"""
try:
argss = args.split()
version = argss[0]
rootPath = ''
lcgVersion = ''
del argss[0]
while len( argss ) > 0:
if argss[0] == '-r':
rootPath = argss[1]
del argss[0]
del argss[0]
elif argss[0] == '-g':
lcgVersion = argss[1]
del argss[0]
del argss[0]
except Exception, x:
gLogger.notice( "ERROR: wrong input:", str( x ) )
gLogger.notice( self.do_update.__doc__ )
return
def __setAutoreload( self, filepath = False ):
devLoader = Devloader()
devLoader.bootstrap()
if filepath:
devLoader.watchFile( filepath )
gLogger.notice( "Devloader started" )
return S_OK()
def do_quit( self, args ):
""" Exit the shell.
usage: quit
"""
gLogger.notice( '' )
sys.exit( 0 )
def do_stop( self, args ):
""" Stop services or agents or database server
usage:
stop <system|*> <service|agent|*>
stop mysql
"""
if not args:
gLogger.notice( self.do_stop.__doc__ )
return
argss = args.split()
if argss[0] != 'mysql':
system = argss[0]
if system != '*':
component = argss[1]
else:
component = '*'
client = SystemAdministratorClient( self.host, self.port )
result = client.stopComponent( system, component )
if not result['OK']:
self.__errMsg( result['Message'] )
else:
if system != '*' and component != '*':
gLogger.notice( "\n%s_%s stopped successfully, runit status:\n" % ( system, component ) )
else:
gLogger.notice( "\nComponents stopped successfully, runit status:\n" )
for comp in result['Value']:
gLogger.notice( ( comp.rjust( 32 ), ':', result['Value'][comp]['RunitStatus'] ) )
else:
gLogger.notice( "Not yet implemented" )
def do_execfile( self, args ):
""" Execute a series of administrator CLI commands from a given file
usage:
execfile <filename>
"""
if not args:
gLogger.notice( self.do_execfile.__doc__ )
return
argss = args.split()
fname = argss[0]
execfile = open( fname, 'r' )
lines = execfile.readlines()
execfile.close()
for line in lines:
if line.find( '#' ) != -1 :
line = line[:line.find( '#' )]
line = line.strip()
if not line:
continue
gLogger.notice( "\n--> Executing %s\n" % line )
elements = line.split()
command = elements[0]
args = ' '.join( elements[1:] )
eval( "self.do_%s(args)" % command )
def __do_set_project( self, args ):
project = args[0]
result = self.__getClient().setProject( project )
if not result[ 'OK' ]:
self.__errMsg( "Cannot set project: %s" % result[ 'Message' ] )
else:
gLogger.notice( "Project set to %s" % project )
def uploadProxy( self, userGroup = False ):
issuerCert = self.getIssuerCert()
userDN = issuerCert.getSubjectDN()[ 'Value' ]
if not userGroup:
userGroup = self.__piParams.diracGroup
gLogger.notice( "Uploading proxy for %s..." % userGroup )
if userGroup in self.__proxiesUploaded:
gLogger.info( "Proxy already uploaded" )
return S_OK()
if userDN in self.__uploadedInfo:
expiry = self.__uploadedInfo[ userDN ].get( userGroup )
if expiry:
if issuerCert.getNotAfterDate()[ 'Value' ] - datetime.timedelta( minutes = 10 ) < expiry:
gLogger.info( "SKipping upload for group %s. Already uploaded" % userGroup )
return S_OK()
gLogger.info( "Uploading %s proxy to ProxyManager..." % self.__piParams.diracGroup )
upParams = ProxyUpload.CLIParams()
upParams.onTheFly = True
upParams.proxyLifeTime = issuerCert.getRemainingSecs()[ 'Value' ] - 300
upParams.diracGroup = userGroup
for k in ( 'certLoc', 'keyLoc', 'userPasswd' ):
setattr( upParams, k , getattr( self.__piParams, k ) )
result = ProxyUpload.uploadProxy( upParams )
if not result[ 'OK' ]:
gLogger.error( result[ 'Message' ] )
sys.exit( 1 )
self.__uploadedInfo = result[ 'Value' ]
self.__proxiesUploaded.append( userGroup )
gLogger.info( "Proxy uploaded" )
return S_OK()
def insertCSSection( self, path, pardict ):
""" insert a section and values (or subsections) into the CS
:param str path: full path of the new section
:param str pardict: dictionary of key values in the new section, values can also be dictionaries
:return: S_OK(), S_ERROR()
"""
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
if self.csapi is None:
self.csapi = CSAPI()
for key, value in pardict.iteritems():
newSectionPath = os.path.join(path,key)
gLogger.debug( "Adding to cs %s : %s " % ( newSectionPath, value ) )
self.csapi.createSection( path )
if isinstance( value, dict ):
res = self.insertCSSection( newSectionPath, value )
else:
res = self.csapi.setOption( newSectionPath, value )
if not res['OK']:
return res
else:
gLogger.notice( "Added to CS: %s " % res['Value'] )
return S_OK("Added all things to cs")
def insertApplicationToCS(self, name, csParameter):
"""add given application found via CVMFS to the CS"""
pars = dict(self.parameter)
pars['name'] = name
gLogger.notice("%(name)s: Adding version %(version)s to the CS" % pars)
existingVersions = gConfig.getSections("%(softSec)s/%(platform)s/%(name)s" % pars, [])
if not existingVersions['OK']:
gLogger.error("Could not find all versions available in CS: %s" % existingVersions['Message'])
dexit(255)
if pars['version'] in existingVersions['Value']:
gLogger.always('Application %s %s for %s already in CS, nothing to do' % (name.lower(),
pars['version'],
pars['platform']))
return S_OK()
csPath = self.softSec + ("/%(platform)s/%(name)s/%(version)s/" % pars)
for par, val in csParameter.iteritems():
gLogger.notice("Add: %s = %s" %(csPath+par, val))
result = self.csAPI.setOption(csPath+par, val)
if result['OK']:
self.modifiedCS = True
else:
gLogger.error("Failure to add to CS", result['Message'])
return S_ERROR("")
return S_OK()
def __moduleProcess( self, eType, taskId, taskStub, fastTrackLevel = 0 ):
result = self.__getInstance( eType )
if not result[ 'OK' ]:
return result
modInstance = result[ 'Value' ]
try:
result = modInstance._ex_processTask( taskId, taskStub )
except Exception as excp:
gLogger.exception( "Error while processing task %s" % taskId, lException = excp )
return S_ERROR( "Error processing task %s: %s" % ( taskId, excp ) )
self.__storeInstance( eType, modInstance )
if not result[ 'OK' ]:
return S_OK( ( 'TaskError', taskStub, "Error: %s" % result[ 'Message' ] ) )
taskStub, freezeTime, fastTrackType = result[ 'Value' ]
if freezeTime:
return S_OK( ( "TaskFreeze", taskStub, freezeTime ) )
if fastTrackType:
if fastTrackLevel < 10 and fastTrackType in self.__modules:
gLogger.notice( "Fast tracking task %s to %s" % ( taskId, fastTrackType ) )
return self.__moduleProcess( fastTrackType, taskId, taskStub, fastTrackLevel + 1 )
else:
gLogger.notice( "Stopping %s fast track. Sending back to the mind" % ( taskId ) )
return S_OK( ( "TaskDone", taskStub, True ) )
def sync(self):
""" Main synchronizer method.
"""
ops = Operations()
self.pilotFileServer = ops.getValue("Pilot/pilotFileServer", self.pilotFileServer)
if not self.pilotFileServer:
gLogger.warn("The /Operations/<Setup>/Pilot/pilotFileServer option is not defined")
gLogger.warn("Pilot 3 files won't be updated, and you won't be able to use Pilot 3")
gLogger.warn("The Synchronization steps are anyway displayed")
gLogger.notice('-- Synchronizing the content of the JSON file %s with the content of the CS --' % self.jsonFile)
self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo)
self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo)
self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir)
self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath", self.pilotScriptPath)
self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath", self.pilotVOScriptPath)
result = self._syncJSONFile()
if not result['OK']:
gLogger.error("Error uploading the pilot file: %s" % result['Message'])
return result
gLogger.notice('-- Synchronizing the pilot scripts %s with the content of the repository --' % self.pilotRepo)
self._syncScripts()
return S_OK()
def updateDelayCounters( self, siteName, jid ):
#Get the info from the CS
siteSection = "%s/%s" % ( self.__matchingDelaySection, siteName )
result = self.__extractCSData( siteSection )
if not result['OK']:
return result
delayDict = result[ 'Value' ]
#limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
if not delayDict:
return S_OK()
attNames = []
for attName in delayDict:
if attName not in gJobDB.jobAttributeNames:
gLogger.error( "Attribute %s does not exist in the JobDB. Please fix it!" % attName )
else:
attNames.append( attName )
result = gJobDB.getJobAttributes( jid, attNames )
if not result[ 'OK' ]:
gLogger.error( "While retrieving attributes coming from %s: %s" % ( siteSection, result[ 'Message' ] ) )
return result
atts = result[ 'Value' ]
#Create the DictCache if not there
if siteName not in Limiter.__delayMem:
Limiter.__delayMem[ siteName ] = DictCache()
#Update the counters
delayCounter = Limiter.__delayMem[ siteName ]
for attName in atts:
attValue = atts[ attName ]
if attValue in delayDict[ attName ]:
delayTime = delayDict[ attName ][ attValue ]
gLogger.notice( "Adding delay for %s/%s=%s of %s secs" % ( siteName, attName,
attValue, delayTime ) )
delayCounter.add( ( attName, attValue ), delayTime )
return S_OK()
def notifyAboutNewSoftware(self):
"""Send an email to the mailing list if a new software version was defined"""
#Only send email when something was actually added
if not self.modifiedCS:
return
subject = '%s %s added to DIRAC CS' % (self.appName, self.appVersion)
msg = 'New application %s %s declared into Configuration service\n %s' % (self.appName,
self.appVersion,
self.comment)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUserOption
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
notifyClient = NotificationClient()
gLogger.notice('Sending mail for software installation to %s' % (self.mailadress))
res = getProxyInfo()
if not res['OK']:
sender = '*****@*****.**'
else:
if 'username' in res['Value']:
sender = getUserOption(res['Value']['username'],'Email')
else:
sender = '*****@*****.**'
gLogger.info('*'*80)# surround email with stars
res = notifyClient.sendMail(self.mailadress, subject, msg, sender, localAttempt = False)
gLogger.info('*'*80)
if not res[ 'OK' ]:
gLogger.error('The mail could not be sent: %s' % res['Message'])
def addCommentToCS(self):
"""adds the comment for the TarBall to the CS"""
gLogger.notice("Adding comment to CS: %s" % self.comment)
result = self.diracAdmin.csSetOptionComment("%(softSec)s/%(platform)s/%(appname)s/%(appVersion)s/TarBall"% self.parameter,
self.comment)
if not result['OK']:
gLogger.error("Error setting comment in CS")
def __checkoutFromGit(self):
if self.params.vcsBranch:
brCmr = "-b %s" % self.params.vcsBranch
else:
brCmr = ""
fDirName = os.path.join(self.params.destination, self.params.name)
cmd = "git clone %s '%s' '%s'" % (brCmr, self.params.sourceURL, fDirName)
gLogger.verbose("Executing: %s" % cmd)
if os.system(cmd):
return S_ERROR("Error while retrieving sources from git")
branchName = "DIRACDistribution-%s" % os.getpid()
isTagCmd = "( cd '%s'; git tag -l | grep '%s' )" % (fDirName, self.params.version)
if os.system(isTagCmd):
# No tag found, assume branch
branchSource = "origin/%s" % self.params.version
else:
branchSource = self.params.version
cmd = "( cd '%s'; git checkout -b '%s' '%s' )" % (fDirName, branchName, branchSource)
gLogger.verbose("Executing: %s" % cmd)
exportRes = os.system(cmd)
# Add the keyword substitution
gLogger.notice("Replacing keywords (can take a while)...")
self.replaceKeywordsWithGit(fDirName)
shutil.rmtree("%s/.git" % fDirName)
if exportRes:
return S_ERROR("Error while exporting from git")
return S_OK()
def initialize( self ):
""" service initialisation
:param self: self reference
"""
gLogger.notice( "CacheDirectory: %s" % self.cacheDir() )
return S_OK()
def checkCAs( self ):
if not "X509_CERT_DIR" in os.environ:
gLogger.warn( "X509_CERT_DIR is unset. Abort check of CAs" )
return
caDir = os.environ[ "X509_CERT_DIR" ]
# In globus standards .r0 files are CRLs. They have the same names of the CAs but diffent file extension
searchExp = os.path.join( caDir, "*.r0" )
crlList = glob.glob( searchExp )
if not crlList:
gLogger.warn( "No CRL files found for %s. Abort check of CAs" % searchExp )
return
newestFPath = max( crlList, key=os.path.getmtime )
newestFTime = os.path.getmtime( newestFPath )
if newestFTime > ( time.time() - ( 2 * 24 * 3600 ) ):
# At least one of the files has been updated in the last 2 days
return S_OK()
if not os.access(caDir, os.W_OK):
gLogger.error("Your CRLs appear to be outdated, but you have no access to update them.")
# Try to continue anyway...
return S_OK()
# Update the CAs & CRLs
gLogger.notice( "Your CRLs appear to be outdated; attempting to update them..." )
bdc = BundleDeliveryClient()
res = bdc.syncCAs()
if not res[ 'OK' ]:
gLogger.error( "Failed to update CAs", res[ 'Message' ] )
res = bdc.syncCRLs()
if not res[ 'OK' ]:
gLogger.error( "Failed to update CRLs", res[ 'Message' ] )
# Continue even if the update failed...
return S_OK()
def convertCatalogs():
global csapi, defaultSite
gLogger.notice( 'Converting Catalog services' )
result = gConfig.getSections('/Resources/FileCatalogs')
if not result['OK']:
gLogger.error(result['Message'])
return result
catalogs = result['Value']
for catalog in catalogs:
gLogger.notice( 'Processing catalog %s' % catalog )
if defaultSite:
inputSite = raw_input('Hosting site for %s [%s]: ' % (catalog, defaultSite) )
if not inputSite:
site = defaultSite
else:
site = inputSite
else:
inputSite = raw_input('Hosting site for %s: ' % catalog )
if not inputSite:
gLogger.error(result['Message'])
return
site = inputSite
result = csapi.copySection( '/Resources/FileCatalogs/%s' % catalog, '%s/Sites/%s/Catalog/%s' % (RESOURCES_NEW_SECTION,site,catalog) )
if not result['OK']:
gLogger.error(result['Message'])
return result
csapi.setOptionComment( '%s/Sites/%s/Catalog' % (RESOURCES_NEW_SECTION,site), 'Catalog resources' )
return S_OK()
def addUserToEgroup(clip):
"""Add user to e-group"""
login = gConfig.getValue("/Security/egroupAdmin","").strip('"')
pwd = gConfig.getValue("/Security/egroupPass","").strip('"')
url = 'https://foundservices.cern.ch/ws/egroups/v1/EgroupsWebService/EgroupsWebService.wsdl'
if not ( login and pwd ):
gLogger.warn("Missing configuration parameters: username or password for WSDL interactions")
gLogger.warn("Add options: -o /Security/egroupAdmin=<cernusername> -o /Security/egroupPass=<password>")
gLogger.error("User registration in e-group must be done manually")
return
try:
client = Client(url=url, username=login, password=pwd)
#gLogger.notice(client)
except suds.transport.TransportError as exc:
gLogger.error("Failed to get the WSDL client:%s" %exc)
gLogger.error("User registration in e-group must be done manually")
return
except:
gLogger.error("Something unexpected happened with the suds client, aborting")
return
if clip.external:
sudsUser = client.factory.create("ns0:MemberType")
sudsUser['Type'] = 'External'
sudsUser['Email'] = clip.email
userl = [sudsUser]
else:
user = getUserInfoFromPhonebook(client, clip)
userl = [user]
res = client.service.AddEgroupMembers('ilc-dirac',False, userl)
if hasattr(res, 'warnings'):
gLogger.notice(res.warnings)
def add(self, section, option, new_value):
"""
Add a value into the configuration system.
This method will overwrite any existing option's value.
Args:
section (str): The section
option (str): The option to be created/modified
new_value: The value to be assigned
Example:
>>> cs = ConfigurationSystem()
>>> cs.add('/Registry', 'DefaultGroup', 'dteam_user')
"""
if isinstance(new_value, (tuple, list, set, GeneratorType)):
new_value = ', '.join(sorted(map(str, new_value)))
else:
new_value = str(new_value)
old_value = gConfig.getValue(cfgPath(section, option), None)
if old_value == new_value:
return
if old_value is None:
gLogger.notice("Setting %s/%s: -> %s"
% (section, option, new_value))
self.setOption(cfgPath(section, option), new_value)
else:
gLogger.notice("Modifying %s/%s: %s -> %s"
% (section, option, old_value, new_value))
self.modifyValue(cfgPath(section, option), new_value)
self._num_changes += 1
def _getProdInfoFromIDs(self):
"""get the processName, energy and eventsPerJob from the MetaData catalog
:raises: AttributeError if some of the information cannot be found
:returns: None
"""
if not self.prodIDs:
raise AttributeError("No prodIDs defined")
self.eventsPerJobs = []
self.processes = []
self.energies = []
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
trc = TransformationClient()
fc = FileCatalogClient()
for prodID in self.prodIDs:
gLogger.notice("Getting information for %s" % prodID)
tRes = trc.getTransformation(str(prodID))
if not tRes['OK']:
raise AttributeError("No prodInfo found for %s" % prodID)
self.eventsPerJobs.append(int(tRes['Value']['EventsPerTask']))
lfnRes = fc.findFilesByMetadata({'ProdID': prodID})
if not lfnRes['OK'] or not lfnRes['Value']:
raise AttributeError("Could not find files for %s: %s " % (prodID, lfnRes.get('Message', lfnRes.get('Value'))))
path = os.path.dirname(lfnRes['Value'][0])
fileRes = fc.getDirectoryUserMetadata(path)
self.processes.append(fileRes['Value']['EvtType'])
self.energies.append(fileRes['Value']['Energy'])
gLogger.notice("Found (Evts,Type,Energy): %s %s %s " %
(self.eventsPerJobs[-1], self.processes[-1], self.energies[-1]))
def createTransformations(self, taskDict):
"""Create all the transformations we want to create."""
for pType, createProduction in [('GEN', self.createGenerationProduction),
('SPLIT', self.createSplitProduction)]:
for task in taskDict.get(pType, []):
meta = createProduction(task)
self.addSimTask(taskDict, meta, originalTask=task)
taskDict['MOVE_' + pType].append(dict(meta))
for task in taskDict.get('SIM', []):
if not self._flags.sim:
continue
gLogger.notice("Creating task %s" % task)
simMeta = self.createSimulationProduction(task)
self.addRecTask(taskDict, simMeta, originalTask=task)
taskDict['MOVE_SIM'].append(dict(simMeta))
for task in taskDict.get('REC', []):
for name, over, enabled in [('REC', False, self._flags.rec),
('OVER', True, self._flags.over)]:
if enabled:
recMeta = self.createReconstructionProduction(task, over=over)
taskDict['MOVE_' + name].append(dict(recMeta))
for name, pType in [('GEN', 'MCGeneration'),
('SPLIT', 'MCGeneration'),
('SIM', 'MCSimulation'),
('REC', 'MCReconstruction'),
('OVER', 'MCReconstruction_Overlay')]:
for meta in taskDict.get('MOVE_' + name, []):
self.createMovingTransformation(meta, pType)
def uploadProxy( self, userGroup = False ):
""" Upload the proxy to the proxyManager service
"""
issuerCert = self.getIssuerCert()
resultUserDN = issuerCert.getSubjectDN() #pylint: disable=no-member
if not resultUserDN['OK']:
return resultUserDN
userDN = resultUserDN['Value']
if not userGroup:
userGroup = self.__piParams.diracGroup
gLogger.notice( "Uploading proxy for %s..." % userGroup )
if userDN in self.__uploadedInfo:
expiry = self.__uploadedInfo[ userDN ].get( userGroup )
if expiry:
if issuerCert.getNotAfterDate()[ 'Value' ] - datetime.timedelta( minutes = 10 ) < expiry: #pylint: disable=no-member
gLogger.info( "SKipping upload for group %s. Already uploaded" % userGroup )
return S_OK()
gLogger.info( "Uploading %s proxy to ProxyManager..." % self.__piParams.diracGroup )
upParams = ProxyUpload.CLIParams()
upParams.onTheFly = True
upParams.proxyLifeTime = issuerCert.getRemainingSecs()[ 'Value' ] - 300 #pylint: disable=no-member
upParams.rfcIfPossible = self.__piParams.rfc
upParams.diracGroup = userGroup
for k in ( 'certLoc', 'keyLoc', 'userPasswd' ):
setattr( upParams, k , getattr( self.__piParams, k ) )
resultProxyUpload = ProxyUpload.uploadProxy( upParams )
if not resultProxyUpload[ 'OK' ]:
gLogger.error( resultProxyUpload[ 'Message' ] )
sys.exit( 1 )
self.__uploadedInfo = resultProxyUpload[ 'Value' ]
gLogger.info( "Proxy uploaded" )
return S_OK()
def _addTask(self, taskDict, metaInput, originalTask, prodType, applicationName):
"""Add a task to the given prodType and applicatioName."""
options = defaultdict(list)
nTasks = 0
for option, value in self.applicationOptions[applicationName].items():
if option.startswith('FE.'):
optionName = option.split('.', 1)[1]
options[optionName] = listify(value)
gLogger.notice("Found option %s with values %s" % (optionName, pformat(options[optionName])))
nTasks = len(options[optionName])
theTask = Task(metaInput,
parameterDict=originalTask.parameterDict,
eventsPerJob=originalTask.eventsPerJob,
metaPrev=originalTask.meta,
dryRun=self._flags.dryRun,
sinFile=originalTask.sinFile,
nbTasks=originalTask.nbTasks,
)
theTask.sourceName = originalTask.taskName
if not nTasks:
taskDict[prodType].append(theTask)
return
taskList = [deepcopy(theTask) for _ in xrange(nTasks)]
taskDict[prodType].extend(taskList)
self.addTaskOptions(options, taskList)
return
def doTheMagic( self ):
proxy = self.createProxy()
if not proxy[ 'OK' ]:
return proxy
self.checkCAs()
pI.certLifeTimeCheck()
resultProxyWithVOMS = pI.addVOMSExtIfNeeded()
if not resultProxyWithVOMS[ 'OK' ]:
if "returning a valid AC for the user" in resultProxyWithVOMS['Message']:
gLogger.error( resultProxyWithVOMS[ 'Message' ] )
gLogger.error("\n Are you sure you are properly registered in the VO?")
elif "Missing voms-proxy" in resultProxyWithVOMS['Message']:
gLogger.notice( "Failed to add VOMS extension: no standard grid interface available" )
else:
gLogger.error( resultProxyWithVOMS['Message'] )
if self.__piParams.strict:
return resultProxyWithVOMS
for pilotGroup in pI.getGroupsToUpload():
resultProxyUpload = pI.uploadProxy( userGroup = pilotGroup )
if not resultProxyUpload[ 'OK' ]:
if self.__piParams.strict:
return resultProxyUpload
return S_OK()
def do_show(self, args):
"""
Show list of components with various related information
usage:
show software - show components for which software is available
show installed - show components installed in the host with runit system
show setup - show components set up for automatic running in the host
show project - show project to install or upgrade
show status - show status of the installed components
show database - show status of the databases
show mysql - show status of the MySQL server
show log <system> <service|agent> [nlines]
- show last <nlines> lines in the component log file
show info - show version of software and setup
show doc <type> <system> <name>
- show documentation for a given service or agent
show host - show host related parameters
show hosts - show all available hosts
show ports [host] - show all ports used by a host. If no host is given, the host currently connected to is used
show installations [ list | current | -n <Name> | -h <Host> | -s <System> | -m <Module> | -t <Type> | -itb <InstallationTime before>
| -ita <InstallationTime after> | -utb <UnInstallationTime before> | -uta <UnInstallationTime after> ]*
- show all the installations of components that match the given parameters
show profile <system> <component> [ -s <size> | -h <host> | -id <initial date DD/MM/YYYY> | -it <initial time hh:mm>
| -ed <end date DD/MM/YYYY | -et <end time hh:mm> ]*
- show <size> log lines of profiling information for a component in the machine <host>
show errors [*|<system> <service|agent>]
- show error count for the given component or all the components
in the last hour and day
"""
argss = args.split()
if not argss:
gLogger.notice(self.do_show.__doc__)
return
option = argss[0]
del argss[0]
if option == 'software':
client = SystemAdministratorClient(self.host, self.port)
result = client.getSoftwareComponents()
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice('')
pprint.pprint(result['Value'])
elif option == 'installed':
client = SystemAdministratorClient(self.host, self.port)
result = client.getInstalledComponents()
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice('')
pprint.pprint(result['Value'])
elif option == 'setup':
client = SystemAdministratorClient(self.host, self.port)
result = client.getSetupComponents()
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice('')
pprint.pprint(result['Value'])
elif option == 'project':
result = SystemAdministratorClient(self.host, self.port).getProject()
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice("Current project is %s" % result['Value'])
elif option == 'status':
client = SystemAdministratorClient(self.host, self.port)
result = client.getOverallStatus()
if not result['OK']:
self._errMsg(result['Message'])
else:
fields = ["System", 'Name', 'Module', 'Type', 'Setup', 'Installed', 'Runit', 'Uptime', 'PID']
records = []
rDict = result['Value']
for compType in rDict:
for system in rDict[compType]:
components = sorted(rDict[compType][system].keys())
for component in components:
record = []
if rDict[compType][system][component]['Installed']:
module = str(rDict[compType][system][component]['Module'])
record += [system, component, module, compType.lower()[:-1]]
if rDict[compType][system][component]['Setup']:
record += ['Setup']
else:
record += ['NotSetup']
if rDict[compType][system][component]['Installed']:
record += ['Installed']
else:
record += ['NotInstalled']
record += [str(rDict[compType][system][component]['RunitStatus'])]
record += [str(rDict[compType][system][component]['Timeup'])]
record += [str(rDict[compType][system][component]['PID'])]
records.append(record)
printTable(fields, records)
elif option == 'database' or option == 'databases':
client = SystemAdministratorClient(self.host, self.port)
if not gComponentInstaller.mysqlPassword:
gComponentInstaller.mysqlPassword = "******"
gComponentInstaller.getMySQLPasswords()
result = client.getDatabases(gComponentInstaller.mysqlRootPwd)
if not result['OK']:
self._errMsg(result['Message'])
return
resultSW = client.getAvailableDatabases()
if not resultSW['OK']:
self._errMsg(resultSW['Message'])
return
sw = resultSW['Value']
installed = result['Value']
gLogger.notice('')
for db in sw:
if db in installed:
gLogger.notice(db.rjust(25), ': Installed')
else:
gLogger.notice(db.rjust(25), ': Not installed')
if not sw:
gLogger.notice("No database found")
elif option == 'mysql':
client = SystemAdministratorClient(self.host, self.port)
result = client.getMySQLStatus()
if not result['OK']:
self._errMsg(result['Message'])
elif result['Value']:
gLogger.notice('')
for par, value in result['Value'].items():
gLogger.notice((par.rjust(28), ':', value))
else:
gLogger.notice("No MySQL database found")
elif option == "log":
self.getLog(argss)
elif option == "info":
client = SystemAdministratorClient(self.host, self.port)
result = client.getInfo()
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice('')
gLogger.notice("Setup:", result['Value']['Setup'])
gLogger.notice("DIRAC version:", result['Value']['DIRAC'])
if result['Value']['Extensions']:
for e, v in result['Value']['Extensions'].items():
gLogger.notice("%s version" % e, v)
gLogger.notice('')
elif option == "host":
client = SystemAdministratorClient(self.host, self.port)
result = client.getHostInfo()
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice('')
gLogger.notice("Host info:")
gLogger.notice('')
fields = ['Parameter', 'Value']
records = []
for parameter in result['Value'].iteritems():
if parameter[0] == 'Extension':
extensions = parameter[1].split(',')
for extension in extensions:
extensionName, extensionVersion = extension.split(':')
records.append(['%sVersion' % extensionName, str(extensionVersion)])
else:
records.append([parameter[0], str(parameter[1])])
printTable(fields, records)
elif option == "hosts":
client = ComponentMonitoringClient()
result = client.getHosts({}, False, False)
if not result['OK']:
self._errMsg('Error retrieving the list of hosts: %s' % (result['Message']))
else:
hostList = result['Value']
gLogger.notice('')
gLogger.notice(' ' + 'Host'.center(32) + ' ' + 'CPU'.center(34) + ' ')
gLogger.notice(('-' * 69))
for element in hostList:
gLogger.notice('|' + element['HostName'].center(32) + '|' + element['CPU'].center(34) + '|')
gLogger.notice(('-' * 69))
gLogger.notice('')
elif option == "ports":
if not argss:
client = SystemAdministratorClient(self.host)
else:
hostname = argss[0]
del argss[0]
client = ComponentMonitoringClient()
result = client.hostExists({'HostName': hostname})
if not result['OK']:
self._errMsg(result['Message'])
return
else:
if not result['Value']:
self._errMsg('Given host does not exist')
return
client = SystemAdministratorClient(hostname)
result = client.getUsedPorts()
if not result['OK']:
self._errMsg(result['Message'])
return
pprint.pprint(result['Value'])
elif option == "errors":
self.getErrors(argss)
elif option == "installations":
self.getInstallations(argss)
elif option == "doc":
if len(argss) > 2:
if argss[0] in ['service', 'agent']:
compType = argss[0]
compSystem = argss[1]
compModule = argss[2]
client = SystemAdministratorClient(self.host, self.port)
result = client.getComponentDocumentation(compType, compSystem, compModule)
if result['OK']:
gLogger.notice(result['Value'])
else:
self._errMsg(result['Message'])
else:
gLogger.notice(self.do_show.__doc__)
else:
gLogger.notice(self.do_show.__doc__)
elif option == "profile":
if len(argss) > 1:
system = argss[0]
del argss[0]
component = argss[0]
del argss[0]
component = '%s_%s' % (system, component)
argDict = {'-s': None, '-h': self.host, '-id': None, '-it': '00:00', '-ed': None, '-et': '00:00'}
key = None
for arg in argss:
if not key:
key = arg
else:
argDict[key] = arg
key = None
size = None
try:
if argDict['-s']:
size = int(argDict['-s'])
except ValueError:
self._errMsg("Argument 'size' must be an integer")
return
host = argDict['-h']
initialDate = argDict['-id']
initialTime = argDict['-it']
endingDate = argDict['-ed']
endingTime = argDict['-et']
if initialDate:
initialDate = '%s %s' % (initialDate, initialTime)
else:
initialDate = ''
if endingDate:
endingDate = '%s %s' % (endingDate, endingTime)
else:
endingDate = ''
client = MonitoringClient()
if size:
result = client.getLimitedData(host, component, size)
else:
result = client.getDataForAGivenPeriod(host, component, initialDate, endingDate)
if result['OK']:
text = ''
headers = [result['Value'][0].keys()]
for header in headers:
text += str(header).ljust(15)
gLogger.notice(text)
for record in result['Value']:
for metric in record.itervalues():
text += str(metric).ljust(15)
gLogger.notice(text)
else:
self._errMsg(result['Message'])
else:
gLogger.notice(self.do_show.__doc__)
else:
gLogger.notice("Unknown option:", option)
gLogger.verbose("%s does not have requirements for %s installation" % (entry, instType))
continue
for req in reqList:
reqName = False
reqCond = ""
for cond in ("==", ">="):
iP = cond.find(req)
if iP > 0:
reqName = req[:iP]
reqCond = req[iP:]
break
if not reqName:
reqName = req
if reqName not in reqDict:
reqDict[reqName] = (reqCond, entry)
else:
gLogger.notice("Skipping %s, it's already requested by %s" % (reqName, reqDict[reqName][1]))
if not reqDict:
gLogger.notice("No extra python module requested to be installed")
sys.exit(0)
for reqName in reqDict:
package = "%s%s" % (reqName, reqDict[reqName][0])
gLogger.notice("Requesting installation of %s" % package)
status, output = pipInstall(package)
if status != 0:
gLogger.error(output)
else:
gLogger.notice("Successfully installed %s" % package)
res = resourceStatus.getStorageElementStatus(ses)
if not res['OK']:
gLogger.error("Storage Element %s does not exist" % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-ban-se by %s' % userName
for se, seOptions in res['Value'].items():
resW = resC = resR = {'OK': False}
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if read and seOptions.has_key('Read'):
if not seOptions['Read'] in ['Active', 'Bad']:
gLogger.notice('Read option for %s is %s, instead of %s' %
(se, seOptions['Read'], ['Active', 'Bad']))
gLogger.notice('Try specifying the command switchs')
continue
resR = resourceStatus.setStorageElementStatus(se, 'Read', 'Banned',
reason, userName)
#res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" )
if not resR['OK']:
gLogger.error('Failed to update %s read access to Banned' % se)
else:
gLogger.notice('Successfully updated %s read access to Banned' %
se)
readBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if write and seOptions.has_key('Write'):
def do_restart(self, args):
""" Restart services or agents or database server
usage:
restart <system|*> <service|agent|*>
restart mysql
"""
if not args:
gLogger.notice(self.do_restart.__doc__)
return
argss = args.split()
option = argss[0]
del argss[0]
if option != 'mysql':
if option != "*":
if len(argss) < 1:
gLogger.notice(self.do_restart.__doc__)
return
system = option
if system != '*':
component = argss[0]
else:
component = '*'
client = SystemAdministratorClient(self.host, self.port)
result = client.restartComponent(system, component)
if not result['OK']:
if system == '*':
gLogger.notice("All systems are restarted, connection to SystemAdministrator is lost")
else:
self._errMsg(result['Message'])
else:
if system != '*' and component != '*':
gLogger.notice("\n%s_%s started successfully, runit status:\n" % (system, component))
else:
gLogger.notice("\nComponents started successfully, runit status:\n")
for comp in result['Value']:
gLogger.notice((comp.rjust(32), ':', result['Value'][comp]['RunitStatus']))
else:
gLogger.notice("Not yet implemented")
def do_uninstall(self, args):
"""
Uninstall a DIRAC component or host along with all its components
usage:
uninstall db <database>
uninstall host <hostname>
uninstall <-f ForceLogUninstall> <system> <component>
"""
argss = args.split()
if not argss:
gLogger.notice(self.do_uninstall.__doc__)
return
# Retrieve user uninstalling the component
result = getProxyInfo()
if not result['OK']:
self._errMsg(result['Message'])
user = result['Value']['username']
option = argss[0]
if option == 'db':
del argss[0]
if not argss:
gLogger.notice(self.do_uninstall.__doc__)
return
component = argss[0]
client = SystemAdministratorClient(self.host, self.port)
result = client.getHostInfo()
if not result['OK']:
self._errMsg(result['Message'])
return
else:
cpu = result['Value']['CPUModel']
hostname = self.host
result = client.getAvailableDatabases()
if not result['OK']:
self._errMsg(result['Message'])
return
system = result['Value'][component]['System']
result = MonitoringUtilities.monitorUninstallation(system, component, hostname=hostname, cpu=cpu)
if not result['OK']:
self._errMsg(result['Message'])
return
result = client.uninstallDatabase(component)
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice("Successfully uninstalled %s" % (component))
elif option == 'host':
del argss[0]
if not argss:
gLogger.notice(self.do_uninstall.__doc__)
return
hostname = argss[0]
client = ComponentMonitoringClient()
result = client.hostExists({'HostName': hostname})
if not result['OK']:
self._errMsg(result['Message'])
else:
if not result['Value']:
self._errMsg('Given host does not exist')
else:
result = client.getHosts({'HostName': hostname}, True, False)
if not result['OK']:
self._errMsg(result['Message'])
else:
host = result['Value'][0]
# Remove every installation associated with the host
for installation in host['Installations']:
result = client.removeInstallations(installation, {}, {'HostName': hostname})
if not result['OK']:
self._errMsg(result['Message'])
break
# Finally remove the host
result = client.removeHosts({'HostName': hostname})
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice('Host %s was successfully removed' % hostname)
else:
if option == '-f':
force = True
del argss[0]
else:
force = False
if len(argss) != 2:
gLogger.notice(self.do_uninstall.__doc__)
return
system, component = argss
client = SystemAdministratorClient(self.host, self.port)
monitoringClient = ComponentMonitoringClient()
result = monitoringClient.getInstallations({'Instance': component, 'UnInstallationTime': None},
{'System': system},
{'HostName': self.host}, True)
if not result['OK']:
self._errMsg(result['Message'])
return
if len(result['Value']) < 1:
self._errMsg("Given component does not exist")
return
if len(result['Value']) > 1:
self._errMsg("Too many components match")
return
removeLogs = False
if force:
removeLogs = True
else:
if result['Value'][0]['Component']['Type'] in self.runitComponents:
result = promptUser('Remove logs?', ['y', 'n'], 'n')
if result['OK']:
removeLogs = result['Value'] == 'y'
result = client.uninstallComponent(system, component, removeLogs)
if not result['OK']:
self._errMsg(result['Message'])
else:
gLogger.notice("Successfully uninstalled %s/%s" % (system, component))
result = client.getHostInfo()
if not result['OK']:
self._errMsg(result['Message'])
return
else:
cpu = result['Value']['CPUModel']
hostname = self.host
result = MonitoringUtilities.monitorUninstallation(system, component, hostname=hostname, cpu=cpu)
if not result['OK']:
return result
def do_install(self, args):
"""
Install various DIRAC components
usage:
install db <databaseName>
install service <system> <service> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install agent <system> <agent> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install executor <system> <executor> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
"""
argss = args.split()
if not argss:
gLogger.notice(self.do_install.__doc__)
return
option = argss[0]
del argss[0]
# Databases
if option == "db":
if not argss:
gLogger.notice(self.do_install.__doc__)
return
database = argss[0]
client = SystemAdministratorClient(self.host, self.port)
result = client.getAvailableDatabases()
if not result['OK']:
self._errMsg("Can not get database list: %s" % result['Message'])
return
if database not in result['Value']:
self._errMsg("Unknown database %s: " % database)
return
system = result['Value'][database]['System']
dbType = result['Value'][database]['Type']
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
self._errMsg("Unknown current setup")
return
instance = gConfig.getValue('/DIRAC/Setups/%s/%s' % (setup, system), '')
if not instance:
self._errMsg("No instance defined for system %s" % system)
self._errMsg("\tAdd new instance with 'add instance %s <instance_name>'" % system)
return
if dbType == 'MySQL':
if not gComponentInstaller.mysqlPassword:
gComponentInstaller.mysqlPassword = '******'
gComponentInstaller.getMySQLPasswords()
result = client.installDatabase(database, gComponentInstaller.mysqlRootPwd)
if not result['OK']:
self._errMsg(result['Message'])
return
extension, system = result['Value']
result = client.getHostInfo()
if not result['OK']:
self._errMsg(result['Message'])
return
else:
cpu = result['Value']['CPUModel']
hostname = self.host
if not result['OK']:
self._errMsg(result['Message'])
return
if database != 'InstalledComponentsDB':
result = MonitoringUtilities.monitorInstallation(
'DB', system.replace('System', ''), database, cpu=cpu, hostname=hostname)
if not result['OK']:
self._errMsg(result['Message'])
return
gComponentInstaller.mysqlHost = self.host
result = client.getInfo()
if not result['OK']:
self._errMsg(result['Message'])
hostSetup = result['Value']['Setup']
result = gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, database, hostSetup, overwrite=True)
if not result['OK']:
self._errMsg(result['Message'])
return
gLogger.notice("Database %s from %s/%s installed successfully" % (database, extension, system))
# DIRAC components
elif option in self.runitComponents:
if len(argss) < 2:
gLogger.notice(self.do_install.__doc__)
return
system = argss[0]
del argss[0]
component = argss[0]
del argss[0]
specialOptions = {}
module = ''
for i in range(len(argss)):
if argss[i] == "-m":
specialOptions['Module'] = argss[i + 1]
module = argss[i + 1]
if argss[i] == "-p":
opt, value = argss[i + 1].split('=')
specialOptions[opt] = value
if module == component:
module = ''
client = SystemAdministratorClient(self.host, self.port)
# First need to update the CS
# result = client.addDefaultOptionsToCS( option, system, component )
gComponentInstaller.host = self.host
result = client.getInfo()
if not result['OK']:
self._errMsg(result['Message'])
return
hostSetup = result['Value']['Setup']
# Install Module section if not yet there
if module:
result = gComponentInstaller.addDefaultOptionsToCS(gConfig, option, system, module,
getCSExtensions(), hostSetup)
# in case of Error we must stop, this can happen when the module name is wrong...
if not result['OK']:
self._errMsg(result['Message'])
return
# Add component section with specific parameters only
result = gComponentInstaller.addDefaultOptionsToCS(gConfig, option, system, component,
getCSExtensions(), hostSetup, specialOptions,
addDefaultOptions=True)
else:
# Install component section
result = gComponentInstaller.addDefaultOptionsToCS(gConfig, option, system, component,
getCSExtensions(), hostSetup, specialOptions)
if not result['OK']:
self._errMsg(result['Message'])
return
# Then we can install and start the component
result = client.setupComponent(option, system, component, module)
if not result['OK']:
self._errMsg(result['Message'])
return
compType = result['Value']['ComponentType']
runit = result['Value']['RunitStatus']
gLogger.notice("%s %s_%s is installed, runit status: %s" % (compType, system, component, runit))
# And register it in the database
result = client.getHostInfo()
if not result['OK']:
self._errMsg(result['Message'])
return
else:
cpu = result['Value']['CPUModel']
hostname = self.host
if component == 'ComponentMonitoring':
# Make sure that the service is running before trying to use it
nTries = 0
maxTries = 5
mClient = ComponentMonitoringClient()
result = mClient.ping()
while not result['OK'] and nTries < maxTries:
time.sleep(3)
result = mClient.ping()
nTries = nTries + 1
if not result['OK']:
self._errMsg("ComponentMonitoring service taking too long to start, won't be logged into the database")
return
result = MonitoringUtilities.monitorInstallation(
'DB', system, 'InstalledComponentsDB', cpu=cpu, hostname=hostname)
if not result['OK']:
self._errMsg('Error registering installation into database: %s' % result['Message'])
return
result = MonitoringUtilities.monitorInstallation(option, system, component, module, cpu=cpu, hostname=hostname)
if not result['OK']:
self._errMsg('Error registering installation into database: %s' % result['Message'])
return
else:
gLogger.notice("Unknown option:", option)
def getInstallations(self, argss):
""" Get data from the component monitoring database
"""
display = 'table'
installationFilter = {}
componentFilter = {}
hostFilter = {}
key = ''
for arg in argss:
if not key:
if arg == 'list':
display = 'list'
elif arg == 'current':
installationFilter['UnInstallationTime'] = None
elif arg == '-t':
key = 'Component.Type'
elif arg == '-m':
key = 'Component.Module'
elif arg == '-s':
key = 'Component.System'
elif arg == '-h':
key = 'Host.HostName'
elif arg == '-n':
key = 'Instance'
elif arg == '-itb':
key = 'InstallationTime.smaller'
elif arg == '-ita':
key = 'InstallationTime.bigger'
elif arg == '-utb':
key = 'UnInstallationTime.smaller'
elif arg == '-uta':
key = 'UnInstallationTime.bigger'
else:
if 'Component.' in key:
componentFilter[key.replace('Component.', '')] = arg
elif 'Host.' in key:
hostFilter[key.replace('Host.', '')] = arg
else:
if 'Time.' in key:
arg = datetime.datetime.strptime(arg, '%d-%m-%Y')
installationFilter[key] = arg
key = None
client = ComponentMonitoringClient()
result = client.getInstallations(installationFilter, componentFilter, hostFilter, True)
if not result['OK']:
self._errMsg('Could not retrieve the installations: %s' % (result['Message']))
installations = None
else:
installations = result['Value']
if installations:
if display == 'table':
gLogger.notice('')
gLogger.notice(' ' + 'Num'.center(5) + ' ' +
'Host'.center(20) + ' ' +
'Name'.center(20) + ' ' +
'Module'.center(20) + ' ' +
'System'.center(16) + ' ' +
'Type'.center(12) + ' ' +
'Installed on'.center(18) + ' ' +
'Install by'.center(12) + ' ' +
'Uninstalled on'.center(18) + ' ' +
'Uninstall by'.center(12))
gLogger.notice(('-') * 164)
for i, installation in enumerate(installations):
if not installation['InstalledBy']:
installedBy = ''
else:
installedBy = installation['InstalledBy']
if not installation['UnInstalledBy']:
uninstalledBy = ''
else:
uninstalledBy = installation['UnInstalledBy']
if installation['UnInstallationTime']:
uninstalledOn = installation['UnInstallationTime'].strftime("%d-%m-%Y %H:%M")
isInstalled = 'No'
else:
uninstalledOn = ''
isInstalled = 'Yes'
if display == 'table':
gLogger.notice('|' + str(i + 1).center(5) + '|' +
installation['Host']['HostName'].center(20) + '|' +
installation['Instance'].center(20) + '|' +
installation['Component']['Module'].center(20) + '|' +
installation['Component']['System'].center(16) + '|' +
installation['Component']['Type'].center(12) + '|' +
installation['InstallationTime'].strftime("%d-%m-%Y %H:%M").center(18) + '|' +
installedBy.center(12) + '|' +
uninstalledOn.center(18) + '|' +
uninstalledBy.center(12) + '|')
gLogger.notice(('-') * 164)
elif display == 'list':
gLogger.notice('')
gLogger.notice('Installation: '.rjust(20) + str(i + 1))
gLogger.notice('Installed: '.rjust(20) + isInstalled)
gLogger.notice('Host: '.rjust(20) + installation['Host']['HostName'])
gLogger.notice('Name: '.rjust(20) + installation['Instance'])
gLogger.notice('Module: '.rjust(20) + installation['Component']['Module'])
gLogger.notice('System: '.rjust(20) + installation['Component']['System'])
gLogger.notice('Type: '.rjust(20) + installation['Component']['Type'])
gLogger.notice('Installed on: '.rjust(20) + installation['InstallationTime'].strftime("%d-%m-%Y %H:%M"))
if installedBy != '':
gLogger.notice('Installed by: '.rjust(20) + installedBy)
if uninstalledOn != '':
gLogger.notice('Uninstalled on: '.rjust(20) + uninstalledOn)
gLogger.notice('Uninstalled by: '.rjust(20) + uninstalledBy)
else:
self._errMsg('No display mode was selected')
gLogger.notice('')
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
from LHCbDIRAC.BookkeepingSystem.Client.BKQuery import BKQuery
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tr = TransformationClient()
for prod in prodList:
startTime = time.time()
cc = ConsistencyChecks()
# Setting the prod also sets its type
try:
cc.prod = prod
except RuntimeError as e:
gLogger.exception(lException=e)
continue
if fileType and cc.transType in ('Merge', 'MCMerge'):
gLogger.notice("It is not allowed to select file type for merging transformation", prod)
continue
cc.verbose = verbose
cc.noFC = noFC
cc.descendantsDepth = depth
if prod != prodList[0]:
gLogger.notice("====================")
gLogger.notice("Processing %s production %d" % (cc.transType, cc.prod))
if status:
res = tr.getTransformationFiles({'TransformationID': prod, 'Status': status})
if res['OK']:
lfnList = [trFile['LFN'] for trFile in res['Value']]
gLogger.notice('Found %d files with status %s' % (len(lfnList), status))
else:
gLogger.fatal("Error getting files %s" % status, res['Message'])
def main():
global overwrite
global specialOptions
global module
global specialOptions
from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
gComponentInstaller.exitOnError = True
Script.registerSwitch("w", "overwrite",
"Overwrite the configuration in the global CS",
setOverwrite)
Script.registerSwitch("m:", "module=",
"Python module name for the component code",
setModule)
Script.registerSwitch("p:", "parameter=", "Special component option ",
setSpecialOption)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument((
"System/Component: Full component name (ie: WorkloadManagement/Matcher)",
"System: Name of the DIRAC system (ie: WorkloadManagement)",
))
Script.registerArgument(
" Component: Name of the DIRAC service (ie: Matcher)",
mandatory=False)
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split("/")
if len(args) != 2:
Script.showHelp()
DIRACexit(1)
system = args[0]
component = args[1]
result = gComponentInstaller.addDefaultOptionsToCS(
gConfig,
"service",
system,
component,
extensionsByPriority(),
specialOptions=specialOptions,
overwrite=overwrite,
)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
result = gComponentInstaller.addTornadoOptionsToCS(gConfig)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
result = gComponentInstaller.installTornado()
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
gLogger.notice(
"Successfully installed component %s in %s system, now setting it up" %
(component, system))
result = gComponentInstaller.setupTornadoService(system, component,
extensionsByPriority(),
module)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
result = MonitoringUtilities.monitorInstallation("service", system,
component, module)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
gLogger.notice("Successfully completed the installation of %s/%s" %
(system, component))
DIRACexit()
def do_update(self, args):
""" Update the software on the target host to a given version
usage:
update <version> [ -r <rootPath> ] [ -g <lcgVersion> ]
where rootPath - path to the DIRAC installation
lcgVersion - version of the LCG bindings to install
"""
try:
argss = args.split()
version = argss[0]
rootPath = ''
lcgVersion = ''
del argss[0]
while len(argss) > 0:
if argss[0] == '-r':
rootPath = argss[1]
del argss[0]
del argss[0]
elif argss[0] == '-g':
lcgVersion = argss[1]
del argss[0]
del argss[0]
except Exception as x:
gLogger.notice("ERROR: wrong input:", str(x))
gLogger.notice(self.do_update.__doc__)
return
client = SystemAdministratorClient(self.host, self.port)
gLogger.notice("Software update can take a while, please wait ...")
result = client.updateSoftware(version, rootPath, lcgVersion, timeout=300)
if not result['OK']:
self._errMsg("Failed to update the software")
gLogger.notice(result['Message'])
else:
gLogger.notice("Software successfully updated.")
gLogger.notice("You should restart the services to use the new software version.")
gLogger.notice("Think of updating /Operations/<vo>/<setup>/Pilot/Versions section in the CS")
def setOnlyReleaseNotes(optionValue):
global onlyReleaseNotes
gLogger.notice("Only updating release notes!")
onlyReleaseNotes = True
return S_OK()
res = resourceStatus.getStorageElementStatus(ses)
if not res['OK']:
gLogger.error('Storage Element %s does not exist' % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-allow-se by %s' % userName
for se, seOptions in res['Value'].items():
resW = resC = resR = {'OK': False}
# InActive is used on the CS model, Banned is the equivalent in RSS
for statusType in STATUS_TYPES:
if statusFlagDict[statusType]:
if seOptions.get(statusType) == "Active":
gLogger.notice('%s status of %s is already Active' %
(statusType, se))
continue
if seOptions.has_key(statusType):
if not seOptions[statusType] in ALLOWED_STATUSES:
gLogger.notice('%s option for %s is %s, instead of %s' %
(statusType, se, seOptions['ReadAccess'],
ALLOWED_STATUSES))
gLogger.notice('Try specifying the command switches')
continue
resR = resourceStatus.setStorageElementStatus(
se, statusType, 'Active', reason, userName)
if not resR['OK']:
gLogger.error("Failed to update %s %s to Active" %
(se, statusType))
else:
def run():
from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient
client = StorageManagerClient()
queryDict = {}
if 'status' in switchDict:
queryDict['Status'] = str(switchDict['status'])
if 'se' in switchDict:
queryDict['SE'] = str(switchDict['se'])
# weird: if there are no switches (dictionary is empty), then the --limit is ignored!!
# must FIX that in StorageManagementDB.py!
# ugly fix:
newer = '1903-08-02 06:24:38' # select newer than
if 'limit' in switchDict:
gLogger.notice("Query limited to %s entries" % switchDict['limit'])
res = client.getCacheReplicas(queryDict, None, newer, None, None,
int(switchDict['limit']))
else:
res = client.getCacheReplicas(queryDict)
if not res['OK']:
gLogger.error(res['Message'])
outStr = "\n"
if res['Records']:
replicas = res['Value']
outStr += " %s" % ("Status".ljust(15))
outStr += " %s" % ("LastUpdate".ljust(20))
outStr += " %s" % ("LFN".ljust(80))
outStr += " %s" % ("SE".ljust(10))
outStr += " %s" % ("Reason".ljust(10))
if 'showJobs' in switchDict:
outStr += " %s" % ("Jobs".ljust(10))
outStr += " %s" % ("PinExpiryTime".ljust(15))
outStr += " %s" % ("PinLength(sec)".ljust(15))
outStr += "\n"
for crid, info in replicas.iteritems():
outStr += " %s" % (info['Status'].ljust(15))
outStr += " %s" % (str(info['LastUpdate']).ljust(20))
outStr += " %s" % (info['LFN'].ljust(30))
outStr += " %s" % (info['SE'].ljust(15))
outStr += " %s" % (str(info['Reason']).ljust(10))
# Task info
if 'showJobs' in switchDict:
resTasks = client.getTasks({'ReplicaID': crid})
if resTasks['OK']:
if resTasks['Value']:
tasks = resTasks['Value']
jobs = []
for tid in tasks:
jobs.append(tasks[tid]['SourceTaskID'])
outStr += ' %s ' % (str(jobs).ljust(10))
else:
outStr += ' %s ' % (" --- ".ljust(10))
# Stage request info
# what if there's no request to the site yet?
resStageRequests = client.getStageRequests({'ReplicaID': crid})
if not resStageRequests['OK']:
gLogger.error(resStageRequests['Message'])
if resStageRequests['Records']:
stageRequests = resStageRequests['Value']
for info in stageRequests.itervalues():
outStr += " %s" % (str(info['PinExpiryTime']).ljust(20))
outStr += " %s" % (str(info['PinLength']).ljust(10))
outStr += "\n"
gLogger.notice(outStr)
else:
gLogger.notice("No entries")
string = inputFile.read()
inputFile.close()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
from DIRAC.Core.Utilities.List import sortList, breakListIntoChunks
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
errorReasons = {}
successfullyRemoved = 0
for lfnList in breakListIntoChunks(lfns, 100):
res = dm.removeFile(lfnList)
if not res['OK']:
gLogger.error("Failed to remove data", res['Message'])
DIRAC.exit(-2)
for lfn, r in res['Value']['Failed'].items():
reason = str(r)
if not reason in errorReasons.keys():
errorReasons[reason] = []
errorReasons[reason].append(lfn)
successfullyRemoved += len(res['Value']['Successful'].keys())
for reason, lfns in errorReasons.items():
gLogger.notice("Failed to remove %d files with error: %s" %
(len(lfns), reason))
if successfullyRemoved > 0:
gLogger.notice("Successfully removed %d files" % successfullyRemoved)
DIRAC.exit(0)
currentDir = activeDirs.pop()
res = fc.listDirectory(currentDir, withMetadata, timeout=360)
if not res['OK']:
gLogger.error("Error retrieving directory contents",
"%s %s" % (currentDir, res['Message']))
elif currentDir in res['Value']['Failed']:
gLogger.error(
"Error retrieving directory contents",
"%s %s" % (currentDir, res['Value']['Failed'][currentDir]))
else:
dirContents = res['Value']['Successful'][currentDir]
subdirs = dirContents['SubDirs']
files = dirContents['Files']
if not subdirs and not files:
emptyDirs.append(currentDir)
gLogger.notice('%s: empty directory' % currentDir)
else:
for subdir in sorted(subdirs, reverse=True):
if (not withMetadata) or isOlderThan(
subdirs[subdir]['CreationDate'], totalDays):
activeDirs.append(subdir)
for filename in sorted(files):
fileOK = False
if (not withMetadata) or isOlderThan(
files[filename]['MetaData']['CreationDate'],
totalDays):
if wildcard is None or fnmatch.fnmatch(filename, wildcard):
fileOK = True
if not fileOK:
files.pop(filename)
allFiles += sorted(files)
setVersion)
Script.registerSwitch("p:", "package=",
"packages to tag comma separated (default = DIRAC)",
setPackage)
Script.registerSwitch("u:", "username="******"svn username to use", setUsername)
Script.registerSwitch("n", "releaseNotes", "Only refresh release notes",
setOnlyReleaseNotes)
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ...' % Script.scriptName
]))
Script.parseCommandLine(ignoreErrors=False)
gLogger.notice('Executing: %s ' % (' '.join(sys.argv)))
if not svnVersions:
Script.showHelp()
def generateAndUploadReleaseNotes(packageDistribution, svnPath,
versionReleased):
tmpDir = tempfile.mkdtemp()
packageName = packageDistribution.getPackageName()
gLogger.notice("Generating release notes for %s under %s" %
(packageName, tmpDir))
for suffix, singleVersion in (("history", False), ("notes", True)):
gLogger.notice("Generating %s rst" % suffix)
rstHistory = os.path.join(tmpDir, "release%s.rst" % suffix)
htmlHistory = os.path.join(tmpDir, "release%s.html" % suffix)
def __reloadAppCB(self):
gLogger.notice("\n !!!!!! Reloading web app...\n")
def main():
Script.registerSwitch("", "Path=", " Path to search for")
Script.registerSwitch(
"", "SE=",
" (comma-separated list of) SEs/SE-groups to be searched")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
[
"metaspec: metadata index specification (of the form: "
'"meta=value" or "meta<value", "meta!=value", etc.)'
],
mandatory=False,
)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
path = "/"
seList = None
for opt, val in Script.getUnprocessedSwitches():
if opt == "Path":
path = val
elif opt == "SE":
seList = resolveSEGroup(val.split(","))
if seList:
args.append("SE=%s" % ",".join(seList))
fc = FileCatalog()
result = fc.getMetadataFields()
if not result["OK"]:
gLogger.error("Can not access File Catalog:", result["Message"])
DIRAC.exit(-1)
typeDict = result["Value"]["FileMetaFields"]
typeDict.update(result["Value"]["DirectoryMetaFields"])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
if len(args) < 1:
print("Error: No argument provided\n%s:" % Script.scriptName)
gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict))
Script.showHelp(exitCode=1)
mq = MetaQuery(typeDict=typeDict)
result = mq.setMetaQuery(args)
if not result["OK"]:
gLogger.error("Illegal metaQuery:", result["Message"])
DIRAC.exit(-1)
metaDict = result["Value"]
path = metaDict.pop("Path", path)
result = fc.findFilesByMetadata(metaDict, path)
if not result["OK"]:
gLogger.error("Can not access File Catalog:", result["Message"])
DIRAC.exit(-1)
lfnList = sorted(result["Value"])
gLogger.notice("\n".join(lfn for lfn in lfnList))
cancel = False
listJobs = False
force = False
targetSE = set()
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Job':
try:
jobs = []
for arg in switch[1].split(','):
if os.path.exists(arg):
lines = open(arg, 'r').readlines()
jobs += [
int(job.strip()) for line in lines
for job in line.split(',')
]
gLogger.notice("Found %d jobs in file %s" %
(len(jobs), arg))
else:
jobs.append(int(arg))
except TypeError:
gLogger.fatal("Invalid jobID", job)
elif switch[0] == 'Transformation':
try:
transID = int(switch[1])
except Exception:
gLogger.fatal('Invalid transID', switch[1])
elif switch[0] == 'Tasks':
try:
taskIDs = [int(task) for task in switch[1].split(',')]
except Exception:
gLogger.fatal('Invalid tasks', switch[1])
elif switch[0] == 'Full':
gLogger.debug("Successfully updated %s catalog status to Active" %
site)
allowed.append(site)
if not allowed:
gLogger.error("Failed to allow any catalog mirrors")
DIRAC.exit(-1)
res = csAPI.commitChanges()
if not res['OK']:
gLogger.error("Failed to commit changes to CS", res['Message'])
DIRAC.exit(-1)
subject = '%d catalog instance(s) allowed for use' % len(allowed)
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
body = 'The catalog mirrors at the following sites were allowed'
for site in allowed:
body = "%s\n%s" % (body, site)
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" % addressPath,
body)
DIRAC.exit(0)
NotificationClient().sendMail(address, subject, body,
getUserOption(userName, 'Email', ''))
DIRAC.exit(0)
def setSite(args):
global Site
Site = args
def setQueue(args):
global Queue
Queue = args
Script.registerSwitch("N:", "Name=", "Computing Element Name (Mandatory)",
setCEName)
Script.registerSwitch("S:", "Site=", "Site Name (Mandatory)", setSite)
Script.registerSwitch("Q:", "Queue=", "Queue Name (Mandatory)", setQueue)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getExtraCLICFGFiles()
if len(args) > 1:
Script.showHelp(exitCode=1)
result = Resources.getQueue(Site, ceName, Queue)
if not result['OK']:
gLogger.error("Could not retrieve resource parameters",
": " + result['Message'])
DIRACExit(1)
gLogger.notice(json.dumps(result['Value']))
recursive=recursive,
fcClient=dfc)
if not res['OK']:
gLogger.fatal('\nError changing directory owner', res['Message'])
error = 1
continue
if len(directories) == 1:
gLogger.always(
'Successfully changed owner in %d directories in %.1f seconds' %
(res['Value'], time() - startTime))
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeLfnMetadata
dmScript = DMScript()
dmScript.setLFNs(baseDir)
sys.stdout.write('Directory metadata: ')
sys.stdout.flush()
executeLfnMetadata(dmScript)
elif count % 10 == 0:
sys.stdout.write('.')
sys.stdout.flush()
count += 1
success += res['Value']
error = 0
if len(directories) > 1:
gLogger.notice(
'\nSuccessfully changed owner in %d directories in %.1f seconds' %
(success, time() - startTime))
exit(error)
def comment(self, message, optMsg=''):
""" Print a comment """
fullMsg = '\n' + message + ' %s' % optMsg if optMsg else ''
gLogger.notice(fullMsg)
self._writeTitle()
self.loop(increment=False)
Usage:
%s <dataset>
""" % Script.scriptName )
Script.parseCommandLine( ignoreErrors = True )
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
argss = Script.getPositionalArgs()
fc = FileCatalogClient()
if len( argss ) > 0:
datasetName = argss[0]
else:
Script.showHelp()
result = fc.getDatasetFiles( datasetName )
if not result['OK']:
print "ERROR: failed to get files for dataset:", result['Message']
else:
lfnList = result['Value']['Successful'][datasetName]
f = open ( datasetName + '.list', 'w' )
for lfn in lfnList:
f.write( lfn + '\n' )
f.close()
gLogger.notice( '%d files have been put in %s.list' % ( len( lfnList ), datasetName ) )
gLogger.error('Given component does not exist')
DIRACexit(1)
if len(result['Value']) > 1:
gLogger.error('Too many components match')
DIRACexit(1)
removeLogs = False
if force:
removeLogs = True
else:
if result['Value'][0]['Component']['Type'] in InstallTools.COMPONENT_TYPES:
result = promptUser('Remove logs?', ['y', 'n'], 'n')
if result['OK']:
removeLogs = result['Value'] == 'y'
else:
gLogger.error(result['Message'])
DIRACexit(1)
result = InstallTools.uninstallComponent(system, component, removeLogs)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
result = MonitoringUtilities.monitorUninstallation(system, component)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
gLogger.notice('Successfully uninstalled component %s/%s' %
(system, component))
DIRACexit()
def main():
global overwrite
global specialOptions
global module
global specialOptions
from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
gComponentInstaller.exitOnError = True
Script.registerSwitch("w", "overwrite",
"Overwrite the configuration in the global CS",
setOverwrite)
Script.registerSwitch("m:", "module=",
"Python module name for the component code",
setModule)
Script.registerSwitch("p:", "parameter=", "Special component option ",
setSpecialOption)
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split('/')
if len(args) != 2:
Script.showHelp()
DIRACexit(1)
system = args[0]
component = args[1]
compOrMod = module if module else component
result = gComponentInstaller.addDefaultOptionsToCS(
gConfig,
'service',
system,
component,
getCSExtensions(),
specialOptions=specialOptions,
overwrite=overwrite)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
result = gComponentInstaller.addTornadoOptionsToCS(gConfig)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
result = gComponentInstaller.installTornado()
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
gLogger.notice(
'Successfully installed component %s in %s system, now setting it up' %
(component, system))
result = gComponentInstaller.setupTornadoService(system, component,
getCSExtensions(), module)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
result = MonitoringUtilities.monitorInstallation('service', system,
component, module)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
gLogger.notice('Successfully completed the installation of %s/%s' %
(system, component))
DIRACexit()
def checkUnusedCEs():
global vo, dry, ceBdiiDict
gLogger.notice(
'looking for new computing resources in the BDII database...')
result = getCEsFromCS()
if not result['OK']:
gLogger.error('ERROR: failed to get CEs from CS', result['Message'])
DIRACExit(-1)
knownCEs = result['Value']
result = getGridCEs(vo, ceBlackList=knownCEs)
if not result['OK']:
gLogger.error('ERROR: failed to get CEs from BDII', result['Message'])
DIRACExit(-1)
ceBdiiDict = result['BdiiInfo']
siteDict = result['Value']
if siteDict:
gLogger.notice('New resources available:\n')
for site in siteDict:
diracSite = 'Unknown'
result = getDIRACSiteName(site)
if result['OK']:
diracSite = ','.join(result['Value'])
ces = siteDict[site].keys()
if ces:
gLogger.notice(" %s, DIRAC site %s" % (site, diracSite))
for ce in ces:
gLogger.notice(' ' * 4 + ce)
gLogger.notice(' %s, %s' %
(siteDict[site][ce]['CEType'],
'%s_%s_%s' % siteDict[site][ce]['System']))
else:
gLogger.notice('No new resources available, exiting')
DIRACExit(0)
inp = raw_input("\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if not inp and inp.lower().startswith('n'):
gLogger.notice('Nothing else to be done, exiting')
DIRACExit(0)
gLogger.notice('\nAdding new sites/CEs interactively\n')
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ''
ces = siteDict[site].keys()
for ce in ces:
country = ce.strip().split('.')[-1].lower()
if len(country) == 2:
break
if country == 'gov':
country = 'us'
break
if not country or len(country) != 2:
country = 'xx'
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice(
'\nThe site %s is not yet in the CS, give it a name' % site)
diracSite = raw_input('[help|skip|<domain>.<name>.%s]: ' % country)
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice('%s site details:' % site)
for k, v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice('%s\t%s' % (k, v))
gLogger.notice(
'\nEnter DIRAC site name in the form <domain>.<name>.%s\n'
% country)
diracSite = raw_input('[<domain>.<name>.%s]: ' % country)
try:
domain, siteName, country = diracSite.split('.')
except Exception, x:
gLogger.error(
'ERROR: DIRAC site name does not follow convention: %s' %
diracSite)
continue
diracSites = [diracSite]
else:
diracSites = result['Value']
if len(diracSites) > 1:
gLogger.notice(
'Attention! GOC site %s corresponds to more than one DIRAC sites:'
% site)
gLogger.notice(str(diracSites))
gLogger.notice(
'Please, pay attention which DIRAC site the new CEs will join\n'
)
newCEs = {}
addedCEs = []
for ce in ces:
ceType = siteDict[site][ce]['CEType']
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = raw_input(
"Add CE %s of type %s to %s? [default yes] [yes|no]: " %
(ce, ceType, diracSite))
if yn == '' or yn.lower() == 'y':
newCEs.setdefault(diracSite, [])
newCEs[diracSite].append(ce)
addedCEs.append(ce)
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % (
diracSite, site, ' '.join(newCEs[diracSite]))
gLogger.notice(
"\nNew site/CEs will be added with command:\n%s" % cmd)
yn = raw_input("Add it ? [default yes] [yes|no]: ")
if not (yn == '' or yn.lower() == 'y'):
continue
if dry:
gLogger.notice("Command is skipped in the dry run")
else:
result = shellCall(0, cmd)
if not result['OK']:
gLogger.error(
'Error while executing dirac-admin-add-site command'
)
yn = raw_input(
"Do you want to continue ? [default no] [yes|no]: "
)
if yn == '' or yn.lower().startswith('n'):
if sitesAdded:
gLogger.notice(
'CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" %
(site, diracSite))
DIRACExit(0)
else:
exitStatus, stdData, errData = result['Value']
if exitStatus:
gLogger.error(
'Error while executing dirac-admin-add-site command\n',
'\n'.join([stdData, errData]))
yn = raw_input(
"Do you want to continue ? [default no] [yes|no]: "
)
if yn == '' or yn.lower().startswith('n'):
if sitesAdded:
gLogger.notice(
'CEs were added at the following sites:'
)
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" %
(site, diracSite))
DIRACExit(0)
else:
sitesAdded.append((site, diracSite))
gLogger.notice(stdData)
def __printDictionary(dictionary,
offset=0,
shift=0,
empty="Empty directory",
depth=9999):
""" Dictionary pretty printing """
key_max = 0
value_max = 0
for key, value in dictionary.iteritems():
key_max = max(key_max, len(str(key)))
value_max = max(value_max, len(str(value)))
center = key_max + offset
newOffset = offset + (shift if shift else key_max)
for key in sorted(dictionary):
value = dictionary[key]
if isinstance(value, dict):
if not depth:
value = value.keys()
elif value != {}:
gLogger.notice('%s%s : ' % (offset * ' ', key))
__printDictionary(value,
offset=newOffset,
shift=shift,
empty=empty,
depth=depth - 1)
elif key not in ('Failed', 'Successful'):
gLogger.notice('%s%s : %s' % (offset * ' ', key, empty))
if isinstance(value, (list, set)):
if not value:
gLogger.notice('%s%s : %s' % (offset * ' ', key, '[]'))
else:
gLogger.notice('%s%s : ' % (offset * ' ', key))
for val in sorted(value):
gLogger.notice('%s%s' % (newOffset * ' ', val))
elif not isinstance(value, dict):
gLogger.notice('%s : %s' % (str(key).rjust(center), str(value)))
def handler(signum, frame):
gLogger.notice('\nExit is forced, bye...')
DIRACExit(-1)
from DIRAC.Core.Base import Script
Script.setUsageMessage("""
Get the currently defined user data volume quotas
Usage:
%s [options]
""" % Script.scriptName)
Script.parseCommandLine(ignoreErrors=False)
import DIRAC
from DIRAC import gLogger, gConfig
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo(False, False)
if not res['OK']:
gLogger.error("Failed to get client proxy information.", res['Message'])
DIRAC.exit(2)
proxyInfo = res['Value']
username = proxyInfo['username']
try:
quota = gConfig.getValue('/Registry/DefaultStorageQuota', 0.)
quota = gConfig.getValue('/Registry/Users/%s/Quota' % username, quota)
gLogger.notice('Current quota found to be %.1f GB' % quota)
DIRAC.exit(0)
except Exception as x:
gLogger.exception("Failed to convert retrieved quota", '', x)
DIRAC.exit(-1)