def _getOutputs(): repoLocation = '' clip = _Params() clip.registerSwitches() Script.parseCommandLine( ignoreErrors = False ) repoLocation = clip.repo if not repoLocation: Script.showHelp() dexit(1) from DIRAC import gLogger from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac(True, repoLocation) exitCode = 0 res = dirac.monitorRepository(False) if not res['OK']: gLogger.error("Failed because %s" % res['Message']) dexit(1) res = dirac.retrieveRepositorySandboxes() if not res['OK']: gLogger.error("Failed because %s" % res['Message']) dexit(1) if clip.outputdata: res = dirac.retrieveRepositoryData() if not res['OK']: gLogger.error("Failed because %s" % res['Message']) exit(1) dexit(exitCode)
def _runFSTAgent(): """ read commands line params and run FST agent for a given transformation ID """ params = _Params() params.registerSwitches() Script.parseCommandLine() if not params.checkSettings()['OK']: Script.showHelp() dexit(1) from ILCDIRAC.ILCTransformationSystem.Agent.FileStatusTransformationAgent import FileStatusTransformationAgent fstAgent = FileStatusTransformationAgent('ILCTransformation/FileStatusTransformationAgent', 'ILCTransformation/FileStatusTransformationAgent', 'dirac-ilc-filestatus-transformation') fstAgent.log = gLogger fstAgent.enabled = params.enabled res = fstAgent.getTransformations(transID=params.transID) if not res['OK']: dexit(1) if not res['Value']: print("Transformation Not Found") dexit(1) trans = res['Value'][0] res = fstAgent.processTransformation( int(params.transID), trans['SourceSE'], trans['TargetSE'], trans['DataTransType']) if not res["OK"]: dexit(1) dexit(0)
def addProperty( arg ): global groupProperties if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in groupProperties: groupProperties.append( arg )
def _createTrafo(): """reads command line parameters, makes check and creates replication transformation""" clip = Params() clip.registerSwitches(Script) registerSwitches(Script) Script.parseCommandLine() if not clip.checkSettings(Script)['OK']: LOG.error("ERROR: Missing settings") return 1 for prodID in clip.metaValues: tGroup = getTransformationGroup(prodID, clip.groupName) parDict = dict(flavour='Replication', targetSE=clip.targetSE, sourceSE=clip.sourceSE, metaKey=clip.metaKey, metaValue=prodID, extraData={'Datatype': clip.datatype}, extraname=clip.extraname, plugin=clip.plugin, groupSize=clip.groupSize, tGroup=tGroup, enable=clip.enable, ) LOG.debug("Parameters: %s" % pformat(parDict)) resCreate = createDataTransformation(**parDict) if not resCreate['OK']: return 1 return 0
def _createTrafo(): """reads command line parameters, makes check and creates replication transformation""" clip = Params() clip.registerSwitches(Script) registerSwitches(clip, Script) Script.parseCommandLine() if not clip.checkSettings(Script)['OK']: LOG.error("ERROR: Missing settings") return 1 for index, prodID in enumerate(clip.metaValues): datatype = clip.datatype if clip.datatype else ['GEN', 'SIM', 'REC'][index % 3] plugin = 'Broadcast' if clip.forcemoving or clip.flavour != 'Moving' else 'BroadcastProcessed' retData = checkDatatype(prodID, datatype) if not retData['OK']: LOG.error("ERROR: %s" % retData['Message']) return 1 tGroup = getTransformationGroup(prodID, clip.groupName) parDict = dict(flavour='Moving', targetSE=clip.targetSE, sourceSE=clip.sourceSE, metaKey=clip.metaKey, metaValue=prodID, extraData={'Datatype': datatype}, extraname=clip.extraname, plugin=plugin, groupSize=clip.groupSize, tGroup=tGroup, enable=clip.enable, ) LOG.debug("Parameters: %s" % pformat(parDict)) resCreate = createDataTransformation(**parDict) if not resCreate['OK']: return 1 return 0
def _showSoftware(): """Show available software""" clip = _Params() clip.registerSwitches() Script.parseCommandLine() from DIRAC import gConfig, gLogger base = '/Operations/Defaults/AvailableTarBalls' platforms = gConfig.getSections(base) for platform in platforms['Value']: gLogger.notice("For platform %s, here are the available software" % platform) apps = gConfig.getSections(base + "/" + platform) for app in apps['Value']: if clip.software and app.lower() != clip.software.lower(): continue gLogger.notice(" - %s" % app) versions = gConfig.getSections(base + "/" + platform + "/" + app) if clip.appsOnly: continue for vers in versions['Value']: gLogger.notice(" * %s" % vers) depsb = gConfig.getSections(base + "/" + platform + "/" + app + "/" + vers) if 'Dependencies' in depsb['Value']: gLogger.notice(" Depends on") deps = gConfig.getSections( os.path.join( base, platform, app, vers , "Dependencies") ) for dep in deps['Value']: depversions = gConfig.getOption(base + "/" + platform + "/" + app + "/" + vers + "/Dependencies/" + dep + "/version") gLogger.notice(" %s %s" % (dep, depversions['Value'])) if not len(versions['Value']): gLogger.notice(" No version available")
def main(): if len(args) < 1: Script.showHelp() return fixedStatuses = ['Total', 'Done', 'Failed', 'Running', 'Waiting', 'Deleted'] for taskID in args: taskID = int(taskID) progress = getTaskEventProgress(taskID) if not progress: continue print '='*80 print 'Task %s progress:' % taskID print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12 print '%-12s %-12s %-12s' % ('Status', 'Job Number', 'Event Number') print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12 printALine(progress, 'Total') print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12 for fixedStatus in fixedStatuses: if fixedStatus != 'Total': printALine(progress, fixedStatus) for p in progress: if p not in fixedStatuses: printALine(progress, p) print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12 print ''
def addProperty( arg ): global hostProperties if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in hostProperties: hostProperties.append( arg )
def addUserGroup(arg): global userGroups if not arg: Script.showHelp() DIRAC.exit(-1) if not arg in userGroups: userGroups.append(arg)
def _createTrafo(): """reads command line parameters, makes check and creates replication transformation""" clip = Params() clip.registerSwitches(Script) Script.parseCommandLine() if not clip.checkSettings(Script)['OK']: gLogger.error("ERROR: Missing settings") return 1 for metaValue in clip.metaValues: resCreate = createDataTransformation(flavour=clip.flavour, targetSE=clip.targetSE, sourceSE=clip.sourceSE, metaKey=clip.metaKey, metaValue=metaValue, extraData=clip.extraData, extraname=clip.extraname, groupSize=clip.groupSize, plugin=clip.plugin, enable=clip.enable, ) if not resCreate['OK']: gLogger.error("Failed to create Transformation", resCreate['Message']) return 1 return 0
def addUserName( arg ): global userNames if not arg: Script.showHelp() DIRAC.exit( -1 ) if not arg in userNames: userNames.append( arg )
def parseSwitches(): """ Parses the arguments passed by the user """ Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() if args: subLogger.error( "Found the following positional args '%s', but we only accept switches" % args ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) switches = dict( Script.getUnprocessedSwitches() ) switches.setdefault( 'statusType' , None ) switches.setdefault( 'releaseToken', False ) for key in ( 'element', 'name', 'reason' ): if not key in switches: subLogger.error( "%s Switch missing" % key ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) if not switches[ 'element' ] in ( 'Site', 'Resource', 'Node' ): subLogger.error( "Found %s as element switch" % switches[ 'element' ] ) subLogger.error( "Please, check documentation below" ) Script.showHelp() DIRACExit( 1 ) subLogger.debug( "The switches used are:" ) map( subLogger.debug, switches.iteritems() ) return switches
def runTests(): """runs the tests""" clip = CLIParams() clip.registerSwitches() Script.parseCommandLine() overlayrun = clip.testOverlay myMarlinSteeringFile = "clic_ild_cdr_steering_overlay_1400.0.xml" if overlayrun else "clic_ild_cdr_steering.xml" myLCSimPreSteeringFile = "clic_cdr_prePandoraOverlay_1400.0.lcsim" if overlayrun else "clic_cdr_prePandora.lcsim" myLCSimPostSteeringFile = "clic_cdr_postPandoraOverlay.lcsim" parameterDict = dict( mokkaVersion="0706P08", mokkaSteeringFile="clic_ild_cdr.steer", detectorModel="CLIC_ILD_CDR", steeringFileVersion="V22", machine="clic_cdr", backgroundType="gghad", energy=1400, marlinVersion="v0111Prod", rootVersion="5.34", marlinSteeringFile=myMarlinSteeringFile, marlinInputdata = "/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio", gearFile='clic_ild_cdr.gear', lcsimPreSteeringFile=myLCSimPreSteeringFile, lcsimPostSteeringFile=myLCSimPostSteeringFile ) myTests = TestCreater(clip, parameterDict) res = myTests.checkForTests() if not res['OK']: dexit(1) myTests.run() return
def getBoolean( value ): if value.lower() == 'true': return True elif value.lower() == 'false': return False else: Script.showHelp()
def registerSwitches(self): Script.registerSwitch('P:', "Platform=", 'Platform to use', self.setPlatform) Script.registerSwitch('p:', "Path=", "Path to the Whizard results directory", self.setPath) Script.registerSwitch("V:", "Version=", "Whizard version", self.setVersion) Script.registerSwitch('b:', 'BeamSpectra=', 'Beam spectra version', self.setBeamSpectra) Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1], '\nUsage:', ' %s [option|cfgfile] ...\n' % Script.scriptName ] ) )
def setUp( self ): from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8 ) self.processPool.daemonize()
def __init__( self, taskID, timeWait, raiseException=False ): from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger self.log = gLogger.getSubLogger( self.__class__.__name__ + "/%s" % taskID ) self.taskID = taskID self.timeWait = timeWait self.raiseException = raiseException
def main(): if len(args) < 1: Script.showHelp() return for taskID in args: taskID = int(taskID) deleteTask(taskID) print ""
def setUserMail(arg): global userMail if userMail or not arg: Script.showHelp() DIRAC.exit(-1) if not arg.find("@") > 0: Script.gLogger.error("Not a valid mail address", arg) DIRAC.exit(-1) userMail = arg
def registerUsageMessage(): ''' Takes the script __doc__ and adds the DIRAC version to it ''' usageMessage = 'DIRAC version: %s \n' % version usageMessage += __doc__ Script.setUsageMessage( usageMessage )
def registerUsageMessage(): """ Takes the script __doc__ and adds the DIRAC version to it """ usageMessage = "DIRAC version: %s \n" % version usageMessage += __doc__ Script.setUsageMessage(usageMessage)
def error( msg ): ''' Format error messages ''' subLogger.error( "\nERROR:" ) subLogger.error( "\t" + msg ) subLogger.error( "\tPlease, check documentation below" ) Script.showHelp() DIRACExit( 1 )
def isOK(self): if not self.cliParams.releasesToBuild: gLogger.error("Missing releases to build!") Script.showHelp() return False if not self.cliParams.destination: self.cliParams.destination = tempfile.mkdtemp('DiracDist') else: mkDir(self.cliParams.destination) gLogger.notice("Will generate tarballs in %s" % self.cliParams.destination) return True
def __init__( self, taskID, timeWait, raiseException=False ): from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger self.log = gLogger.getSubLogger( self.__class__.__name__ + "/%s" % taskID ) self.taskID = taskID self.log.always( "pid=%s task=%s I'm locked" % ( os.getpid(), self.taskID ) ) gLock.acquire() self.log.always("you can't see that line, object is stuck by gLock" ) self.timeWait = timeWait self.raiseException = raiseException gLock.release()
def main(): if len(args) < 1: Script.showHelp() return jobIDs = [] for jobID in args: jobIDs.append(int(jobID)) showJobs(jobIDs) print 'Totally %s job(s) displayed' % len(jobIDs)
def main(): """ main program entry point """ options = Params() options.registerCLISwitches() Script.parseCommandLine( ignoreErrors = True ) if options.delete_all and options.vos: gLogger.error( "-a and -v options are mutually exclusive. Please pick one or the other." ) return 1 proxyLoc = Locations.getDefaultProxyLocation() if not os.path.exists( proxyLoc ): gLogger.error( "No local proxy found in %s, exiting." % proxyLoc ) return 1 result = ProxyInfo.getProxyInfo( proxyLoc, True ) if not result[ 'OK' ]: raise RuntimeError( 'Failed to get local proxy info.' ) if result[ 'Value' ][ 'secondsLeft' ] < 60 and options.needsValidProxy(): raise RuntimeError( 'Lifetime of local proxy too short, please renew proxy.' ) userDN=result[ 'Value' ][ 'identity' ] if options.delete_all: # delete remote proxies remote_groups = getProxyGroups() if not remote_groups: gLogger.notice( 'No remote proxies found.' ) for vo_group in remote_groups: deleteRemoteProxy( userDN, vo_group ) # delete local proxy deleteLocalProxy( proxyLoc ) elif options.vos: vo_groups = set() for voname in options.vos: vo_groups.update(mapVoToGroups( voname ) ) # filter set of all groups to only contain groups for which there is a user proxy user_groups = getProxyGroups() vo_groups.intersection_update( user_groups ) if not vo_groups: gLogger.notice( 'You have no proxies registered for any of the specified VOs.' ) for group in vo_groups: deleteRemoteProxy( userDN, group ) else: deleteLocalProxy( proxyLoc ) return 0
def setUpClass( cls ): # destroy kerberos token #try: # subprocess.call(['kdestroy']) #except subprocess.CalledProcessError as err: # print "WARNING: kdestroy did not succeed." # print err.output # Constants for the tests Script.parseCommandLine() user = ProxyInfo.getProxyInfo()['Value']['username'] SETestCase.lfntestfilepath += '%s/%s/setests/%s/' % (user[0], user, randomFolder()) print "Using lfn %s" % SETestCase.lfntestfilepath SETestCase.lfntestfile = SETestCase.lfntestfilepath + SETestCase.lfntestfilename
def _createTrafo(): """reads command line parameters, makes check and creates replication transformation""" from DIRAC import exit as dexit clip = _Params() clip.registerSwitches() Script.parseCommandLine() if not clip.checkSettings()['OK']: gLogger.error("ERROR: Missing settings") dexit(1) resCreate = _createReplication( clip.targetSE, clip.sourceSE, clip.prodID, clip.datatype, clip.extraname ) if not resCreate['OK']: dexit(1) dexit(0)
def registerSwitches(): ''' Registers all switches that can be used while calling the script from the command line interface. ''' switches = ( ( 'init', 'Initialize the element to the status in the CS ( applicable for StorageElements )' ), ( 'element=', 'Element family to be Synchronized ( Site, Resource or Node ) or `all`' ), ) for switch in switches: Script.registerSwitch( '', switch[ 0 ], switch[ 1 ] )
def registerUsageMessage(): ''' Takes the script __doc__ and adds the DIRAC version to it ''' hLine = ' ' + '='*78 + '\n' usageMessage = hLine usageMessage += ' DIRAC %s\n' % version usageMessage += __doc__ usageMessage += '\n' + hLine Script.setUsageMessage( usageMessage )
#!/usr/bin/env python ######################################################################## # $HeadURL$ # File : dirac-production-job-delete # Author : Stuart Paterson ######################################################################## """ Delete DIRAC job from WMS, if running it will be killed """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:', ' JobID: DIRAC Job ID' ])) Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() import os.path if __name__ == "__main__":
#!/usr/bin/env python ######################################################################## # $HeadURL$ # File : dirac-jobexec # Author : Stuart Paterson ######################################################################## __RCSID__ = "$Id$" """ The dirac-jobexec script is equipped to execute workflows that are specified via their XML description. The main client of this script is the Job Wrapper. """ import DIRAC from DIRAC.Core.Base import Script # Register workflow parameter switch Script.registerSwitch('p:', 'parameter=', 'Parameters that are passed directly to the workflow') Script.parseCommandLine() from DIRAC.Core.Workflow.Parameter import * from DIRAC.Core.Workflow.Module import * from DIRAC.Core.Workflow.Step import * from DIRAC.Core.Workflow.Workflow import * from DIRAC.Core.Workflow.WorkflowReader import * from DIRAC import S_OK, S_ERROR, gConfig, gLogger from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer import DIRAC import os, os.path, sys, string
#! /usr/bin/env python ######################################################################## # $HeadURL: $ ######################################################################## __RCSID__ = "$Id: $" from DIRAC.Core.Base import Script Script.setUsageMessage(""" Resolve problematic files for the specified transformations Usage: %s [options] TransID [TransID] """ % Script.scriptName) Script.parseCommandLine() import DIRAC from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.List import sortList from LHCbDIRAC.DataManagementSystem.Client.DataIntegrityClient import DataIntegrityClient integrityClient = DataIntegrityClient() def resolveTransforamtionProblematics(transID): gLogger.notice("Obtaining problematic files for transformation %d" % transID) res = integrityClient.getTransformationProblematics(transID) if not res['OK']: gLogger.error("Failed to get transformation problematic files", res['Message'])
def setHostDN(arg): global hostDN if hostDN or not arg: Script.showHelp() DIRAC.exit(-1) hostDN = arg
hostDN = arg def addProperty(arg): global hostProperties if not arg: Script.showHelp() DIRAC.exit(-1) if not arg in hostProperties: hostProperties.append(arg) Script.setUsageMessage('\n'.join([ 'Add or Modify a Host info in DIRAC', '\nUsage:\n', ' %s [option|cfgfile] ... Property=<Value> ...' % Script.scriptName, '\nArguments:\n', ' Property=<Value>: Other properties to be added to the User like (Responsible=XXXX)', ])) Script.registerSwitch('H:', 'HostName:', 'Name of the Host (Mandatory)', setHostName) Script.registerSwitch('D:', 'HostDN:', 'DN of the Host Certificate (Mandatory)', setHostDN) Script.registerSwitch( 'P:', 'Property:', 'Property to be added to the Host (Allow Multiple instances or None)', addProperty) Script.parseCommandLine(ignoreErrors=True)
#!/usr/bin/env python ######################################################################## # File : dirac-admin-get-job-pilot-output # Author : Stuart Paterson ######################################################################## """ Retrieve the output of the pilot that executed a given job """ from __future__ import print_function __RCSID__ = "$Id$" from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:', ' JobID: DIRAC ID of the Job' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() from DIRAC import exit as DIRACExit from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() exitCode = 0 errorList = [] for job in args:
#!/usr/bin/env python ######################################################################## # $Header: $ ######################################################################## __RCSID__ = "$Id$" from DIRAC import exit as dexit from DIRAC.Core.Base import Script from DIRAC import gLogger Script.setUsageMessage(""" Remove the given file replica or a list of file replicas from the File Catalog This script should be used with great care as it may leave dark data in the storage! Use dirac-dms-remove-replicas instead Usage: %s <LFN | fileContainingLFNs> <SE> """ % Script.scriptName) Script.parseCommandLine() from DIRAC.Core.Security.ProxyInfo import getProxyInfo res = getProxyInfo() if not res['OK']: gLogger.fatal("Can't get proxy info", res['Message']) dexit(1) properties = res['Value'].get('groupProperties', []) if not 'FileCatalogManagement' in properties: gLogger.error( "You need to use a proxy from a group with FileCatalogManagement") dexit(5)
from DIRAC.Core.Base import Script from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC import exit as DIRACExit, gConfig, gLogger from DIRAC.Core.Utilities.List import intListToString from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getPropertiesForGroup from DIRAC.Core.Utilities.SitesDIRACGOCDBmapping import getDIRACSiteName if __name__ == "__main__": Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... DIRACSiteName GridSiteName CE [CE] ...' % Script.scriptName, 'Arguments:', ' DIRACSiteName: Name of the site for DIRAC in the form GRID.LOCATION.COUNTRY (ie:LCG.CERN.ch)', ' GridSiteName: Name of the site in the Grid (ie: CERN-PROD)', ' CE: Name of the CE to be included in the site (ie: ce111.cern.ch)' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() csAPI = CSAPI() if len(args) < 3: Script.showHelp() DIRACExit(-1) diracSiteName = args[0] gridSiteName = args[1] ces = args[2:]
#/usr/bin/env python ############################################# #$HeadURL:$ #data:2013/08/07 #author:gang ############################################# """ upload a set of file to SE and register them in DFC. """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.registerSwitch("r", "dir", "the directory that dataset files located") Script.setUsageMessage('\n'.join([ __doc__, 'Usage:', '%s dir' % Script.scriptName, 'Arguments:' ' dir: dir is a logical directory in DFC' ])) Script.parseCommandLine(ignoreErrors=True) dir = Script.getPositionalArgs() #print dir if len(dir) != 1: Script.showHelp() from BESDIRAC.Badger.API.Badger import Badger badger = Badger() localdir = dir[0] exitCode = 0
for tableName in [ 'MessageRepository', 'FixedTextMessages', 'SubSystems', 'Systems', 'AgentPersistentData', 'ClientIPs', 'Sites', 'UserDNs' ]: result = db._update('DROP TABLE `%s`' % tableName) assert result['OK'] gLogger.info('\n OK\n') except AssertionError: print 'ERROR ', if not result['OK']: print result['Message'] else: print result sys.exit(1) if __name__ == '__main__': from DIRAC.Core.Base import Script Script.parseCommandLine() gLogger.setLevel('VERBOSE') if 'PYTHONOPTIMIZE' in os.environ and os.environ['PYTHONOPTIMIZE']: gLogger.info('Unset pyhthon optimization "PYTHONOPTIMIZE"') sys.exit(0) testSystemLoggingDB()
######################################################################## # File : dirac-externals-requirements # Author : Adri/Federico ######################################################################## """ If /RequiredExternals/ is found in releases.cfg, then some packages to install with pip may be found. This will do it. """ import os import sys from pip.index import PackageFinder from pip.req import InstallRequirement, RequirementSet from pip.locations import user_dir, src_prefix from pip.download import PipSession from DIRAC.Core.Base import Script Script.disableCS() from DIRAC import gLogger, rootPath, S_OK from DIRAC.Core.Utilities.CFG import CFG __RCSID__ = "$Id$" instType = "server" def setInstallType(val): global instType instType = val return S_OK()
def registerSwitches(self): Script.registerSwitch("P:", "Platform=", "Platform ex. %s" % self.platform, self.setPlatform) Script.registerSwitch("A:", "Applications=", "Comma separated list of applications", self.setName) Script.registerSwitch("V:", "Version=", "Version name", self.setVersion) Script.registerSwitch("C:", "Comment=", "Comment", self.setComment) Script.registerSwitch("S:", "Script=", "Full path to initScript", self.setInitScript) Script.registerSwitch("B:", "Base=", "Path to Installation Base", self.setBasePath) Script.registerSwitch( "O:", "Config=", "Path To [Clic|ILD]Config (if it is in ApplicationPath)", self.setConfig) Script.registerSwitch( "", "ILDConfig=", "Path To ILDConfig (if one is in ApplicationPath) [DEPRECATED]", self.setConfig) Script.registerSwitch("Q:", "DBSlice=", "Path to Mokka DB Slice", self.setDBSlice) Script.registerSwitch("N", "dry-run", "DryRun: do not commit to CS", self.dryRun) Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], '\nUsage:', ' %s [option|cfgfile] ...\n' % Script.scriptName ]))
#! /usr/bin/env python ######################################################################## # $HeadURL$ # File : dirac-wms-jobs-select-output-search # Author : Vladimir Romanovsky ######################################################################## """ Retrieve output sandbox for DIRAC Jobs for the given selection and search for a string in their std.out """ __RCSID__ = "$Id$" import os from shutil import rmtree import DIRAC from DIRAC.Core.Base import Script Script.registerSwitch("", "Status=", "Primary status") Script.registerSwitch("", "MinorStatus=", "Secondary status") Script.registerSwitch("", "ApplicationStatus=", "Application status") Script.registerSwitch("", "Site=", "Execution site") Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") Script.registerSwitch( "", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") Script.registerSwitch("", "File=", "File name,if not specified default is std.out ") Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... String ...' % Script.scriptName, 'Arguments:', ' String: string to search for' ]))
""" Prod3 MC Pipe Script to create a Transformation JB, LA December 2016 """ from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s array_layout site particle pointing_dir zenith_angle nShower' % Script.scriptName, 'Arguments:', ' array_layout: hex or square', ' site: Paranal, Aar, Armazones_2K', ' particle: gamma, proton, electron', ' pointing_dir: North or South', ' zenith_agle: 20', ' nShower: from 5 to 25000', '\ne.g: %s hex Paranal gamma South 20 5' % Script.scriptName, ])) Script.parseCommandLine() import DIRAC from DIRAC.TransformationSystem.Client.Transformation import Transformation from DIRAC.Core.Workflow.Parameter import Parameter #from CTADIRAC.Interfaces.API.Prod3MCPipeJob import Prod3MCPipeJob from Prod3MCPipeJob import Prod3MCPipeJob #from DIRAC.Interfaces.API.Dirac import Dirac def submitTS(job):
import json import sys from DIRAC import gLogger from DIRAC.Core.Base import Script from DIRAC.Core.Security.X509Chain import X509Chain from DIRAC.Core.Security.ProxyInfo import getProxyInfo class CertEncoder(json.JSONEncoder): """ JSON encoder for data structures possibly including certificate objects. """ def default(self, obj): if isinstance(obj, X509Chain): return obj.dumpAllToString() return json.JSONEncoder.default(self, obj) if __name__ == "__main__": # Suppress error messages, since stdout of this script is expected to be in JSON format gLogger.setLevel("FATAL") Script.enableCS() # Required so dict includes username ProxyInfo = getProxyInfo() if not ProxyInfo["OK"]: sys.exit(1) print(json.dumps(ProxyInfo["Value"], cls=CertEncoder))
#!/usr/bin/env python """ Get VM instances available in the configured cloud sites """ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script from DIRAC import gLogger, exit as DIRACExit Script.setUsageMessage('\n'.join([ 'Get VM nodes information', 'Usage:', '%s site ce node [option]... [cfgfile]' % Script.scriptName, 'Arguments:', ' cfgfile: DIRAC Cfg with description of the configuration (optional)' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() from VMDIRAC.WorkloadManagementSystem.Client.VMClient import VMClient from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup if len(args) != 3: print Script.showHelp() DIRACExit(-1) site, ce, node = args vmClient = VMClient() result = vmClient.stopInstance(site, ce, node) if not result['OK']:
from DIRAC import gLogger from DIRAC import S_OK, S_ERROR, exit from DIRAC.Core.Base import Script Script.setUsageMessage(""" Get and merge output files of the task Usage: %(script)s [options] TaskID Examples: # Download all output files in task 165 to the current directory %(script)s 165 # Download all output files in task 165 and merge all the files into larger ones, where the merged file size will not exceed 1GB %(script)s -g 1G 165 # Download all dst files in task 237 to directory "/some/dir" %(script)s -f '*.dst' -D /some/dir 237 # List all root file names in task 46. Do NOT download %(script)s -l -f '*.root' 46 # Download all root files in task 329 to directory "output" and merge to files smaller than 800MB, reserve the root files after merge %(script)s -g 800M -k -f '*.root' -D output 329 """ % {'script': Script.scriptName}) Script.registerSwitch( "m:", "method=", "Downloading method: local_rsync, dfc, cp, daemon_rsync") Script.registerSwitch("l", "list", "List all output files. Do NOT download") Script.registerSwitch("D:", "dir=", "Output directory")
#!/usr/bin/env python ######################################################################## # $Header: $ ######################################################################## from __future__ import print_function __RCSID__ = "$Id: $" import DIRAC from DIRAC.Core.Base import Script from DIRAC import exit as dexit from DIRAC import gLogger Script.setUsageMessage(""" Remove the given file or a list of files from the File Catalog Usage: %s <LFN | fileContainingLFNs> """ % Script.scriptName) Script.parseCommandLine() from DIRAC.Core.Security.ProxyInfo import getProxyInfo res = getProxyInfo() if not res['OK']: gLogger.fatal("Can't get proxy info", res['Message']) dexit(1) properties = res['Value'].get('groupProperties', []) if not 'FileCatalogManagement' in properties: gLogger.error( "You need to use a proxy from a group with FileCatalogManagement") dexit(5)
#!/usr/bin/env python """ Simple data management script for PROD3 MC create DFC MetaData structure put and register files in DFC """ __RCSID__ = "$Id$" # generic imports import os, glob, json # DIRAC imports import DIRAC from DIRAC.Core.Base import Script Script.parseCommandLine() # Specific DIRAC imports from CTADIRAC.Core.Workflow.Modules.Prod3DataManager import Prod3DataManager #################################################### def putAndRegisterPROD3(args): """ simple wrapper to put and register all PROD3 files Keyword arguments: args -- a list of arguments in order [] """ metadata = args[0] metadatafield = args[1] filemetadata = args[2] inputpath = args[3]
def setSites(optVal_): global sites sites = optVal_.split(',') return S_OK() description = """Get computing resources capable to execute a job with the given description. Note that only statically defined computing resource parameters are considered although sites can fail matching due to their dynamic state, e.g. occupancy by other jobs. Also input data proximity is not taken into account. """ Script.setUsageMessage('\n'.join( [description, 'Usage:', ' %s [option]... <job_JDL>' % Script.scriptName])) Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) Script.registerSwitch("S:", "site=", "Check matching for these sites (comma separated list)", setSites) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) == 0: gLogger.error("Error: No job description provided") Script.showHelp(exitCode=1)
def setHostName(arg): global hostName if hostName or not arg: Script.showHelp() DIRAC.exit(-1) hostName = arg
Staged GRIDKA-RDST 1 4.5535 StageSubmitted GRIDKA-RDST 5 22.586 Waiting PIC-RDST 3 13.6478 WARNING: the Size for files with Status=New is not yet determined at the point of selection! --------------------- current status of the SE Caches from the DB----------- GRIDKA-RDST : 6 replicas with a size of 29.141 GB. """ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script from DIRAC import gConfig, gLogger, exit as DIRACExit, S_OK, version Script.setUsageMessage(__doc__) Script.parseCommandLine(ignoreErrors=False) from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient client = StorageManagerClient() res = client.getCacheReplicasSummary() if not res['OK']: gLogger.fatal(res['Message']) DIRACExit(2) stagerInfo = res['Value'] outStr = "\n" outStr += " %s" % ("Status".ljust(20)) outStr += " %s" % ("SE".ljust(20)) outStr += " %s" % ("NumberOfFiles".ljust(20)) outStr += " %s" % ("Size(GB)".ljust(20)) outStr += " \n--------------------------------------------------------------------------\n"
######################################################################## # $HeadURL$ # File : DiracAdmin.py # Author : Stuart Paterson ######################################################################## from DIRAC.Core.Base import Script Script.initialize() """DIRAC Administrator API Class All administrative functionality is exposed through the DIRAC Admin API. Examples include site banning and unbanning, WMS proxy uploading etc. """ __RCSID__ = "$Id$" import DIRAC from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.ConfigurationSystem.Client.Helpers import getVO from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.Core.Utilities.SiteCEMapping import getCEsForSite, getSiteCEMapping from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient from DIRAC.Core.Security.X509Chain import X509Chain from DIRAC.Core.Security import Locations, CS from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.Grid import ldapSite, ldapCluster, ldapCE, ldapService, ldapCEState, ldapCEVOView, ldapSA import re, os, sys, string, time, shutil, types import pprint
#!/usr/bin/env python """ Start a given production """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], 'Usage:', ' %s prodID' % Script.scriptName, 'Arguments:', ' prodID: Production ID (mandatory)' ])) Script.parseCommandLine() from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp(exitCode=1) # get arguments prodID = args[0] prodClient = ProductionClient()
#!/usr/bin/env python """ Find files in the FileCatalog using file metadata """ from __future__ import print_function import DIRAC if __name__ == "__main__": from DIRAC.Core.Base import Script Script.registerSwitch('', 'Path=', ' Path to search for') Script.registerSwitch('', 'SE=', ' (comma-separated list of) SEs/SE-groups to be searched') Script.setUsageMessage( '\n'.join( [ __doc__.split('\n')[1], 'Usage:', ' %s [options] metaspec [metaspec ...]' % Script.scriptName, 'Arguments:', ' metaspec: metadata index specification \ (of the form: "meta=value" or "meta<value", "meta!=value", etc.)', '', 'Examples:', ' $ dirac-dms-find-lfns Path=/lhcb/user "Size>1000" "CreationDate<2015-05-15"', ])) Script.parseCommandLine(ignoreErrors=True)
def setVO(optionValue): global vo vo = optionValue Script.localCfg.addDefaultEntry('/DIRAC/VirtualOrganization', vo) DIRAC.gConfig.setOptionValue(cfgInstallPath('VirtualOrganization'), vo) return DIRAC.S_OK() def forceUpdate(optionValue): global update update = True return DIRAC.S_OK() Script.disableCS() Script.registerSwitch("S:", "Setup=", "Set <setup> as DIRAC setup", setSetup) Script.registerSwitch("C:", "ConfigurationServer=", "Set <server> as DIRAC configuration server", setServer) Script.registerSwitch("I", "IncludeAllServers", "include all Configuration Servers", setAllServers) Script.registerSwitch("n:", "SiteName=", "Set <sitename> as DIRAC Site Name", setSiteName) Script.registerSwitch("N:", "CEName=", "Determiner <sitename> from <cename>", setCEName) Script.registerSwitch("V:", "VO=", "Set the VO name", setVO) Script.registerSwitch("W:", "gateway=", "Configure <gateway> as DIRAC Gateway for the site", setGateway)
#!/usr/bin/env python ######################################################################## # File : dirac-admin-list-users # Author : Adrian Casajus ######################################################################## from __future__ import print_function __RCSID__ = "$Id$" from DIRAC.Core.Base import Script Script.registerSwitch("e", "extended", "Show extended info") Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() from DIRAC import exit as DIRACExit from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() exitCode = 0 errorList = [] extendedInfo = False for unprocSw in Script.getUnprocessedSwitches(): if unprocSw[0] in ('e', 'extended'): extendedInfo = True if not extendedInfo: result = diracAdmin.csListHosts() for host in result['Value']: print(" %s" % host) else:
#!/usr/bin/env python ######################################################################## # $HeadURL$ # File : dirac-admin-list-users # Author : Adrian Casajus ######################################################################## """ Lists the users in the Configuration. If no group is specified return all users. """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.registerSwitch("e", "extended", "Show extended info") Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... [Group] ...' % Script.scriptName, 'Arguments:', ' Group: Only users from this group (default: all)' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) == 0: args = ['all'] from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() exitCode = 0 errorList = [] extendedInfo = False
# Author : Ricardo Graciani ######################################################################## """ Report Normalized CPU length of queue This script is used by the dirac-pilot script to set the CPUTime limit for the matching """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import queueNormalizedCPU Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, 'Arguments:', ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() exitCode = 0 for ceUniqueID in args: normCPU = queueNormalizedCPU(ceUniqueID) if not normCPU['OK']:
#!/usr/bin/env python """ Get summary informations of all productions """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script from DIRAC.Core.Utilities.PrettyPrint import printTable Script.setUsageMessage('\n'.join([__doc__.split('\n')[1]])) Script.parseCommandLine() from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient prodClient = ProductionClient() res = prodClient.getProductions() fields = [ 'ProductionName', 'Status', 'ProductionID', 'CreationDate', 'LastUpdate', 'AuthorDN', 'AuthorGroup'] records = []