def checkOrGetGroupProxy( group ): """Check if current proxy corresponds to the given group, and if not call the :any:`getNewProxy` method to obtain a proxy for this group :param string group: dirac group of the desired proxy :returns: :func:`S_OK() <DIRAC:DIRAC.Core.Utilities.ReturnValues.S_OK>`, :func:`~DIRAC:DIRAC.Core.Utilities.ReturnValues.S_ERROR` """ result = getProxyInfo() if result['OK'] and 'group' in result['Value']: if result['Value']['group'] == group: return S_OK() else: print "You don't have an %s proxy, trying to get one..." % group else: print "Error to get proxy information, trying to get proxy" newProxyRetVal = getNewProxy( group=group ) if newProxyRetVal: ## != 0 return S_ERROR("dirac-proxy-init failed") result = getProxyInfo() if result['OK'] and 'group' in result['Value']: if result['Value']['group'] == group: return S_OK() else: print 'You do not have a valid group' return S_ERROR( "Could not obtain valid group" ) elif result['OK'] and 'group' not in result['Value']: return S_ERROR( "Could not obtain group information from proxy" ) print 'Could not obtain proxy information: %s' % result['Message'] return result
def __setupProxy(self, ownerDN, ownerGroup): """ Retrieve a proxy for the execution of the job """ if gConfig.getValue("/DIRAC/Security/UseServerCertificate", False): proxyResult = self.__requestProxyFromProxyManager(ownerDN, ownerGroup) if not proxyResult["OK"]: self.log.error("Invalid Proxy", proxyResult["Message"]) return S_ERROR("Failed to setup proxy: %s" % proxyResult["Message"]) return S_OK(proxyResult["Value"]) else: ret = getProxyInfo(disableVOMS=True) if not ret["OK"]: self.log.error("Invalid Proxy", ret["Message"]) return S_ERROR("Invalid Proxy") proxyChain = ret["Value"]["chain"] if not "groupProperties" in ret["Value"]: print ret["Value"] print proxyChain.dumpAllToString() self.log.error("Invalid Proxy", "Group has no properties defined") return S_ERROR("Proxy has no group properties defined") groupProps = ret["Value"]["groupProperties"] if Properties.GENERIC_PILOT in groupProps or Properties.PILOT in groupProps: proxyResult = self.__requestProxyFromProxyManager(ownerDN, ownerGroup) if not proxyResult["OK"]: self.log.error("Invalid Proxy", proxyResult["Message"]) return S_ERROR("Failed to setup proxy: %s" % proxyResult["Message"]) proxyChain = proxyResult["Value"] return S_OK(proxyChain)
def monitorUninstallation( system, component, cpu = None, hostname = None ): """ Register the uninstallation of a component in the ComponentMonitoringDB """ monitoringClient = ComponentMonitoringClient() # Retrieve user uninstalling the component result = getProxyInfo() if result[ 'OK' ]: user = result[ 'Value' ][ 'username' ] else: return result if not user: user = '******' if not cpu: cpu = 'Not available' for line in open( '/proc/cpuinfo' ): if line.startswith( 'model name' ): cpu = line.split( ':' )[1][0:64] cpu = cpu.replace( '\n', '' ).lstrip().rstrip() if not hostname: hostname = socket.getfqdn() instance = component[ 0 : 32 ] result = monitoringClient.updateInstallations \ ( { 'Instance': instance, 'UnInstallationTime': None }, { 'System': system }, { 'HostName': hostname, 'CPU': cpu }, { 'UnInstallationTime': datetime.datetime.utcnow(), 'UnInstalledBy': user } ) return result
def getToken(key): """ Function that gets the userName from the proxy """ proxyInfo = getProxyInfo() if not proxyInfo["OK"]: error(str(proxyInfo)) if key.lower() == "owner": userName = proxyInfo["Value"]["username"] tokenOwner = S_OK(userName) if not tokenOwner["OK"]: error(tokenOwner["Message"]) return tokenOwner["Value"] elif key.lower() == "expiration": expiration = proxyInfo["Value"]["secondsLeft"] tokenExpiration = S_OK(expiration) if not tokenExpiration["OK"]: error(tokenExpiration["Message"]) now = Time.dateTime() # datetime.datetime.utcnow() expirationDate = now + datetime.timedelta(seconds=tokenExpiration["Value"]) expirationDate = Time.toString(expirationDate) expirationDate = expirationDate.split(".")[0] return expirationDate
def setSiteStatus(self, site, status, comment='No comment'): """ Set the status of a site in the 'SiteStatus' table of RSS examples >>> siteStatus.banSite( 'site1.test.test' ) S_OK() >>> siteStatus.banSite( None ) S_ERROR( ... ) :Parameters: **site** - `String` the site that is going to be banned **comment** - `String` reason for banning :return: S_OK() || S_ERROR() """ if not status: return S_ERROR(DErrno.ERESUNK, 'status parameter is empty') # fix case sensitive string status = status.capitalize() allowedStateList = ['Active', 'Banned', 'Degraded', 'Probing', 'Error', 'Unknown'] if status not in allowedStateList: return S_ERROR(errno.EINVAL, 'Not a valid status, parameter rejected') if self.rssFlag: result = getProxyInfo() if result['OK']: tokenOwner = result['Value']['username'] else: return S_ERROR("Unable to get user proxy info %s " % result['Message']) tokenExpiration = datetime.utcnow() + timedelta(days=1) self.rssCache.acquireLock() try: result = self.rsClient.modifyStatusElement('Site', 'Status', status=status, name=site, tokenExpiration=tokenExpiration, reason=comment, tokenOwner=tokenOwner) if result['OK']: self.rssCache.refreshCache() else: _msg = 'Error updating status of site %s to %s' % (site, status) gLogger.warn('RSS: %s' % _msg) # Release lock, no matter what. finally: self.rssCache.releaseLock() else: if status in ['Active', 'Degraded']: result = RPCClient('WorkloadManagement/WMSAdministrator').allowSite() else: result = RPCClient('WorkloadManagement/WMSAdministrator').banSite() return result
def __setupProxy( self, ownerDN, ownerGroup ): """ Retrieve a proxy for the execution of the job """ if gConfig.getValue( '/DIRAC/Security/UseServerCertificate' , False ): proxyResult = self.__requestProxyFromProxyManager( ownerDN, ownerGroup ) if not proxyResult['OK']: self.log.error( 'Invalid Proxy', proxyResult['Message'] ) return S_ERROR( 'Failed to setup proxy: %s' % proxyResult[ 'Message' ] ) return S_OK( proxyResult['Value'] ) else: ret = getProxyInfo( disableVOMS = True ) if not ret['OK']: self.log.error( 'Invalid Proxy', ret['Message'] ) return S_ERROR( 'Invalid Proxy' ) proxyChain = ret['Value']['chain'] if not 'groupProperties' in ret['Value']: print ret['Value'] print proxyChain.dumpAllToString() self.log.error( 'Invalid Proxy', 'Group has no properties defined' ) return S_ERROR( 'Proxy has no group properties defined' ) groupProps = ret['Value']['groupProperties'] if Properties.GENERIC_PILOT in groupProps or Properties.PILOT in groupProps: proxyResult = self.__requestProxyFromProxyManager( ownerDN, ownerGroup ) if not proxyResult['OK']: self.log.error( 'Invalid Proxy', proxyResult['Message'] ) return S_ERROR( 'Failed to setup proxy: %s' % proxyResult[ 'Message' ] ) proxyChain = proxyResult['Value'] return S_OK( proxyChain )
def initialize( self, request ): """ Set default values to attributes and parameters """ if type( request ) == NoneType: # Set some defaults for name in self.attributeNames: self.attributes[name] = 'Unknown' self.attributes['CreationTime'] = str( Time.dateTime() ) self.attributes['Status'] = "New" result = getProxyInfo() if result['OK']: proxyDict = result[ 'Value' ] self.attributes['OwnerDN'] = proxyDict[ 'identity' ] if 'group' in proxyDict: self.attributes['OwnerGroup'] = proxyDict[ 'group' ] self.attributes['DIRACSetup'] = gConfig.getValue( '/DIRAC/Setup', 'Unknown' ) elif isinstance( request, RequestContainer ): for attr in self.attributeNames: self.attributes[attr] = request.attributes[attr] # initialize request from an XML string if type( request ) in StringTypes: for name in self.attributeNames: self.attributes[name] = 'Unknown' self.parseRequest( request ) # Initialize request from another request elif isinstance( request, RequestContainer ): self.subRequests = copy.deepcopy( request.subrequests )
def getToken( key ): ''' Function that gets the userName from the proxy ''' proxyInfo = getProxyInfo() if not proxyInfo[ 'OK' ]: error( str( proxyInfo ) ) if key.lower() == 'owner': userName = proxyInfo[ 'Value' ][ 'username' ] tokenOwner = S_OK( userName ) if not tokenOwner[ 'OK' ]: error( tokenOwner[ 'Message' ] ) return tokenOwner[ 'Value' ] elif key.lower() == 'expiration': expiration = proxyInfo[ 'Value' ][ 'secondsLeft' ] tokenExpiration = S_OK( expiration ) if not tokenExpiration[ 'OK' ]: error( tokenExpiration[ 'Message' ] ) now = Time.dateTime() #datetime.datetime.utcnow() expirationDate = now + datetime.timedelta( seconds=tokenExpiration['Value'] ) expirationDate = Time.toString( expirationDate ) expirationDate = expirationDate.split('.')[0] return expirationDate
def _banStorageElement( self, storageElement ): endpoints = getFTS3Servers()[ 'Value' ] blacklist = {} for endpoint in endpoints: # endpoint = 'https://fts3-pilot.cern.ch:8446' # TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API proxyPath = getProxyInfo() if not proxyPath['OK']: return proxyPath try: proxyPath = proxyPath['Value']['path'] except Exception as e: return S_ERROR( repr( e ).replace( ',)', ')' ) ) context = fts3.Context( endpoint, proxyPath ) timeout = 3600 # or...? status = 'wait' # or...? allow_submit = False # or...? # TODO: ban_se returns the list of jobIDs interrupted by the banning pausedJobIDs = fts3.ban_se( context, storageElement, status, timeout, allow_submit ) self.log.info( "fts3.ban_se: %s" % pausedJobIDs ) blacklist[endpoint] = json.loads( context.get( "ban/se" ) ) return S_OK( blacklist )
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN='', bulkSubmissionFlag=False): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] try: transJson = json.loads(transBody) self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup) except ValueError: # #json couldn't load self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup) return S_OK(taskDict)
def _unbanStorageElement(self, storageElement): endpoints = getFTS3Servers() if not endpoints["OK"]: return endpoints endpoints = endpoints["Value"] blacklist = {} for endpoint in endpoints: # endpoint = 'https://fts3-pilot.cern.ch:8446' # TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API proxyPath = getProxyInfo() if not proxyPath["OK"]: return proxyPath try: proxyPath = proxyPath["Value"]["path"] except Exception as e: return S_ERROR(repr(e).replace(",)", ")")) context = fts3.Context(endpoint, proxyPath) fts3.unban_se(context, storageElement) blacklist[endpoint] = json.loads(context.get("ban/se")) return S_OK(blacklist)
def __getPrefix(self): op = Operations("glast.org") self.userprefix = None res = getProxyInfo() if res["OK"]: if "username" in res["Value"]: user = res["Value"]["username"] self.userprefix = "/glast.org/user/%s/%s/" % (user[0], user) else: self.log.error("Proxy could not be found") return 1 task_category = os.environ["GPL_TASKCATEGORY"] if not task_category: task_category = op.getValue("Pipeline/TaskCategory", None) if not task_category: self.log.error("Could not find task category") return 1 self.prefixDest = op.getValue("Pipeline/StorageElementBasePath", self.userprefix) self.stagingDest = ( self.prefixDest + "/" + task_category + "/" + os.environ["PIPELINE_TASK"] + "/" + os.environ["PIPELINE_STREAM"] ) return 0
def _banStorageElement( self, storageElement ): endpoints = getFTS3Servers()[ 'Value' ] blacklist = {} for endpoint in endpoints: #endpoint = 'https://fts3-pilot.cern.ch:8446' #TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API proxyPath = getProxyInfo() if not proxyPath.get('OK'): return S_ERROR("Proxy not found!") try: proxyPath = proxyPath.get('Value').get('path') except Exception as e: return S_ERROR(e.message) context = fts3.Context(endpoint, proxyPath) timeout = 3600 #or...? status = 'wait' #or...? allow_submit = False #or...? #TODO: ban_se returns the list of jobIDs interrupted by the banning pausedJobIDs = fts3.ban_se(context, storageElement, status, timeout, allow_submit) blacklist[endpoint] = json.loads(context.get("ban/se")) return S_OK( blacklist ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def __init__(self, script = None): super(ProductionJob, self).__init__( script ) self.prodVersion = __RCSID__ self.dryrun = False self.created = False self.checked = False self.call_finalization = False self.finalsdict = {} self.transfid = 0 self.type = 'Production' self.csSection = '/Production/Defaults' self.ops = Operations() self.fc = FileCatalogClient() self.trc = TransformationClient() self.defaultProdID = '12345' self.defaultProdJobID = '12345' self.jobFileGroupSize = 1 self.nbtasks = 1 self.slicesize =0 self.basename = '' self.basepath = self.ops.getValue('/Production/CLIC/BasePath','/ilc/prod/clic/') self.evttype = '' self.datatype = '' self.energycat = '' self.detector = '' self.currtrans = None self.description = '' self.finalpaths = [] self.finalMetaDict = defaultdict( dict ) self.prodMetaDict = {} self.finalMetaDictNonSearch = {} self.metadict_external = {} self.outputStorage = '' self.proxyinfo = getProxyInfo() self.inputdataquery = False self.inputBKSelection = {} self.plugin = 'Standard' self.prodGroup = '' self.prodTypes = ['MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction', 'MCReconstruction_Overlay', 'Merge', 'Split', 'MCGeneration_ILD', 'MCSimulation_ILD', 'MCReconstruction_ILD', 'MCReconstruction_Overlay_ILD', 'Split_ILD' ] self.prodparameters = {} self.prodparameters['NbInputFiles'] = 1 self.prodparameters['nbevts'] = 0 #self.prodparameters["SWPackages"] = '' self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job") if not script: self.__setDefaults() self._recBasePaths = {} self.maxFCFoldersToCheck = 100000
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '', bulkSubmissionFlag = False ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB jobClass is by default "DIRAC.Interfaces.API.Job.Job". An extension of it also works. :param transBody: transformation job template :param taskDict: dictionary of per task parameters :param owner: owner of the transformation :param ownerGroup: group of the owner of the transformation :param ownerDN: DN of the owner of the transformation :return: S_OK/S_ERROR with updated taskDict """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] if bulkSubmissionFlag: return self.__prepareTransformationTasksBulk( transBody, taskDict, owner, ownerGroup, ownerDN ) else: return self.__prepareTransformationTasks( transBody, taskDict, owner, ownerGroup, ownerDN )
def __changeSiteStatus( self, site, comment, statusType, status, printOutput = False ): """ Change the RSS status of the given site """ result = self.__checkSiteIsValid( site ) if not result['OK']: return result wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) result = wmsAdmin.allowSite( site, comment ) if not result['OK']: return result rsc = ResourceStatusClient() proxyInfo = getProxyInfo() if not proxyInfo[ 'OK' ]: return proxyInfo userName = proxyInfo[ 'Value' ][ 'username' ] tomorrow = datetime.utcnow().replace( microsecond = 0 ) + timedelta( days = 1 ) result = rsc.modifyStatusElement( 'Site', 'Status', name = site, statusType = statusType, status = status, reason = comment, tokenOwner = userName, tokenExpiration = tomorrow ) return result
def notifyAboutNewSoftware(self): """Send an email to the mailing list if a new software version was defined""" #Only send email when something was actually added if not self.modifiedCS: return subject = '%s %s added to DIRAC CS' % (self.appName, self.appVersion) msg = 'New application %s %s declared into Configuration service\n %s' % (self.appName, self.appVersion, self.comment) from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUserOption from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient notifyClient = NotificationClient() gLogger.notice('Sending mail for software installation to %s' % (self.mailadress)) res = getProxyInfo() if not res['OK']: sender = '*****@*****.**' else: if 'username' in res['Value']: sender = getUserOption(res['Value']['username'],'Email') else: sender = '*****@*****.**' gLogger.info('*'*80)# surround email with stars res = notifyClient.sendMail(self.mailadress, subject, msg, sender, localAttempt = False) gLogger.info('*'*80) if not res[ 'OK' ]: gLogger.error('The mail could not be sent: %s' % res['Message'])
def initialize(self, request): """ Set default values to attributes and parameters """ if type(request) == NoneType: # Set some defaults for name in self.attributeNames: self.attributes[name] = "Unknown" self.attributes["CreationTime"] = str(Time.dateTime()) self.attributes["Status"] = "New" result = getProxyInfo() if result["OK"]: proxyDict = result["Value"] self.attributes["OwnerDN"] = proxyDict["identity"] if "group" in proxyDict: self.attributes["OwnerGroup"] = proxyDict["group"] self.attributes["DIRACSetup"] = gConfig.getValue("/DIRAC/Setup", "Unknown") elif isinstance(request, RequestContainer): for attr in self.attributeNames: self.attributes[attr] = request.attributes[attr] # initialize request from an XML string if type(request) in StringTypes: for name in self.attributeNames: self.attributes[name] = "Unknown" self.parseRequest(request) # Initialize request from another request elif isinstance(request, RequestContainer): self.subRequests = copy.deepcopy(request.subrequests)
def uploadProxyToMyProxy( params, DNAsUsername ): """ Upload proxy to the MyProxy server """ myProxy = MyProxy() if DNAsUsername: gLogger.verbose( "Uploading pilot proxy with group %s to %s..." % ( params.getDIRACGroup(), myProxy.getMyProxyServer() ) ) else: gLogger.verbose( "Uploading user proxy with group %s to %s..." % ( params.getDIRACGroup(), myProxy.getMyProxyServer() ) ) retVal = myProxy.getInfo( proxyInfo[ 'path' ], useDNAsUserName = DNAsUsername ) if retVal[ 'OK' ]: remainingSecs = ( int( params.getProxyRemainingSecs() / 3600 ) * 3600 ) - 7200 myProxyInfo = retVal[ 'Value' ] if 'timeLeft' in myProxyInfo and remainingSecs < myProxyInfo[ 'timeLeft' ]: gLogger.verbose( " Already uploaded" ) return True retVal = generateProxy( params ) if not retVal[ 'OK' ]: gLogger.error( " There was a problem generating proxy to be uploaded to myproxy: %s" % retVal[ 'Message' ] ) return False retVal = getProxyInfo( retVal[ 'Value' ] ) if not retVal[ 'OK' ]: gLogger.error( " There was a problem generating proxy to be uploaded to myproxy: %s" % retVal[ 'Message' ] ) return False generatedProxyInfo = retVal[ 'Value' ] retVal = myProxy.uploadProxy( generatedProxyInfo[ 'path' ], useDNAsUserName = DNAsUsername ) if not retVal[ 'OK' ]: gLogger.error( " Can't upload to myproxy: %s" % retVal[ 'Message' ] ) return False gLogger.verbose( " Uploaded" ) return True
def _banStorageElement( self, storageElement ): endpoints = getFTS3Servers() if not endpoints['OK']: return endpoints endpoints = endpoints['Value'] blacklist = {} for endpoint in endpoints: # endpoint = 'https://fts3-pilot.cern.ch:8446' # TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API proxyPath = getProxyInfo() if not proxyPath['OK']: return proxyPath try: proxyPath = proxyPath['Value']['path'] except Exception as e: return S_ERROR( repr( e ).replace( ',)', ')' ) ) context = fts3.Context( endpoint, proxyPath ) status = 'wait' # This status leaves the jobs queued. The only alternative is "cancel" pausedJobIDs = fts3.ban_se( context, storageElement, status, timeout = 3600, allow_submit = False ) self.log.info( "fts3.ban_se: paused jobs: %s" % ','.join(pausedJobIDs) ) blacklist[endpoint] = json.loads( context.get( "ban/se" ) ) return S_OK( blacklist )
def _unbanStorageElement( self, storageElement ): endpoints = getFTS3Servers()[ 'Value' ] blacklist = {} for endpoint in endpoints: #endpoint = 'https://fts3-pilot.cern.ch:8446' #TODO: maybe proxyPath is not needed since it is picked from the environment by the REST API proxyPath = getProxyInfo() if not proxyPath.get('OK'): return S_ERROR("Proxy not found!") try: proxyPath = proxyPath.get('Value').get('path') except Exception as e: return S_ERROR(e.message) context = fts3.Context(endpoint, proxyPath) fts3.unban_se(context, storageElement) blacklist[endpoint] = json.loads(context.get("ban/se")) return S_OK( blacklist )
def __init__(self, script=None): """Initialize UserJob, including proxy and splitmixin.""" super(UserJob, self).__init__(script) self.type = 'User' self.diracinstance = None self.usergroup = ['ilc_user', 'calice_user'] self.proxyinfo = getProxyInfo() SplitMixin._initialize(self)
def execute( self ): res = getProxyInfo( disableVOMS = True ) if not res['OK']: return res self.proxyInfoDict = res['Value'] return self.monitorStageRequests()
def __init__( self, fromDict = None ): """c'tor :param self: self reference :param fromDict : if false, new request. Can be json string that represents the object, or the dictionary directly """ self.__waiting = None now = datetime.datetime.utcnow().replace( microsecond = 0 ) self._CreationTime = now self._SubmitTime = now self._LastUpdate = now # the time before which the request should not be executed # If None, no delay self._NotBefore = now self._Status = "Done" self.JobID = 0 self.Error = None self.DIRACSetup = None self.OwnerDN = None self.RequestName = None self.OwnerGroup = None self.SourceComponent = None self.dmsHelper = DMSHelpers() proxyInfo = getProxyInfo() if proxyInfo["OK"]: proxyInfo = proxyInfo["Value"] if proxyInfo["validGroup"] and proxyInfo["validDN"]: self.OwnerDN = proxyInfo["identity"] self.OwnerGroup = proxyInfo["group"] self.__operations__ = [] fromDict = fromDict if isinstance( fromDict, dict )\ else json.loads( fromDict ) if isinstance( fromDict, StringTypes )\ else {} if "Operations" in fromDict: for opDict in fromDict.get( "Operations", [] ): self +=Operation( opDict ) del fromDict["Operations"] for key, value in fromDict.items(): # The JSON module forces the use of UTF-8, which is not properly # taken into account in DIRAC. # One would need to replace all the '== str' with 'in StringTypes' if type( value ) in StringTypes: value = value.encode() if value: setattr( self, key, value ) self._notify()
def setUp( self ): gLogger.setLevel( 'DEBUG' ) self.wrapperFile = None # get proxy proxyInfo = getProxyInfo( disableVOMS = True ) proxyChain = proxyInfo['Value']['chain'] proxyDumped = proxyChain.dumpAllToString() self.payloadProxy = proxyDumped['Value']
def monitorInstallation( componentType, system, component, module = None, cpu = None, hostname = None ): """ Register the installation of a component in the ComponentMonitoringDB """ monitoringClient = ComponentMonitoringClient() if not module: module = component # Retrieve user installing the component user = None result = getProxyInfo() if result[ 'OK' ]: proxyInfo = result[ 'Value' ] if 'username' in proxyInfo: user = proxyInfo[ 'username' ] else: return result if not user: user = '******' if not cpu: cpu = 'Not available' for line in open( '/proc/cpuinfo' ): if line.startswith( 'model name' ): cpu = line.split( ':' )[1][ 0 : 64 ] cpu = cpu.replace( '\n', '' ).lstrip().rstrip() if not hostname: hostname = socket.getfqdn() instance = component[ 0 : 32 ] result = monitoringClient.installationExists \ ( { 'Instance': instance, 'UnInstallationTime': None }, { 'Type': componentType, 'System': system, 'Module': module }, { 'HostName': hostname, 'CPU': cpu } ) if not result[ 'OK' ]: return result if result[ 'Value' ]: return S_OK( 'Monitoring of %s is already enabled' % component ) result = monitoringClient.addInstallation \ ( { 'InstallationTime': datetime.datetime.utcnow(), 'InstalledBy': user, 'Instance': instance }, { 'Type': componentType, 'System': system, 'Module': module }, { 'HostName': hostname, 'CPU': cpu }, True ) return result
def loadCredentials( self ): retVal = getProxyInfo() if retVal[ 'OK' ]: credDict = retVal[ 'Value' ] self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ], credDict[ 'group' ], Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) ) return retVal return retVal
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): if (not owner) or (not ownerGroup): res = getProxyInfo(False,False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] oJob = Job(transBody) for taskNumber in sortList(taskDict.keys()): paramsDict = taskDict[taskNumber] transID = paramsDict['TransformationID'] self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup)) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) transGroup = str(transID).zfill(8) self.log.verbose('Adding default transformation group of %s' % (transGroup)) oJob.setJobGroup(transGroup) constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8) self.log.verbose('Setting task name to %s' % constructedName) oJob.setName(constructedName) oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8)) oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8)) inputData = None for paramName,paramValue in paramsDict.items(): self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue)) if paramName=='InputData': if paramValue: self.log.verbose('Setting input data to %s' %paramValue) oJob.setInputData(paramValue) elif paramName=='Site': if paramValue: self.log.verbose('Setting allocated site to: %s' %(paramValue)) oJob.setDestination(paramValue) elif paramValue: self.log.verbose('Setting %s to %s' % (paramName,paramValue)) oJob._addJDLParameter(paramName,paramValue) hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])] if int(transID) in hospitalTrans: hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch') hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[]) oJob.setType('Hospital') oJob.setDestination(hospitalSite) oJob.setInputDataPolicy('download',dataScheduling=False) if hospitalCEs: oJob._addJDLParameter('GridRequiredCEs',hospitalCEs) taskDict[taskNumber]['TaskObject'] = '' res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData}) if not res ['OK']: self.log.error("Failed to generate output data",res['Message']) continue for name,output in res['Value'].items(): oJob._addJDLParameter(name,string.join(output,';')) taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML()) return S_OK(taskDict)
def submitJob(self, executableFile, proxy, **kwargs): """ Method to submit job. """ if self.pPool is None: self.pPool = ProcessPool(minSize=self.processors, maxSize=self.processors, poolCallback=self.finalizeJob) self.pPool.processResults() processorsInUse = self.getProcessorsInUse() if kwargs.get('wholeNode'): if processorsInUse > 0: return S_ERROR('Can not take WholeNode job') # , %d/%d slots used' % (self.slotsInUse,self.slots) ) else: requestedProcessors = self.processors elif "numberOfProcessors" in kwargs: requestedProcessors = int(kwargs['numberOfProcessors']) if requestedProcessors > 0: if (processorsInUse + requestedProcessors) > self.processors: return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors, self.processors - processorsInUse)) else: requestedProcessors = 1 if self.processors - processorsInUse < requestedProcessors: return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors, self.processors - processorsInUse)) ret = getProxyInfo() if not ret['OK']: pilotProxy = None else: pilotProxy = ret['Value']['path'] self.log.notice('Pilot Proxy:', pilotProxy) kwargs = {'UseSudo': False} if self.useSudo: for nUser in range(MAX_NUMBER_OF_SUDO_UNIX_USERS): if nUser not in self.userNumberPerTask.values(): break kwargs['NUser'] = nUser kwargs['PayloadUser'] = os.environ['USER'] + 'p%s' % str(nUser).zfill(2) kwargs['UseSudo'] = True result = self.pPool.createAndQueueTask(executeJob, args=(executableFile, proxy, self.taskID), kwargs=kwargs, taskID=self.taskID, usePoolCallbacks=True) self.processorsPerTask[self.taskID] = requestedProcessors self.taskID += 1 self.pPool.processResults() return result
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def proxyUser(): """ Read proxy to get username. """ res = getProxyInfo() if not res["OK"]: return res return S_OK(res["Value"]["username"])
def getTokenOwner(): ''' Function that gets the userName from the proxy ''' proxyInfo = getProxyInfo() if not proxyInfo['OK']: return proxyInfo userName = proxyInfo['Value']['username'] return S_OK(userName)
def proxyUser(): """ Read proxy to get username. """ res = getProxyInfo() if not res[ 'OK' ]: return res return S_OK( res[ 'Value' ][ 'username' ] )
def execute( self ): res = getProxyInfo( disableVOMS = True ) if not res['OK']: return res self.proxyInfoDict = res['Value'] res = self.monitorStageRequests() return res
def loadCredentials(self): retVal = getProxyInfo() if retVal["OK"]: credDict = retVal["Value"] self.commiterId = "%s@%s - %s" % ( credDict["username"], credDict["group"], datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), ) return retVal return retVal
def __ownerAndGroup(self): res = getProxyInfo(False, False) if not res['OK']: raise Exception('GetProxyInfo error: {0}'.format(res['Message'])) self.__owner = res['Value']['username'] self.__ownerGroup = res['Value']['group'] self.__vo = Registry.getVOMSVOForGroup(self.__ownerGroup) self.__voHome = '/{0}'.format(self.__vo) self.__prodHome = '/{0}/reconstruction'.format(self.__vo) self.__userHome = '/{0}/user/{1:.1}/{1}'.format( self.__vo, self.__owner)
def getCurrentOwner(): """Simple function to return current DIRAC username. """ result = getProxyInfo() if not result['OK']: return S_ERROR('Could not obtain proxy information') if 'username' not in result['Value']: return S_ERROR('Could not get username from proxy') username = result['Value']['username'] return S_OK(username)
def setSiteStatus(self, site, status, comment='No comment'): """ Set the status of a site in the 'SiteStatus' table of RSS examples >>> siteStatus.banSite( 'site1.test.test' ) S_OK() >>> siteStatus.banSite( None ) S_ERROR( ... ) :Parameters: **site** - `String` the site that is going to be banned **comment** - `String` reason for banning :return: S_OK() || S_ERROR() """ if not status: return S_ERROR(DErrno.ERESUNK, 'status parameter is empty') # fix case sensitive string status = status.capitalize() allowedStateList = [ 'Active', 'Banned', 'Degraded', 'Probing', 'Error', 'Unknown' ] if status not in allowedStateList: return S_ERROR(errno.EINVAL, 'Not a valid status, parameter rejected') result = getProxyInfo() if result['OK']: tokenOwner = result['Value']['username'] else: return S_ERROR("Unable to get user proxy info %s " % result['Message']) tokenExpiration = datetime.utcnow() + timedelta(days=1) result = self.rsClient.modifyStatusElement( 'Site', 'Status', status=status, name=site, tokenExpiration=tokenExpiration, reason=comment, tokenOwner=tokenOwner) if not result['OK']: return result return S_OK()
def _getCurrentUser( self ): res = getProxyInfo( False, False ) if not res['OK']: return self._errorReport( 'No proxy found in local environment', res['Message'] ) proxyInfo = res['Value'] gLogger.debug( formatProxyInfoAsString( proxyInfo ) ) if 'group' not in proxyInfo: return self._errorReport( 'Proxy information does not contain the group', res['Message'] ) res = getDNForUsername( proxyInfo['username'] ) if not res['OK']: return self._errorReport( 'Failed to get proxies for user', res['Message'] ) return S_OK( proxyInfo['username'] )
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names")) Script.parseCommandLine() from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations allowUsers = Operations().getValue( "DataManagement/AllowUserReplicaManagement", False) from DIRAC.Core.Security.ProxyInfo import getProxyInfo res = getProxyInfo() if not res["OK"]: gLogger.fatal("Can't get proxy info", res["Message"]) dexit(1) properties = res["Value"].get("groupProperties", []) if not allowUsers: if "FileCatalogManagement" not in properties: gLogger.error( "You need to use a proxy from a group with FileCatalogManagement" ) dexit(5) from DIRAC.Resources.Catalog.FileCatalog import FileCatalog fc = FileCatalog() import os # parseCommandLine show help when mandatory arguments are not specified or incorrect argument args = Script.getPositionalArgs() inputFileName = args[0] if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") string = inputFile.read() lfns = [lfn.strip() for lfn in string.splitlines()] inputFile.close() else: lfns = [inputFileName] res = fc.removeFile(lfns) if not res["OK"]: print("Error:", res["Message"]) dexit(1) for lfn in sorted(res["Value"]["Failed"].keys()): message = res["Value"]["Failed"][lfn] print("Error: failed to remove %s: %s" % (lfn, message)) print("Successfully removed %d catalog files." % (len(res["Value"]["Successful"])))
def __init__(self, fromDict=None): """c'tor :param self: self reference :param fromDict: if false, new request. Can be json string that represents the object, or the dictionary directly """ self.__waiting = None now = datetime.datetime.utcnow().replace(microsecond=0) self._CreationTime = now self._SubmitTime = now self._LastUpdate = now # the time before which the request should not be executed # If None, no delay self._NotBefore = now self._Status = "Done" self.JobID = 0 self.Error = None self.DIRACSetup = None self.OwnerDN = None self.RequestName = None self.OwnerGroup = None self._SourceComponent = None self.dmsHelper = DMSHelpers() proxyInfo = getProxyInfo() if proxyInfo["OK"]: proxyInfo = proxyInfo["Value"] if proxyInfo["validGroup"] and proxyInfo["validDN"]: self.OwnerDN = proxyInfo["identity"] self.OwnerGroup = proxyInfo["group"] self.__operations__ = [] if isinstance(fromDict, str): fromDict = json.loads(fromDict) elif not isinstance(fromDict, dict): fromDict = {} if "Operations" in fromDict: for opDict in fromDict.get("Operations", []): self += Operation(opDict) del fromDict["Operations"] for key, value in fromDict.items(): if value: setattr(self, key, value) self._notify()
def getDatasetNamePrefix(self): """descide the prefix of a datasetName""" prefix = '' result = getProxyInfo(False, False) if result['OK']: userGroup = result['Value']['group'] if userGroup == 'bes_user': prefix = 'User_' elif userGroup == 'production': prefix = 'Proc_' return prefix else: return prefix
def execute(self): self.log.info("Start Execution") result = getProxyInfo() if not result['OK']: return result infoDict = result['Value'] self.log.info(formatProxyInfoAsString(infoDict)) self.__lookForCE() self.__infoFromCE() self.log.info("End Execution") return S_OK()
def getCurrentVO(): """Simple function to return current DIRAC username. """ result = getProxyInfo() if not result['OK']: return S_ERROR('Could not obtain proxy information') if 'group' not in result['Value']: return S_ERROR('Could not get group from proxy') group = result['Value']['group'] vo = group.split("_")[0] return S_OK(vo)
def submitJob( self, executableFile, proxy, dummy = None ): """ Method to submit job, should be overridden in sub-class. """ ret = getProxyInfo() if not ret['OK']: pilotProxy = None else: pilotProxy = ret['Value']['path'] self.log.notice( 'Pilot Proxy:', pilotProxy ) payloadEnv = dict( os.environ ) payloadProxy = '' if proxy: self.log.verbose( 'Setting up proxy for payload' ) result = self.writeProxyToFile( proxy ) if not result['OK']: return result payloadProxy = result['Value'] # pilotProxy = os.environ['X509_USER_PROXY'] payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy self.log.verbose( 'Starting process for monitoring payload proxy' ) gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy, taskArgs = ( pilotProxy, payloadProxy ), executions = 0, elapsedTime = 0 ) if not os.access( executableFile, 5 ): os.chmod( executableFile, 0755 ) cmd = os.path.abspath( executableFile ) self.log.verbose( 'CE submission command: %s' % ( cmd ) ) result = systemCall( 0, cmd, callbackFunction = self.sendOutput, env = payloadEnv ) if payloadProxy: os.unlink( payloadProxy ) ret = S_OK() if not result['OK']: self.log.error( 'Fail to run InProcess', result['Message'] ) elif result['Value'][0] < 0: self.log.error( 'InProcess Job Execution Failed' ) self.log.info( 'Exit status:', result['Value'][0] ) return S_ERROR( 'InProcess Job Execution Failed' ) elif result['Value'][0] > 0: self.log.error( 'Fail in payload execution' ) self.log.info( 'Exit status:', result['Value'][0] ) ret['PayloadFailed'] = result['Value'][0] else: self.log.debug( 'InProcess CE result OK' ) self.submittedJobs += 1 return ret
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ): """Instantiates the Workflow object and some default parameters. """ self.log = gLogger self.section = COMPONENT_NAME self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True #gConfig.getValue('Tier0SE-tape','SEName') self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo( disableVOMS = True ) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup( ret['Value']['group'] ) self.group = vo self.site = 'ANY' #ANY #self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] self.systemConfig = 'ANY' self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE', 'MinCPUTime': 'other.NAME<=VALUE', 'Site': 'other.NAME=="VALUE"', 'Platform': 'other.NAME=="VALUE"', #'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately 'BannedSites': 'other.Site!="VALUE"', 'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'} ##Add member to handle Parametric jobs self.parametric = {} self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow( script ) #Global error dictionary self.errorDict = {}
def main(): Script.parseCommandLine() from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations allowUsers = Operations().getValue( "DataManagement/AllowUserReplicaManagement", False) from DIRAC.Core.Security.ProxyInfo import getProxyInfo res = getProxyInfo() if not res['OK']: gLogger.fatal("Can't get proxy info", res['Message']) dexit(1) properties = res['Value'].get('groupProperties', []) if not allowUsers: if 'FileCatalogManagement' not in properties: gLogger.error( "You need to use a proxy from a group with FileCatalogManagement" ) dexit(5) from DIRAC.Resources.Catalog.FileCatalog import FileCatalog fc = FileCatalog() import os args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp(exitCode=1) else: inputFileName = args[0] if os.path.exists(inputFileName): inputFile = open(inputFileName, 'r') string = inputFile.read() lfns = [lfn.strip() for lfn in string.splitlines()] inputFile.close() else: lfns = [inputFileName] res = fc.removeFile(lfns) if not res['OK']: print("Error:", res['Message']) dexit(1) for lfn in sorted(res['Value']['Failed'].keys()): message = res['Value']['Failed'][lfn] print('Error: failed to remove %s: %s' % (lfn, message)) print('Successfully removed %d catalog files.' % (len(res['Value']['Successful'])))
def getAuthorDNfromProxy(self): """ gets the AuthorDN and username of the transformation from the uploaded proxy """ username = "" author = "" res = getProxyInfo() if res['OK']: author = res['Value']['identity'] username = res['Value']['username'] else: gLogger.error("Unable to get uploaded proxy Info %s " % res['Message']) return S_ERROR(res['Message']) res = {'username': username, 'authorDN': author} return S_OK(res)
def getAuthorDNfromProxy(self): """gets the AuthorDN and username of the transformation from the uploaded proxy""" username = "" author = "" res = getProxyInfo() if res["OK"]: author = res["Value"]["identity"] username = res["Value"]["username"] else: gLogger.error("Unable to get uploaded proxy Info %s " % res["Message"]) return S_ERROR(res["Message"]) res = {"username": username, "authorDN": author} return S_OK(res)
def __init__(self, script=None): super(UserJob, self).__init__(script) self.type = 'User' self.diracinstance = None self.usergroup = ['ilc_user', 'calice_user'] self.proxyinfo = getProxyInfo() ########## SPLITTING STUFF: ATTRIBUTES ########## self._data = [] self.splittingOption = None self._switch = {} self.numberOfJobs = None self.totalNumberOfEvents = None self.eventsPerJob = None self.numberOfFilesPerJob = 1
def checkCredentials(self, requested_group): """ Utility method to check the credentials and that the use belongs to a specific group """ # First locate the DIRAC code try: from DIRAC.Core.Security.ProxyInfo import getProxyInfo except: raise DiracNotFoundException("DIRAC not found") # Now check the credentials themselves # Now using dirac to lookup the proxy info try: info = getProxyInfo() except Exception, e: raise InvalidGridCredentialsException(str(e))
def main(): Script.parseCommandLine() from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations allowUsers = Operations().getValue( "DataManagement/AllowUserReplicaManagement", False) from DIRAC.Core.Security.ProxyInfo import getProxyInfo res = getProxyInfo() if not res['OK']: gLogger.fatal("Can't get proxy info", res['Message']) dexit(1) properties = res['Value'].get('groupProperties', []) if not allowUsers: if 'FileCatalogManagement' not in properties: gLogger.error( "You need to use a proxy from a group with FileCatalogManagement" ) dexit(5) from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() args = Script.getPositionalArgs() if len(args) < 2: Script.showHelp(exitCode=1) else: inputFileName = args[0] storageElementName = args[1] if os.path.exists(inputFileName): inputFile = open(inputFileName, 'r') string = inputFile.read() lfns = [lfn.strip() for lfn in string.splitlines()] inputFile.close() else: lfns = [inputFileName] res = dm.removeReplicaFromCatalog(storageElementName, lfns) if not res['OK']: print(res['Message']) dexit(0) for lfn in sorted(res['Value']['Failed']): message = res['Value']['Failed'][lfn] print('Failed to remove %s replica of %s: %s' % (storageElementName, lfn, message)) print('Successfully remove %d catalog replicas at %s' % (len(res['Value']['Successful']), storageElementName))
def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"): # Warm up local CS CSHelpers.warmUp() if rStatus is None: self.rStatus = ResourceStatusClient() if rManagement is None: self.rManagement = ResourceManagementClient() self.defaultStatus = defaultStatus self.rssConfig = RssConfiguration() self.tokenOwner = "rs_svc" result = getProxyInfo() if result['OK']: self.tokenOwner = result['Value']['username']
def _getCurrentUser(self): """Get current user :return: S_OK(dict)/S_ERROR() """ res = getProxyInfo(False, False) if not res["OK"]: return self._errorReport("No proxy found in local environment", res["Message"]) proxyInfo = res["Value"] gLogger.debug(formatProxyInfoAsString(proxyInfo)) if "group" not in proxyInfo: return self._errorReport("Proxy information does not contain the group", res["Message"]) res = getDNForUsername(proxyInfo["username"]) if not res["OK"]: return self._errorReport("Failed to get proxies for user", res["Message"]) return S_OK(proxyInfo["username"])
def submitTasks( self ): """ Submit the tasks to an external system, using the taskManager provided """ gLogger.info( "submitTasks: Submitting tasks for transformations" ) res = getProxyInfo( False, False ) if not res['OK']: gLogger.error( "submitTasks: Failed to determine credentials for submission", res['Message'] ) return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] gLogger.info( "submitTasks: Tasks will be submitted with the credentials %s:%s" % ( owner, ownerGroup ) ) # Get the transformations which should be submitted tasksPerLoop = self.am_getOption( 'TasksPerLoop', 50 ) status = self.am_getOption( 'SubmitStatus', ['Active', 'Completing'] ) res = self._selectTransformations( transType = self.transType, status = status ) if not res['OK']: return res for transformation in res['Value']: transID = transformation['TransformationID'] transBody = transformation['Body'] res = self.transClient.getTasksToSubmit( transID, tasksPerLoop ) if not res['OK']: gLogger.error( "submitTasks: Failed to obtain tasks for transformation", "%s %s" % ( transID, res['Message'] ) ) continue tasks = res['Value']['JobDictionary'] if not tasks: gLogger.verbose( "submitTasks: No tasks found for submission for transformation %s" % transID ) continue gLogger.info( "submitTasks: Obtained %d tasks for submission for transformation %s" % ( len( tasks ), transID ) ) res = self.taskManager.prepareTransformationTasks( transBody, tasks, owner, ownerGroup ) if not res['OK']: gLogger.error( "submitTasks: Failed to prepare tasks for transformation", "%s %s" % ( transID, res['Message'] ) ) continue res = self.taskManager.submitTransformationTasks( res['Value'] ) if not res['OK']: gLogger.error( "submitTasks: Failed to submit prepared tasks for transformation", "%s %s" % ( transID, res['Message'] ) ) continue res = self.taskManager.updateDBAfterTaskSubmission( res['Value'] ) if not res['OK']: gLogger.error( "submitTasks: Failed to update DB after task submission for transformation", "%s %s" % ( transID, res['Message'] ) ) continue gLogger.info( "submitTasks: Submission of transformation tasks complete" ) return S_OK()
def _prepareProxy(self): """Set the environment variable X509_USER_PROXY""" if not self.proxy: result = getProxyInfo() if not result["OK"]: return S_ERROR("No proxy available") if "path" in result["Value"]: os.environ["X509_USER_PROXY"] = result["Value"]["path"] return S_OK() else: result = gProxyManager.dumpProxyToFile(self.proxy, requiredTimeLeft=self.minProxyTime) if not result["OK"]: return result os.environ["X509_USER_PROXY"] = result["Value"] self.log.debug("Set proxy variable X509_USER_PROXY to %s" % os.environ["X509_USER_PROXY"]) return S_OK()
def _getCurrentOwner(self): """Simple function to return current DIRAC username. """ if 'OwnerName' in self.workflow_commons: return self.workflow_commons['OwnerName'] result = getProxyInfo() if not result['OK']: if not self._enableModule(): return 'testUser' raise RuntimeError('Could not obtain proxy information') if 'username' not in result['Value']: raise RuntimeError('Could not get username from proxy') return result['Value']['username']
def __init__(self, script=None, stdout='std.out', stderr='std.err'): """Instantiates the Workflow object and some default parameters. """ super(Job, self).__init__() self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 vo = '' ret = getProxyInfo(disableVOMS=True) if ret['OK'] and 'group' in ret['Value']: vo = getVOForGroup(ret['Value']['group']) self.group = vo self.site = None # self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified # $DIRACROOT is set by the JobWrapper at execution time self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] # #Add member to handle Parametric jobs self.numberOfParameters = 0 self.parameterSeqs = {} self.wfArguments = {} self.parametricWFArguments = {} # loading the function that will be used to determine the platform (it can be VO specific) res = ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", 'getDIRACPlatforms') if not res['OK']: self.log.fatal(res['Message']) self.getDIRACPlatforms = res['Value'] self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow(script) self._siteSet = set(getSites().get('Value', []))
def monitorUninstallation(system, component, cpu=None, hostname=None): """ Register the uninstallation of a component in the ComponentMonitoringDB """ monitoringClient = ComponentMonitoringClient() # Retrieve user uninstalling the component user = None result = getProxyInfo() if result["OK"]: proxyInfo = result["Value"] if "username" in proxyInfo: user = proxyInfo["username"] else: return result if not user: user = "******" if not cpu: cpu = "Not available" for line in open("/proc/cpuinfo"): if line.startswith("model name"): cpu = line.split(":")[1][0:64] cpu = cpu.replace("\n", "").lstrip().rstrip() if not hostname: hostname = socket.getfqdn() instance = component[0:32] result = monitoringClient.updateInstallations( { "Instance": instance, "UnInstallationTime": None }, {"DIRACSystem": system}, { "HostName": hostname, "CPU": cpu }, { "UnInstallationTime": datetime.datetime.utcnow(), "UnInstalledBy": user }, ) return result
def submitJob(self, executableFile, proxy, **kwargs): """ Method to submit job. """ self.pPool.processResults() coresInUse = self.getCoresInUse() if "WholeNode" in kwargs and kwargs['WholeNode']: if coresInUse > 0: return S_ERROR( 'Can not take WholeNode job' ) #, %d/%d slots used' % (self.slotsInUse,self.slots) ) else: requestedCores = self.cores elif "NumberOfCores" in kwargs: requestedCores = int(kwargs['NumberOfCores']) if requestedCores > 0: if (coresInUse + requestedCores) > self.cores: return S_ERROR( 'Not enough slots: requested %d, available %d' % (requestedCores, self.cores - coresInUse)) else: requestedCores = 1 if self.cores - coresInUse < requestedCores: return S_ERROR('Not enough slots: requested %d, available %d' % (requestedCores, self.cores - coresInUse)) ret = getProxyInfo() if not ret['OK']: pilotProxy = None else: pilotProxy = ret['Value']['path'] self.log.notice('Pilot Proxy:', pilotProxy) result = self.pPool.createAndQueueTask( executeJob, [executableFile, proxy, self.taskID], None, self.taskID, usePoolCallbacks=True) self.taskID += 1 self.coresPerTask[self.taskID] = requestedCores self.pPool.processResults() return result