def test_01_parse( self ): """ pfnparse and pfnparse_old :param self: self reference """ for pfn, result in self.pfns.items(): self.assertEqual( pfnparse( pfn )['OK'], result['OK'] ) self.assertEqual( pfnparse( pfn ).get('Errno'), result.get('Errno') )
def getJobStatus(self, jobIDList): """ Get status of the jobs in the given list """ hostDict = {} for job in jobIDList: result = pfnparse(job) if not result['OK']: continue host = result['Value']['Host'] hostDict.setdefault(host, []) hostDict[host].append(job) resultDict = {} failed = [] for host, jobIDList in hostDict.items(): result = self._getJobStatusOnHost(jobIDList, host) if not result['OK']: failed.extend(jobIDList) continue resultDict.update(result['Value']) for job in failed: if not job in resultDict: resultDict[job] = 'Unknown' return S_OK(resultDict)
def __generatePfnDict( self, lfns, storage ): """ Generates a dictionnary (pfn : lfn ), where the pfn are constructed from the lfn using the getProtocolPfn method of the storage plugins. :param: lfns : dictionnary {lfn:whatever} :returns dictionnary {constructed pfn : lfn} """ self.log.verbose( "StorageElement.__generatePfnDict: generating pfn dict for %s lfn in %s." % ( len( lfns ), self.name ) ) pfnDict = {} # pfn : lfn failed = {} # lfn : string with errors for lfn in lfns: if ":" in lfn: errStr = "StorageElement.__generatePfnDict: received a pfn as input. It should not happen anymore, please check your code" self.log.verbose( errStr, lfn ) res = pfnparse( lfn ) # pfnparse can take an lfn as input, it will just fill the path and filename if not res['OK']: errStr = "StorageElement.__generatePfnDict: Failed to parse supplied LFN." self.log.debug( errStr, "%s: %s" % ( lfn, res['Message'] ) ) if lfn not in failed: failed[lfn] = '' failed[lfn] = "%s %s" % ( failed[lfn], errStr ) else: res = storage.getProtocolPfn( res['Value'], True ) if not res['OK']: errStr = "StorageElement.__generatePfnDict %s." % res['Message'] self.log.debug( errStr, 'for %s' % ( lfn ) ) if lfn not in failed: failed[lfn] = '' failed[lfn] = "%s %s" % ( failed[lfn], errStr ) else: pfnDict[res['Value']] = lfn res = S_OK( pfnDict ) res['Failed'] = failed return res
def __generatePfnDict(self, pfns, storage): """ whatever, it creates PFN dict """ pfnDict = {} failed = {} for pfn in pfns: res = pfnparse(pfn) if not res['OK']: errStr = "__generatePfnDict: Failed to parse supplied PFN." self.log.error(errStr, "%s: %s" % (pfn, res['Message'])) if pfn not in failed: failed[pfn] = '' failed[pfn] = "%s %s" % (failed[pfn], errStr) else: res = storage.getProtocolPfn(res['Value'], True) if not res['OK']: errStr = "__generatePfnDict %s." % res['Message'] self.log.error(errStr, 'for %s' % (pfn)) if pfn not in failed: failed[pfn] = '' failed[pfn] = "%s %s" % (failed[pfn], errStr) else: pfnDict[res['Value']] = pfn res = S_OK(pfnDict) res['Failed'] = failed return res
def __putFile( self, src_file, dest_url ): res = pfnparse( src_file ) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: localCache = True srcSEURL = srcDict['Protocol'] + '://' + srcDict['Host'] + ':' + srcDict['Port'] + srcDict['WSUrl'] transferClient = TransferClient( srcSEURL ) res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) ) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists( src_file ): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) sourceSize = getSize( src_file ) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) transferClient = TransferClient( self.url ) res = transferClient.sendFile( src_file, dest_url, token = self.checkSum ) if localCache: os.unlink( src_file ) if res['OK']: return S_OK( sourceSize ) else: return res
def killJob( self, jobIDs ): """ Kill specified jobs """ jobIDList = list( jobIDs ) if type( jobIDs ) == type( ' ' ): jobIDList = [jobIDs] hostDict = {} for job in jobIDList: result = pfnparse( job ) if not result['OK']: continue host = result['Value']['Host'] hostDict.setdefault(host,[]) hostDict[host].append( job ) failed = [] for host,jobIDList in hostDict.items(): result = self._killJobOnHost( jobIDList, host ) if not result['OK']: failed.extend( jobIDList ) message = result['Message'] if failed: result = S_ERROR(message) result['Failed'] = failed else: result = S_OK() return result
def getPfnPath(self, pfn): """ Get the part of the PFN path below the basic storage path. This path must coincide with the LFN of the file in order to be compliant with the LHCb conventions. """ if not self.valid: return S_ERROR(self.errorReason) res = pfnparse(pfn) if not res['OK']: return res fullPfnPath = '%s/%s' % (res['Value']['Path'], res['Value']['FileName']) # Check all available storages and check whether the pfn is for that protocol pfnPath = '' for storage in self.storages: res = storage.isPfnForProtocol(pfn) if res['OK']: if res['Value']: res = storage.getParameters() saPath = res['Value']['Path'] if not saPath: # If the sa path doesn't exist then the pfn path is the entire string pfnPath = fullPfnPath else: if re.search(saPath, fullPfnPath): # Remove the sa path from the fullPfnPath pfnPath = fullPfnPath.replace(saPath, '') if pfnPath: return S_OK(pfnPath) # This should never happen. DANGER!! errStr = "StorageElement.getPfnPath: Failed to get the pfn path for any of the protocols!!" gLogger.error(errStr) return S_ERROR(errStr)
def __getURLPath(self, url): """ Get the part of the URL path below the basic storage path. This path must coincide with the LFN of the file in order to be compliant with the DIRAC conventions. """ log = self.log.getSubLogger('__getURLPath') log.verbose("Getting path from url in %s." % self.name) if not self.valid: return S_ERROR(self.errorReason) res = pfnparse(url) if not res['OK']: return res fullURLPath = '%s/%s' % (res['Value']['Path'], res['Value']['FileName']) # Check all available storages and check whether the url is for that protocol urlPath = '' for storage in self.storages: res = storage.isNativeURL(url) if res['OK']: if res['Value']: parameters = storage.getParameters() saPath = parameters['Path'] if not saPath: # If the sa path doesn't exist then the url path is the entire string urlPath = fullURLPath else: if re.search(saPath, fullURLPath): # Remove the sa path from the fullURLPath urlPath = fullURLPath.replace(saPath, '') if urlPath: return S_OK(urlPath) # This should never happen. DANGER!! errStr = "Failed to get the url path for any of the protocols!!" log.debug(errStr) return S_ERROR(errStr)
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: localCache = True srcSEURL = srcDict['Protocol'] + '://' + srcDict[ 'Host'] + ':' + srcDict['Port'] + srcDict['WSUrl'] transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join(srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def _getKeyFromURL(self, url): """Extract the Key from the URL. The key is basically the LFN without trailing slash I despise such path mangling, expecially after all the efforts to get ride of such method. However, since in the case of S3 we need to go back and forth between URL and LFN (for finding keys, checking accesses in the gw, etc), there is no other option... :param url: s3 url :returns: S_OK(key) / S_ERROR """ res = pfnparse(url, srmSpecific=False) if not res["OK"]: return res splitURL = res["Value"] # The path originally looks like '/bucket/lhcb/user/c/chaen # We remove the trailing slash, and get the relative path # of bucket/lhcb/user/c/chaen starting from bucket, # which gives you basically the LFN without trailing slash path = os.path.relpath(splitURL["Path"].lstrip("/"), start=self.bucketName) key = os.path.join(path, splitURL["FileName"]) return S_OK(key)
def getPfnPath( self, pfn ): """ Get the part of the PFN path below the basic storage path. This path must coincide with the LFN of the file in order to be compliant with the LHCb conventions. """ if not self.valid: return S_ERROR( self.errorReason ) res = pfnparse( pfn ) if not res['OK']: return res fullPfnPath = '%s/%s' % ( res['Value']['Path'], res['Value']['FileName'] ) # Check all available storages and check whether the pfn is for that protocol pfnPath = '' for storage in self.storages: res = storage.isPfnForProtocol( pfn ) if res['OK']: if res['Value']: res = storage.getParameters() saPath = res['Value']['Path'] if not saPath: # If the sa path doesn't exist then the pfn path is the entire string pfnPath = fullPfnPath else: if re.search( saPath, fullPfnPath ): # Remove the sa path from the fullPfnPath pfnPath = fullPfnPath.replace( saPath, '' ) if pfnPath: return S_OK( pfnPath ) # This should never happen. DANGER!! errStr = "getPfnPath: Failed to get the pfn path for any of the protocols!!" self.log.error( errStr ) return S_ERROR( errStr )
def getJobStatus ( self, jobIDList ): """ Get status of the jobs in the given list """ hostDict = {} for job in jobIDList: result = pfnparse( job ) if not result['OK']: continue host = result['Value']['Host'] hostDict.setdefault(host,[]) hostDict[host].append( job ) resultDict = {} failed = [] for host,jobIDList in hostDict.items(): result = self._getJobStatusOnHost( jobIDList, host ) if not result['OK']: failed.extend( jobIDList ) continue resultDict.update( result['Value'] ) for job in failed: if not job in resultDict: resultDict[job] = 'Unknown' return S_OK( resultDict )
def killJob(self, jobIDs): """ Kill specified jobs """ jobIDList = list(jobIDs) if type(jobIDs) == type(' '): jobIDList = [jobIDs] hostDict = {} for job in jobIDList: result = pfnparse(job) if not result['OK']: continue host = result['Value']['Host'] hostDict.setdefault(host, []) hostDict[host].append(job) failed = [] for host, jobIDList in hostDict.items(): result = self._killJobOnHost(jobIDList, host) if not result['OK']: failed.extend(jobIDList) message = result['Message'] if failed: result = S_ERROR(message) result['Failed'] = failed else: result = S_OK() return result
def __generatePfnDict( self, pfns, storage ): """ whatever, it creates PFN dict """ pfnDict = {} failed = {} for pfn in pfns: res = pfnparse( pfn ) if not res['OK']: errStr = "__generatePfnDict: Failed to parse supplied PFN." self.log.error( errStr, "%s: %s" % ( pfn, res['Message'] ) ) if pfn not in failed: failed[pfn] = '' failed[pfn] = "%s %s" % ( failed[pfn], errStr ) else: res = storage.getProtocolPfn( res['Value'], True ) if not res['OK']: errStr = "__generatePfnDict %s." % res['Message'] self.log.error( errStr, 'for %s' % ( pfn ) ) if pfn not in failed: failed[pfn] = '' failed[pfn] = "%s %s" % ( failed[pfn], errStr ) else: pfnDict[res['Value']] = pfn res = S_OK( pfnDict ) res['Failed'] = failed return res
def test_01_parse(self): """ pfnparse and pfnparse_old :param self: self reference """ for pfn, result in self.pfns.items(): self.assertEqual(pfnparse(pfn), result)
def test_01_parse( self ): """ pfnparse and pfnparse_old :param self: self reference """ for pfn, result in self.pfns.items(): self.assertEqual( pfnparse( pfn ), result )
def _getJobStatusOnHost(self, jobIDList, host=None): """ Get the status information for the given list of jobs """ # self.log.verbose( '*** getUnitJobStatus %s - %s\n' % ( jobIDList, host) ) resultDict = {} ssh = SSH(host=host, parameters=self.ceParameters) jobDict = {} for job in jobIDList: result = pfnparse(job) if result["OK"]: stamp = result["Value"]["FileName"] else: self.log.error("Invalid job id", job) continue jobDict[stamp] = job stampList = jobDict.keys() cmd = "bash --login -c '%s/%s job_status %s %s %s'" % ( self.sharedArea, self.controlScript, "#".join(stampList), self.infoArea, self.ceParameters["SSHUser"], ) result = ssh.sshCall(30, cmd) if not result["OK"]: return result sshStatus = result["Value"][0] sshStdout = result["Value"][1] sshStderr = result["Value"][2] if sshStatus == 0: outputLines = sshStdout.strip().replace("\r", "").split("\n") try: index = outputLines.index("============= Start output ===============") outputLines = outputLines[index + 1 :] except: return S_ERROR("Invalid output from job get status: %s" % outputLines[0]) try: status = int(outputLines[0]) except: return S_ERROR("Failed local batch job status: %s" % outputLines[0]) if status != 0: message = "Unknown reason" if len(outputLines) > 1: message = outputLines[1] return S_ERROR("Failed job kill, reason: %s" % message) else: for line in outputLines[1:]: jbundle = line.split(":::") if len(jbundle) == 2: resultDict[jobDict[jbundle[0]]] = jbundle[1] else: return S_ERROR("\n".join([sshStdout, sshStderr])) # self.log.verbose( ' !!! getUnitJobStatus will return : %s\n' % resultDict ) return S_OK(resultDict)
def getGridSRMs( vo, bdiiInfo = None, srmBlackList = None, unUsed = False ): result = ldapService( serviceType = 'SRM', vo = vo ) if not result['OK']: return result srmBdiiDict = result['Value'] knownSRMs = set() if srmBlackList is not None: knownSRMs = knownSRMs.union( set( srmBlackList ) ) siteSRMDict = {} for srm in srmBdiiDict: endPoint = srm.get( 'GlueServiceEndpoint', '') srmHost = '' if endPoint: result = pfnparse( endPoint ) if not result['OK']: continue srmHost = result['Value']['Host'] if not srmHost: continue if srmHost in knownSRMs: continue if unUsed: result = getDIRACSesForSRM( srmHost ) if not result['OK']: return result diracSEs = result['Value'] if diracSEs: # If it is a known SRM and only new SRMs are requested, continue continue site = srm.get( 'GlueForeignKey', '' ).replace( 'GlueSiteUniqueID=', '' ) siteSRMDict.setdefault( site, {} ) siteSRMDict[site][srmHost] = srm if bdiiInfo is None: result = getBdiiSEInfo( vo ) if not result['OK']: return result seBdiiDict = result['Value'] else: seBdiiDict = bdiiInfo srmSeDict = {} for site in siteSRMDict: srms = siteSRMDict[site].keys() for srm in srms: if seBdiiDict.get( site, {} ).get( 'SEs', {} ).get( srm, {} ): srmSeDict.setdefault( site, {} ) srmSeDict[site].setdefault( srm, {} ) srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm] srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm] return S_OK( srmSeDict )
def _getJobStatus(self, jobIDList): """ Get the status information for the given list of jobs """ resultDict = {} jobDict = {} for job in jobIDList: result = pfnparse(job) if result['OK']: stamp = result['Value']['FileName'] else: self.log.error('Invalid job id', job) continue jobDict[stamp] = job stampList = jobDict.keys() cmdTuple = [ self.finalScript, 'job_status', '#'.join(stampList), self.infoArea, self.userName ] result = systemCall(10, cmdTuple) if not result['OK']: return result status = result['Value'][0] stdout = result['Value'][1] stderr = result['Value'][2] # Examine results of the job status if status == 0: outputLines = stdout.strip().replace('\r', '').split('\n') try: index = outputLines.index( '============= Start output ===============') outputLines = outputLines[index + 1:] except: return S_ERROR("Invalid output from CE get status: %s" % outputLines[0]) try: status = int(outputLines[0]) except: return S_ERROR("Failed to get CE status: %s" % outputLines[0]) if status != 0: message = "Unknown reason" if len(outputLines) > 1: message = outputLines[1] return S_ERROR('Failed to get CE status, reason: %s' % message) for line in outputLines[1:]: if ':::' in line: jbundle = line.split(':::') if (len(jbundle) == 2): resultDict[jobDict[jbundle[0]]] = jbundle[1] else: return S_ERROR('\n'.join([stdout, stderr])) return S_OK(resultDict)
def isPfnForProtocol(self, pfn): res = pfnparse(pfn) if not res['OK']: return res pfnDict = res['Value'] if pfnDict['Protocol'] == self.protocol: return S_OK(True) else: return S_OK(False)
def isPfnForProtocol( self, pfn ): res = pfnparse( pfn ) if not res['OK']: return res pfnDict = res['Value'] if pfnDict['Protocol'] == self.protocol: return S_OK( True ) else: return S_OK( False )
def _getJobStatusOnHost(self, jobIDList, host=None): """ Get the status information for the given list of jobs """ # self.log.verbose( '*** getUnitJobStatus %s - %s\n' % ( jobIDList, host) ) resultDict = {} ssh = SSH(host=host, parameters=self.ceParameters) jobDict = {} for job in jobIDList: result = pfnparse(job) if result['OK']: stamp = result['Value']['FileName'] else: self.log.error('Invalid job id', job) continue jobDict[stamp] = job stampList = jobDict.keys() cmd = "bash --login -c '%s/%s job_status %s %s %s'" % ( self.sharedArea, self.controlScript, '#'.join(stampList), self.infoArea, self.ceParameters['SSHUser']) result = ssh.sshCall(10, cmd) if not result['OK']: return result sshStatus = result['Value'][0] sshStdout = result['Value'][1] sshStderr = result['Value'][2] if sshStatus == 0: outputLines = sshStdout.strip().replace('\r', '').split('\n') try: index = outputLines.index( '============= Start output ===============') outputLines = outputLines[index + 1:] except: return S_ERROR("Invalid output from job get status: %s" % outputLines[0]) try: status = int(outputLines[0]) except: return S_ERROR("Failed local batch job status: %s" % outputLines[0]) if status != 0: message = "Unknown reason" if len(outputLines) > 1: message = outputLines[1] return S_ERROR('Failed job kill, reason: %s' % message) else: for line in outputLines[1:]: jbundle = line.split(':::') if (len(jbundle) == 2): resultDict[jobDict[jbundle[0]]] = jbundle[1] else: return S_ERROR('\n'.join([sshStdout, sshStderr])) # self.log.verbose( ' !!! getUnitJobStatus will return : %s\n' % resultDict ) return S_OK(resultDict)
def isPfnForProtocol(self, pfn): res = pfnparse(pfn) if not res["OK"]: return res pfnDict = res["Value"] if pfnDict["Protocol"] == self.protocol: return S_OK(True) else: return S_OK(False)
def test_02_default_parse( self ): """ pfnparse and pfnparse_old :param self: self reference """ for pfn, result in self.default_pfns.items(): parseResult = pfnparse( pfn, srmSpecific = False ) self.assertEqual( parseResult['OK'], result['OK'] ) if result['OK']: self.assertEqual( parseResult['Value'], result['Value'] )
def isNativeURL(self, url): """Check if URL :url: is valid for :self.protocol: :param self: self reference :param str url: URL """ res = pfnparse(url, srmSpecific=self.srmSpecificParse) if not res["OK"]: return res urlDict = res["Value"] return S_OK(urlDict["Protocol"] == self.protocolParameters["Protocol"])
def test_02_default_parse( self ): """ pfnparse and pfnparse_old :param self: self reference """ for pfn, result in self.default_pfns.iteritems(): parseResult = pfnparse( pfn, srmSpecific = False ) self.assertEqual( parseResult['OK'], result['OK'] ) if result['OK']: self.assertEqual( parseResult['Value'], result['Value'] )
def isNativeURL(self, url): """ Check if URL :url: is valid for :self.protocol: :param self: self reference :param str url: URL """ res = pfnparse(url) if not res['OK']: return res urlDict = res['Value'] return S_OK(urlDict['Protocol'] == self.protocolParameters['Protocol'])
def isNativeURL( self, url ): """ Check if URL :url: is valid for :self.protocol: :param self: self reference :param str url: URL """ res = pfnparse( url, srmSpecific = self.srmSpecificParse ) if not res['OK']: return res urlDict = res['Value'] return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] )
def test_01_srm_parse( self ): """ pfnparse and pfnparse_old :param self: self reference """ for pfn, result in self.srm_pfns.items(): parseResult = pfnparse( pfn ) self.assertEqual( parseResult['OK'], result['OK'] ) if result['OK']: self.assertEqual( parseResult['Value'], result['Value'] )
def test_01_srm_parse( self ): """ pfnparse and pfnparse_old :param self: self reference """ for pfn, result in self.srm_pfns.iteritems(): parseResult = pfnparse( pfn ) self.assertEqual( parseResult['OK'], result['OK'] ) if result['OK']: self.assertEqual( parseResult['Value'], result['Value'] )
def _getJobStatus( self, jobIDList ): """ Get the status information for the given list of jobs """ resultDict = {} jobDict = {} for job in jobIDList: result = pfnparse( job ) if result['OK']: stamp = result['Value']['FileName'] else: self.log.error( 'Invalid job id', job ) continue jobDict[stamp] = job stampList = jobDict.keys() cmdTuple = [ self.finalScript, 'job_status', '#'.join( stampList ), self.infoArea, self.userName ] result = systemCall( 10, cmdTuple ) if not result['OK']: return result status = result['Value'][0] stdout = result['Value'][1] stderr = result['Value'][2] # Examine results of the job status if status == 0: outputLines = stdout.strip().replace( '\r', '' ).split( '\n' ) try: index = outputLines.index( '============= Start output ===============' ) outputLines = outputLines[index + 1:] except: return S_ERROR( "Invalid output from CE get status: %s" % outputLines[0] ) try: status = int( outputLines[0] ) except: return S_ERROR( "Failed to get CE status: %s" % outputLines[0] ) if status != 0: message = "Unknown reason" if len( outputLines ) > 1: message = outputLines[1] return S_ERROR( 'Failed to get CE status, reason: %s' % message ) for line in outputLines[1:]: if ':::' in line: jbundle = line.split( ':::' ) if ( len( jbundle ) == 2 ): resultDict[jobDict[jbundle[0]]] = jbundle[1] else: return S_ERROR( '\n'.join( [stdout, stderr] ) ) return S_OK( resultDict )
def _getJobOutputFiles( self, jobID ): """ Get output file names for the specific CE """ result = pfnparse( jobID ) if not result['OK']: return result jobStamp = result['Value']['FileName'] host = result['Value']['Host'] output = '%s/DIRACPilot.o%s' % ( self.batchOutput, jobStamp ) error = '%s/DIRACPilot.e%s' % ( self.batchError, jobStamp ) return S_OK( (jobStamp,host,output,error) )
def _getJobOutputFiles(self, jobID): """ Get output file names for the specific CE """ result = pfnparse(jobID) if not result['OK']: return result jobStamp = result['Value']['FileName'] host = result['Value']['Host'] output = '%s/DIRACPilot.o%s' % (self.batchOutput, jobStamp) error = '%s/DIRACPilot.e%s' % (self.batchError, jobStamp) return S_OK((jobStamp, host, output, error))
def _getJobOutputFiles(self, jobID): """ Get output file names for the specific CE """ result = pfnparse(jobID) if not result["OK"]: return result jobStamp = result["Value"]["FileName"] host = result["Value"]["Host"] output = "%s/%s.out" % (self.batchOutput, jobStamp) error = "%s/%s.err" % (self.batchError, jobStamp) return S_OK((jobStamp, host, output, error))
def _killJobOnHost(self, jobIDList, host=None): """ Kill the jobs for the given list of job IDs """ resultDict = {} ssh = SSH(host=host, parameters=self.ceParameters) jobDict = {} for job in jobIDList: result = pfnparse(job) if result['OK']: stamp = result['Value']['FileName'] else: self.log.error('Invalid job id', job) continue jobDict[stamp] = job stampList = jobDict.keys() cmd = "bash --login -c '%s/%s kill_job %s %s'" % ( self.sharedArea, self.controlScript, '#'.join(stampList), self.infoArea) result = ssh.sshCall(10, cmd) if not result['OK']: return result sshStatus = result['Value'][0] sshStdout = result['Value'][1] sshStderr = result['Value'][2] # Examine results of the job submission if sshStatus == 0: outputLines = sshStdout.strip().replace('\r', '').split('\n') try: index = outputLines.index( '============= Start output ===============') outputLines = outputLines[index + 1:] except: return S_ERROR("Invalid output from job kill: %s" % outputLines[0]) try: status = int(outputLines[0]) except: return S_ERROR("Failed local batch job kill: %s" % outputLines[0]) if status != 0: message = "Unknown reason" if len(outputLines) > 1: message = outputLines[1] return S_ERROR('Failed job kill, reason: %s' % message) else: return S_ERROR('\n'.join([sshStdout, sshStderr])) return S_OK()
def _killJobOnHost(self, jobIDList, host=None): """ Kill the jobs for the given list of job IDs """ resultDict = {} ssh = SSH(host=host, parameters=self.ceParameters) jobDict = {} for job in jobIDList: result = pfnparse(job) if result["OK"]: stamp = result["Value"]["FileName"] else: self.log.error("Invalid job id", job) continue jobDict[stamp] = job stampList = jobDict.keys() cmd = "bash --login -c '%s/%s kill_job %s %s'" % ( self.sharedArea, self.controlScript, "#".join(stampList), self.infoArea, ) result = ssh.sshCall(10, cmd) if not result["OK"]: return result sshStatus = result["Value"][0] sshStdout = result["Value"][1] sshStderr = result["Value"][2] # Examine results of the job submission if sshStatus == 0: outputLines = sshStdout.strip().replace("\r", "").split("\n") try: index = outputLines.index("============= Start output ===============") outputLines = outputLines[index + 1 :] except: return S_ERROR("Invalid output from job kill: %s" % outputLines[0]) try: status = int(outputLines[0]) except: return S_ERROR("Failed local batch job kill: %s" % outputLines[0]) if status != 0: message = "Unknown reason" if len(outputLines) > 1: message = outputLines[1] return S_ERROR("Failed job kill, reason: %s" % message) else: return S_ERROR("\n".join([sshStdout, sshStderr])) return S_OK()
def _killJobs(self, jobIDList, host=None): """ Kill the jobs for the given list of job IDs """ resultDict = {} jobDict = {} for job in jobIDList: result = pfnparse(job) if result['OK']: stamp = result['Value']['FileName'] else: self.log.error('Invalid job id', job) continue jobDict[stamp] = job stampList = jobDict.keys() cmdTuple = [ self.finalScript, 'kill_job', '#'.join(stampList), self.infoArea ] result = systemCall(10, cmdTuple) if not result['OK']: return result status = result['Value'][0] stdout = result['Value'][1] stderr = result['Value'][2] # Examine results of the job submission if status != 0: outputLines = stdout.strip().replace('\r', '').split('\n') try: index = outputLines.index( '============= Start output ===============') outputLines = outputLines[index + 1:] except: return S_ERROR("Invalid output from kill Job: %s" % outputLines[0]) try: status = int(outputLines[0]) except: return S_ERROR("Failed to kill Job: %s" % outputLines[0]) if status != 0: message = "Unknown reason" if len(outputLines) > 1: message = outputLines[1] return S_ERROR('Failed to kill Job, reason: %s' % message) return S_ERROR('\n'.join([stdout, stderr])) return S_OK()
def __addDoubleSlash(self, res): """ Utilities to add the double slash between the host(:port) and the path :param res: DIRAC return structure which contains an URL if S_OK :return: DIRAC structure with corrected URL """ if not res['OK']: return res url = res['Value'] res = pfnparse(url, srmSpecific=self.srmSpecificParse) if not res['OK']: return res urlDict = res['Value'] urlDict['Path'] = '/' + urlDict['Path'] return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def __addDoubleSlash( self, res ): """ Utilities to add the double slash between the host(:port) and the path :param res: DIRAC return structure which contains an URL if S_OK :return: DIRAC structure with corrected URL """ if not res['OK']: return res url = res['Value'] res = pfnparse( url, srmSpecific = self.srmSpecificParse ) if not res['OK']: return res urlDict = res['Value'] urlDict['Path'] = '/' + urlDict['Path'] return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
def updateURL(self, url, withWSUrl=False): """Update the URL according to the current SE parameters""" result = pfnparse(url, srmSpecific=self.srmSpecificParse) if not result["OK"]: return result urlDict = result["Value"] urlDict["Protocol"] = self.protocolParameters["Protocol"] urlDict["Host"] = self.protocolParameters["Host"] urlDict["Port"] = self.protocolParameters["Port"] urlDict["WSUrl"] = "" if withWSUrl: urlDict["WSUrl"] = self.protocolParameters["WSUrl"] return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def getJobStatus(self, jobIDList): """ Get the status information for the given list of jobs """ resultDict = {} ssh = SSH(parameters=self.ceParameters) for jobList in breakListIntoChunks(jobIDList, 100): jobDict = {} for job in jobList: result = pfnparse(job) if result['OK']: stamp = result['Value']['FileName'].split('.')[0] else: self.log.error('Invalid job id', job) continue jobDict[stamp] = job stampList = jobDict.keys() cmd = ['qstat', ' '.join(stampList)] result = ssh.sshCall(10, cmd) if not result['OK']: return result status = result['Value'][0] if status == -1: return S_ERROR('Timeout while SSH call') elif status != 0: return S_ERROR('Error while SSH call') output = result['Value'][1].replace('\r', '') lines = output.split('\n') for job in jobDict: resultDict[jobDict[job]] = 'Unknown' for line in lines: if line.find(job) != -1: if line.find('Unknown') != -1: resultDict[jobDict[job]] = 'Unknown' else: torqueStatus = line.split()[4] if torqueStatus in ['E', 'C']: resultDict[jobDict[job]] = 'Done' elif torqueStatus in ['R']: resultDict[jobDict[job]] = 'Running' elif torqueStatus in ['S', 'W', 'Q', 'H', 'T']: resultDict[jobDict[job]] = 'Waiting' return S_OK(resultDict)
def getJobStatus( self, jobIDList ): """ Get the status information for the given list of jobs """ resultDict = {} ssh = SSH( parameters = self.ceParameters ) for jobList in breakListIntoChunks( jobIDList, 100 ): jobDict = {} for job in jobList: result = pfnparse( job ) if result['OK']: stamp = result['Value']['FileName'].split('.')[0] else: self.log.error( 'Invalid job id', job ) continue jobDict[stamp] = job stampList = jobDict.keys() cmd = [ 'qstat', ' '.join( stampList ) ] result = ssh.sshCall( 10, cmd ) if not result['OK']: return result status = result['Value'][0] if status == -1: return S_ERROR( 'Timeout while SSH call' ) elif status != 0: return S_ERROR( 'Error while SSH call' ) output = result['Value'][1].replace( '\r', '' ) lines = output.split( '\n' ) for job in jobDict: resultDict[jobDict[job]] = 'Unknown' for line in lines: if line.find( job ) != -1: if line.find( 'Unknown' ) != -1: resultDict[jobDict[job]] = 'Unknown' else: torqueStatus = line.split()[4] if torqueStatus in ['E', 'C']: resultDict[jobDict[job]] = 'Done' elif torqueStatus in ['R']: resultDict[jobDict[job]] = 'Running' elif torqueStatus in ['S', 'W', 'Q', 'H', 'T']: resultDict[jobDict[job]] = 'Waiting' return S_OK( resultDict )
def updateURL( self, url, withWSUrl = False ): """ Update the URL according to the current SE parameters """ result = pfnparse( url, srmSpecific = self.srmSpecificParse ) if not result['OK']: return result urlDict = result['Value'] urlDict['Protocol'] = self.protocolParameters['Protocol'] urlDict['Host'] = self.protocolParameters['Host'] urlDict['Port'] = self.protocolParameters['Port'] urlDict['WSUrl'] = '' if withWSUrl: urlDict['WSUrl'] = self.protocolParameters['WSUrl'] return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
def _isInputURL( self, url ): """ Check if the given url can be taken as input :param self: self reference :param str url: URL """ res = pfnparse( url ) if not res['OK']: return res urlDict = res['Value'] # Special case of 'file' protocol which can be just a URL if not urlDict['Protocol'] and 'file' in self.protocolParameters['InputProtocols']: return S_OK( True ) return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] )
def updateURL(self, url, withWSUrl=False): """ Update the URL according to the current SE parameters """ result = pfnparse(url, srmSpecific=self.srmSpecificParse) if not result['OK']: return result urlDict = result['Value'] urlDict['Protocol'] = self.protocolParameters['Protocol'] urlDict['Host'] = self.protocolParameters['Host'] urlDict['Port'] = self.protocolParameters['Port'] urlDict['WSUrl'] = '' if withWSUrl: urlDict['WSUrl'] = self.protocolParameters['WSUrl'] return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def _isInputURL(self, url): """ Check if the given url can be taken as input :param self: self reference :param str url: URL """ res = pfnparse(url) if not res['OK']: return res urlDict = res['Value'] # Special case of 'file' protocol which can be just a URL if not urlDict['Protocol'] and 'file' in self.protocolParameters['InputProtocols']: return S_OK(True) return S_OK(urlDict['Protocol'] == self.protocolParameters['Protocol'])
def __getSinglePfnForProtocol(self, pfn, protocol, withPort=True): """ Transform the input pfn into a pfn with the given protocol for the Storage Element. :param pfn : input PFN :param protocol : string or list of string of the protocol we want :param withPort : includes the port in the returned pfn """ self.log.verbose( "StorageElement.getSinglePfnForProtocol: Getting pfn for given protocols in %s." % self.name) # This test of the available protocols could actually be done in getPfnForProtocol once for all # but it is safer to put it here in case we decide to call this method internally (which I doubt!) res = self.getProtocols() if not res['OK']: return res if type(protocol) == StringType: protocols = [protocol] elif type(protocol) == ListType: protocols = protocol else: errStr = "StorageElement.getSinglePfnForProtocol: Supplied protocol must be string or list of strings." self.log.debug(errStr, "%s %s" % (protocol, self.name)) return S_ERROR(errStr) availableProtocols = res['Value'] protocolsToTry = [] for protocol in protocols: if protocol in availableProtocols: protocolsToTry.append(protocol) else: errStr = "StorageElement.getSinglePfnForProtocol: Requested protocol not available for SE." self.log.debug(errStr, '%s for %s' % (protocol, self.name)) if not protocolsToTry: errStr = "StorageElement.getSinglePfnForProtocol: None of the requested protocols were available for SE." self.log.debug(errStr, '%s for %s' % (protocol, self.name)) return S_ERROR(errStr) # Check all available storages for required protocol then contruct the PFN for storage in self.storages: res = storage.getParameters() if res['Value']['ProtocolName'] in protocolsToTry: res = pfnparse(pfn) if res['OK']: res = storage.getProtocolPfn(res['Value'], withPort) if res['OK']: return res errStr = "StorageElement.getSinglePfnForProtocol: Failed to get PFN for requested protocols." self.log.debug(errStr, "%s for %s" % (protocols, self.name)) return S_ERROR(errStr)
def __getSinglePfnForProtocol( self, pfn, protocol, withPort = True ): """ Transform the input pfn into a pfn with the given protocol for the Storage Element. :param pfn : input PFN :param protocol : string or list of string of the protocol we want :param withPort : includes the port in the returned pfn """ self.log.verbose( "StorageElement.getSinglePfnForProtocol: Getting pfn for given protocols in %s." % self.name ) # This test of the available protocols could actually be done in getPfnForProtocol once for all # but it is safer to put it here in case we decide to call this method internally (which I doubt!) res = self.getProtocols() if not res['OK']: return res if type( protocol ) == StringType: protocols = [protocol] elif type( protocol ) == ListType: protocols = protocol else: errStr = "StorageElement.getSinglePfnForProtocol: Supplied protocol must be string or list of strings." self.log.debug( errStr, "%s %s" % ( protocol, self.name ) ) return S_ERROR( errStr ) availableProtocols = res['Value'] protocolsToTry = [] for protocol in protocols: if protocol in availableProtocols: protocolsToTry.append( protocol ) else: errStr = "StorageElement.getSinglePfnForProtocol: Requested protocol not available for SE." self.log.debug( errStr, '%s for %s' % ( protocol, self.name ) ) if not protocolsToTry: errStr = "StorageElement.getSinglePfnForProtocol: None of the requested protocols were available for SE." self.log.debug( errStr, '%s for %s' % ( protocol, self.name ) ) return S_ERROR( errStr ) # Check all available storages for required protocol then contruct the PFN for storage in self.storages: res = storage.getParameters() if res['Value']['ProtocolName'] in protocolsToTry: res = pfnparse( pfn ) if res['OK']: res = storage.getProtocolPfn( res['Value'], withPort ) if res['OK']: return res errStr = "StorageElement.getSinglePfnForProtocol: Failed to get PFN for requested protocols." self.log.debug( errStr, "%s for %s" % ( protocols, self.name ) ) return S_ERROR( errStr )
def _isInputURL(self, url): """Check if the given url can be taken as input :param self: self reference :param str url: URL """ res = pfnparse(url) if not res["OK"]: return res urlDict = res["Value"] # Special case of 'file' protocol which can be just a URL if not urlDict["Protocol"] and "file" in self.protocolParameters[ "InputProtocols"]: return S_OK(True) return S_OK(urlDict["Protocol"] == self.protocolParameters["Protocol"])
def _killJobs( self, jobIDList, host = None ): """ Kill the jobs for the given list of job IDs """ resultDict = {} jobDict = {} for job in jobIDList: result = pfnparse( job ) if result['OK']: stamp = result['Value']['FileName'] else: self.log.error( 'Invalid job id', job ) continue jobDict[stamp] = job stampList = jobDict.keys() cmdTuple = [ self.finalScript, 'kill_job', '#'.join( stampList ), self.infoArea ] result = systemCall( 10, cmdTuple ) if not result['OK']: return result status = result['Value'][0] stdout = result['Value'][1] stderr = result['Value'][2] # Examine results of the job submission if status != 0: outputLines = stdout.strip().replace( '\r', '' ).split( '\n' ) try: index = outputLines.index( '============= Start output ===============' ) outputLines = outputLines[index + 1:] except: return S_ERROR( "Invalid output from kill Job: %s" % outputLines[0] ) try: status = int( outputLines[0] ) except: return S_ERROR( "Failed to kill Job: %s" % outputLines[0] ) if status != 0: message = "Unknown reason" if len( outputLines ) > 1: message = outputLines[1] return S_ERROR( 'Failed to kill Job, reason: %s' % message ) return S_ERROR( '\n'.join( [stdout, stderr] ) ) return S_OK()
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part serviceDict = dict(srcDict) serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part serviceDict = dict(srcDict) serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join(srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def _checkLFNPFNConvention(self, lfn, pfn, se): """ Check that the PFN corresponds to the LFN-PFN convention """ # Check if the PFN corresponds to the LFN convention if pfn == lfn: return S_OK() lfn_pfn = True # flag that the lfn is contained in the pfn if (len(pfn) < len(lfn)) or (pfn[-len(lfn):] != lfn): return S_ERROR('PFN does not correspond to the LFN convention') if not pfn.endswith(lfn): return S_ERROR() # Check if the pfn corresponds to the SE definition result = self._getStorageElement(se) if not result['OK']: return result selement = result['Value'] res = pfnparse(pfn) if not res['OK']: return res pfnDict = res['Value'] protocol = pfnDict['Protocol'] pfnpath = pfnDict['Path'] result = selement.getStorageParameters(protocol) if not result['OK']: return result seDict = result['Value'] sePath = seDict['Path'] ind = pfnpath.find(sePath) if ind == -1: return S_ERROR( 'The given PFN %s does not correspond to the %s SE definition' % (pfn, se)) # Check the full LFN-PFN-SE convention lfn_pfn_se = True if lfn_pfn: seAccessDict = dict(seDict) seAccessDict['Path'] = sePath + '/' + lfn check_pfn = pfnunparse(seAccessDict) if check_pfn != pfn: return S_ERROR('PFN does not correspond to the LFN convention') return S_OK()
def isURL( self, path ): """ Guess if the path looks like a URL :param self: self reference :param string path: input file LFN or URL :returns boolean: True if URL, False otherwise """ if self.basePath and path.startswith( self.basePath ): return S_OK( True ) result = pfnparse( path, srmSpecific = self.srmSpecificParse ) if not result['OK']: return result if len( result['Value']['Protocol'] ) != 0: return S_OK( True ) if result['Value']['Path'].startswith( self.basePath ): return S_OK( True ) return S_OK( False )
def isURL(self, path): """ Guess if the path looks like a URL :param self: self reference :param string path: input file LFN or URL :returns boolean: True if URL, False otherwise """ if self.basePath and path.startswith(self.basePath): return S_OK(True) result = pfnparse(path) if not result['OK']: return result if len(result['Value']['Protocol']) != 0: return S_OK(True) if result['Value']['Path'].startswith(self.basePath): return S_OK(True) return S_OK(False)
def isURL(self, path): """Guess if the path looks like a URL :param self: self reference :param string path: input file LFN or URL :returns boolean: True if URL, False otherwise """ if self.basePath and path.startswith(self.basePath): return S_OK(True) result = pfnparse(path, srmSpecific=self.srmSpecificParse) if not result["OK"]: return result if len(result["Value"]["Protocol"]) != 0: return S_OK(True) if result["Value"]["Path"].startswith(self.basePath): return S_OK(True) return S_OK(False)
def _checkLFNPFNConvention(self, lfn, pfn, se): """ Check that the PFN corresponds to the LFN-PFN convention """ # Check if the PFN corresponds to the LFN convention if pfn == lfn: return S_OK() lfn_pfn = True # flag that the lfn is contained in the pfn if (len(pfn) < len(lfn)) or (pfn[-len(lfn) :] != lfn): return S_ERROR("PFN does not correspond to the LFN convention") if not pfn.endswith(lfn): return S_ERROR() # Check if the pfn corresponds to the SE definition result = self._getStorageElement(se) if not result["OK"]: return result selement = result["Value"] res = pfnparse(pfn) if not res["OK"]: return res pfnDict = res["Value"] protocol = pfnDict["Protocol"] pfnpath = pfnDict["Path"] result = selement.getStorageParameters(protocol) if not result["OK"]: return result seDict = result["Value"] sePath = seDict["Path"] ind = pfnpath.find(sePath) if ind == -1: return S_ERROR("The given PFN %s does not correspond to the %s SE definition" % (pfn, se)) # Check the full LFN-PFN-SE convention lfn_pfn_se = True if lfn_pfn: seAccessDict = dict(seDict) seAccessDict["Path"] = sePath + "/" + lfn check_pfn = pfnunparse(seAccessDict) if check_pfn != pfn: return S_ERROR("PFN does not correspond to the LFN convention") return S_OK()
def getPfnForProtocol( self, pfn, protocol, withPort = True ): """ Transform the input pfn into another with the given protocol for the Storage Element. """ res = self.getProtocols() if not res['OK']: return res if type( protocol ) == StringType: protocols = [protocol] elif type( protocol ) == ListType: protocols = protocol else: errStr = "getPfnForProtocol: Supplied protocol must be string or list of strings." self.log.error( errStr, "%s %s" % ( protocol, self.name ) ) return S_ERROR( errStr ) availableProtocols = res['Value'] protocolsToTry = [] for protocol in protocols: if protocol in availableProtocols: protocolsToTry.append( protocol ) else: errStr = "getPfnForProtocol: Requested protocol not available for SE." self.log.debug( errStr, '%s for %s' % ( protocol, self.name ) ) if not protocolsToTry: errStr = "getPfnForProtocol: None of the requested protocols were available for SE." self.log.error( errStr, '%s for %s' % ( protocol, self.name ) ) return S_ERROR( errStr ) # Check all available storages for required protocol then contruct the PFN for storage in self.storages: res = storage.getParameters() if res['Value']['ProtocolName'] in protocolsToTry: res = pfnparse( pfn ) if res['OK']: res = storage.getProtocolPfn( res['Value'], withPort ) if res['OK']: return res errStr = "getPfnForProtocol: Failed to get PFN for requested protocols." self.log.error( errStr, "%s for %s" % ( protocols, self.name ) ) return S_ERROR( errStr )