def export_uploadFile(self, se, pfn): """ This method uploads a file present in the local cache to the specified storage element """ res = pythonCall(300, self.__uploadFile, se, pfn) if res['OK']: return res['Value'] return res
def export_prepareFile(self, se, pfn): """ This method simply gets the file to the local storage area """ res = pythonCall(300, self.__prepareFile, se, pfn) if res['OK']: return res['Value'] return res
def export_uploadFile( self, se, pfn ): """ This method uploads a file present in the local cache to the specified storage element """ res = pythonCall( 300, self.__uploadFile, se, pfn ) if res['OK']: return res['Value'] return res
def testTimeouts(self): """ test timeouts """ ## systemCall ret = systemCall(timeout=self.timeout, cmdSeq=self.cmd) self.assertEqual(ret, { 'Message': 'Timed out after 3 seconds', 'OK': False }) ## shellCall ret = shellCall(timeout=self.timeout, cmdSeq=" ".join(self.cmd)) self.assertEqual(ret, { 'Message': 'Timed out after 3 seconds', 'OK': False }) def pyfunc(name): time.sleep(10) return name ## pythonCall ret = pythonCall(self.timeout, pyfunc, "Krzysztof") self.assertEqual(ret, { 'Message': 'Timed out after 3 seconds', 'OK': False })
def export_prepareFile(self, se, pfn): """ This method simply gets the file to the local storage area """ res = pythonCall( 300, self.__prepareFile, se, pfn ) if res['OK']: return res['Value'] return res
def export_callProxyMethod( self, se, name, args, kargs ): """ A generic method to call methods of the Storage Element. """ res = pythonCall( 200, self.__proxyWrapper, se, name, args, kargs ) if res['OK']: return res['Value'] return res
def export_callProxyMethod(self, se, name, args, kargs): """ A generic method to call methods of the Storage Element. """ res = pythonCall(200, self.__proxyWrapper, se, name, args, kargs) if res['OK']: return res['Value'] return res
def export_callProxyMethod(self, fcName, methodName, args, kargs): """ A generic method to call methods of the Storage Element. """ res = pythonCall(120, self.__proxyWrapper, fcName, methodName, args, kargs) if res["OK"]: return res["Value"] else: return res
def export_callProxyMethod(self, fcName, methodName, args, kargs): """A generic method to call methods of the Storage Element.""" res = pythonCall(120, self.__proxyWrapper, fcName, methodName, args, kargs) if res["OK"]: return res["Value"] else: return res
def test_calls(timeout, expected): ret = systemCall(timeout, cmdSeq=cmd) assert ret["OK"] == expected ret = shellCall(timeout, cmdSeq=" ".join(cmd)) assert ret["OK"] == expected ret = pythonCall(timeout, pyfunc, "something") assert ret["OK"] == expected
def export_prepareFile(self, se, pfn): """ This method simply gets the file to the local storage area """ gLogger.debug("se %s, pfn %s" % (se, pfn)) res = pythonCall(300, self.__prepareFile, se, pfn) gLogger.debug("Preparing File %s" % res) if res['OK']: return res['Value'] return res
def r_execute(self, *parms, **kws): """ Read-only method executor. Dispatches execution of the methods which need Read-only access to the mirror LFC instances """ # If the DN argument is given, this is an operation on behalf # of the user with this DN, prepare setAuthorizationId call userDN = '' if kws.has_key('DN'): userDN = kws['DN'] del kws['DN'] result = S_ERROR() # Try the method 3 times just in case of intermittent errors max_retry = 2 count = 0 while (not result['OK']) and (count <= max_retry): i = 0 while not result['OK'] and i < self.nmirrors: # Switch environment to the mirror instance os.environ['LFC_HOST'] = self.mirrors[i].host try: result = S_OK() if userDN: resAuth = pythonCall( self.timeout, self.mirrors[i].setAuthorizationId, userDN) if not resAuth['OK']: result = S_ERROR( 'Failed to set user authorization') if result['OK']: method = getattr(self.mirrors[i], self.call) resMeth = method(*parms, **kws) if not resMeth['OK']: return resMeth else: result = resMeth if not result['Value']['Successful']: for reason in result['Value']['Failed'].values( ): # If the error is 'Timed out', we can not connect to the LFC server if reason == 'Timed out': result = S_ERROR('Time out') break except Exception, x: gLogger.exception( 'Exception while calling LFC Mirror service') result = S_ERROR( 'Exception while calling LFC Mirror service ' + str(x)) i += 1 count += 1
def doNew(self, masterParams=None): ''' Gets the parameters to run, either from the master method or from its own arguments. It queries the srm interface, and hopefully it will not crash. Out of the results, we keep totalsize, guaranteedsuze, and unusedsize. Then, they are recorded and returned. ''' if masterParams is not None: spaceTokenEndpoint, spaceToken = masterParams else: params = self._prepareCommand() if not params['OK']: return params spaceTokenEndpoint, spaceToken = params['Value'] # 10 secs of timeout. If it works, the reply is immediate. occupancyResult = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0) if not occupancyResult['OK']: self.log.error( "Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % \ ( spaceTokenEndpoint, spaceToken, occupancyResult['Message'] ) ) return occupancyResult else: occupancy = occupancyResult['Value'] # Timeout does not work here... # occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 ) if occupancy[0] != 0: return S_ERROR(occupancy) output = occupancy[1][0] sTokenDict = {} sTokenDict['Endpoint'] = spaceTokenEndpoint sTokenDict['Token'] = spaceToken sTokenDict['Total'] = float(output.get( 'totalsize', '0')) / 1e12 # Bytes to Terabytes sTokenDict['Guaranteed'] = float(output.get('guaranteedsize', '0')) / 1e12 sTokenDict['Free'] = float(output.get('unusedsize', '0')) / 1e12 storeRes = self._storeCommand([sTokenDict]) if not storeRes['OK']: return storeRes return S_OK([sTokenDict])
def r_execute( self, *parms, **kws ): """ Read-only method executor. Dispatches execution of the methods which need Read-only access to the mirror LFC instances """ # If the DN argument is given, this is an operation on behalf # of the user with this DN, prepare setAuthorizationId call userDN = '' if kws.has_key( 'DN' ): userDN = kws['DN'] del kws['DN'] result = S_ERROR() # Try the method 3 times just in case of intermittent errors max_retry = 2 count = 0 while ( not result['OK'] ) and ( count <= max_retry ): i = 0 while not result['OK'] and i < self.nmirrors: # Switch environment to the mirror instance os.environ['LFC_HOST'] = self.mirrors[i].host try: result = S_OK() if userDN: resAuth = pythonCall( self.timeout, self.mirrors[i].setAuthorizationId, userDN ) if not resAuth['OK']: result = S_ERROR( 'Failed to set user authorization' ) if result['OK']: method = getattr( self.mirrors[i], self.call ) resMeth = method( *parms, **kws ) if not resMeth['OK']: return resMeth else: result = resMeth if not result['Value'][ 'Successful' ]: for reason in result['Value'][ 'Failed' ].values(): # If the error is 'Timed out', we can not connect to the LFC server if reason == 'Timed out': result = S_ERROR( 'Time out' ) break except Exception, x: gLogger.exception( 'Exception while calling LFC Mirror service' ) result = S_ERROR( 'Exception while calling LFC Mirror service ' + str( x ) ) i += 1 count += 1
def testTimeouts( self ): """ test timeouts """ ## systemCall ret = systemCall( timeout=self.timeout, cmdSeq = self.cmd ) self.assertEqual( ret, {'Message': 'Timed out after 3 seconds', 'OK': False} ) ## shellCall ret = shellCall( timeout=self.timeout, cmdSeq = " ".join( self.cmd ) ) self.assertEqual( ret, {'Message': 'Timed out after 3 seconds', 'OK': False} ) def pyfunc( name ): time.sleep(10) return name ## pythonCall ret = pythonCall( self.timeout, pyfunc, "Krzysztof" ) self.assertEqual( ret, {'Message': 'Timed out after 3 seconds', 'OK': False} )
def testNoTimeouts(self): """ tests no timeouts """ ## systemCall ret = systemCall(timeout=False, cmdSeq=self.cmd) self.assertEqual(ret, {'OK': True, 'Value': (0, '', '')}) ## shellCall ret = shellCall(timeout=False, cmdSeq=" ".join(self.cmd)) self.assertEqual(ret, {'OK': True, 'Value': (0, '', '')}) def pyfunc(name): time.sleep(10) return name ## pythonCall ret = pythonCall(0, pyfunc, "Krzysztof") self.assertEqual(ret, {'OK': True, 'Value': 'Krzysztof'})
def testTimeouts(self): """ test timeouts """ ## systemCall ret = systemCall(timeout=self.timeout, cmdSeq=self.cmd) self.assertFalse(ret['OK']) ## shellCall ret = shellCall(timeout=self.timeout, cmdSeq=" ".join(self.cmd)) self.assertFalse(ret['OK']) def pyfunc(name): time.sleep(10) return name ## pythonCall ret = pythonCall(self.timeout, pyfunc, "Krzysztof") self.assertFalse(ret['OK'])
def testNoTimeouts(self): """ tests no timeouts """ ## systemCall ret = systemCall(timeout=False, cmdSeq=self.cmd) self.assertEqual(ret, {"OK": True, "Value": (0, "", "")}) ## shellCall ret = shellCall(timeout=False, cmdSeq=" ".join(self.cmd)) self.assertEqual(ret, {"OK": True, "Value": (0, "", "")}) def pyfunc(name): time.sleep(10) return name ## pythonCall ret = pythonCall(0, pyfunc, "Krzysztof") self.assertEqual(ret, {"OK": True, "Value": "Krzysztof"})
def testTimeouts(self): """ test timeouts """ ## systemCall ret = systemCall(timeout=self.timeout, cmdSeq=self.cmd) self.assertFalse(ret["OK"]) ## shellCall ret = shellCall(timeout=self.timeout, cmdSeq=" ".join(self.cmd)) self.assertFalse(ret["OK"]) def pyfunc(name): time.sleep(10) return name ## pythonCall ret = pythonCall(self.timeout, pyfunc, "Krzysztof") self.assertFalse(ret["OK"])
def doNew( self, masterParams = None ): ''' Gets the parameters to run, either from the master method or from its own arguments. It queries the srm interface, and hopefully it will not crash. Out of the results, we keep totalsize, guaranteedsuze, and unusedsize. Then, they are recorded and returned. ''' if masterParams is not None: spaceTokenEndpoint, spaceToken = masterParams else: params = self._prepareCommand() if not params[ 'OK' ]: return params spaceTokenEndpoint, spaceToken = params[ 'Value' ] # 10 secs of timeout. If it works, the reply is immediate. occupancy = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 ) if not occupancy[ 'OK' ]: return occupancy occupancy = occupancy[ 'Value' ] #Timeout does not work here... #occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 ) if occupancy[ 0 ] != 0: return S_ERROR( occupancy ) output = occupancy[ 1 ][ 0 ] sTokenDict = {} sTokenDict[ 'Endpoint' ] = spaceTokenEndpoint sTokenDict[ 'Token' ] = spaceToken sTokenDict[ 'Total' ] = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12 sTokenDict[ 'Free' ] = float( output.get( 'unusedsize', '0' ) ) / 1e12 storeRes = self._storeCommand( [ sTokenDict ] ) if not storeRes[ 'OK' ]: return storeRes return S_OK( [ sTokenDict ] )
def w_execute(self, *parms, **kws): """ Write method executor. Dispatches execution of the methods which need Read/Write access to the master LFC instance """ # If the DN argument is given, this is an operation on behalf # of the user with this DN, prepare setAuthorizationId call userDN = '' if kws.has_key('DN'): userDN = kws['DN'] del kws['DN'] # Try the method 3 times just in case of intermittent errors max_retry = 2 count = 0 result = S_ERROR() while (not result['OK']) and (count <= max_retry): if count > 0: # If retrying, wait a bit time.sleep(1) try: result = S_OK() if userDN: resAuth = pythonCall(self.timeout, self.lfc.setAuthorizationId, userDN) if not resAuth['OK']: result = S_ERROR('Failed to set user authorization') if result['OK']: method = getattr(self.lfc, self.call) resMeth = method(*parms, **kws) if not resMeth['OK']: return resMeth else: result = resMeth except Exception, x: gLogger.exception('Exception while calling LFC Master service', '', x) result = S_ERROR( 'Exception while calling LFC Master service ' + str(x)) count += 1
def export_prepareFileForHTTP(self, lfn): """ This method simply gets the file to the local storage area using LFN """ # Do clean-up, should be a separate regular thread gRegister.purgeExpired() key = str( random.getrandbits( 128 ) ) result = pythonCall( 300, self.__prepareFileForHTTP, lfn, key ) if result['OK']: result = result['Value'] if result['OK']: if HTTP_FLAG: host = socket.getfqdn() url = 'http://%s:%d/%s' % ( host, HTTP_PORT, key ) gRegister.add( key, 1800, result['CachePath'] ) result['HttpKey'] = key result['HttpURL'] = url return result return result return result
def export_prepareFileForHTTP(self, lfn): """ This method simply gets the file to the local storage area using LFN """ # Do clean-up, should be a separate regular thread gRegister.purgeExpired() key = str(random.getrandbits(128)) result = pythonCall(300, self.__prepareFileForHTTP, lfn, key) if result['OK']: result = result['Value'] if result['OK']: if HTTP_FLAG: host = socket.getfqdn() url = 'http://%s:%d/%s' % (host, HTTP_PORT, key) gRegister.add(key, 1800, result['CachePath']) result['HttpKey'] = key result['HttpURL'] = url return result return result return result
def export_prepareFileForHTTP(self, lfn): """This method simply gets the file to the local storage area using LFN""" # Do clean-up, should be a separate regular thread gRegister.purgeExpired() key = str(random.getrandbits(128)) result = pythonCall(300, self.__prepareFileForHTTP, lfn, key) if result["OK"]: result = result["Value"] # pylint believes it is a tuple because it is the only possible return type # it finds in Subprocess.py if result["OK"]: # pylint: disable=invalid-sequence-index if HTTP_FLAG: host = socket.getfqdn() url = "http://%s:%d/%s" % (host, HTTP_PORT, key) gRegister.add(key, 1800, result["CachePath"]) result["HttpKey"] = key result["HttpURL"] = url return result return result return result
def w_execute( self, *parms, **kws ): """ Write method executor. Dispatches execution of the methods which need Read/Write access to the master LFC instance """ # If the DN argument is given, this is an operation on behalf # of the user with this DN, prepare setAuthorizationId call userDN = '' if kws.has_key( 'DN' ): userDN = kws['DN'] del kws['DN'] # Try the method 3 times just in case of intermittent errors max_retry = 2 count = 0 result = S_ERROR() while ( not result['OK'] ) and ( count <= max_retry ): if count > 0: # If retrying, wait a bit time.sleep( 1 ) try: result = S_OK() if userDN: resAuth = pythonCall( self.timeout, self.lfc.setAuthorizationId, userDN ) if not resAuth['OK']: result = S_ERROR( 'Failed to set user authorization' ) if result['OK']: method = getattr( self.lfc, self.call ) resMeth = method( *parms, **kws ) if not resMeth['OK']: return resMeth else: result = resMeth except Exception, x: gLogger.exception( 'Exception while calling LFC Master service', '', x ) result = S_ERROR( 'Exception while calling LFC Master service ' + str( x ) ) count += 1
def export_prepareFileForHTTP(self, lfn): """ This method simply gets the file to the local storage area using LFN """ # Do clean-up, should be a separate regular thread gRegister.purgeExpired() key = str( random.getrandbits( 128 ) ) result = pythonCall( 300, self.__prepareFileForHTTP, lfn, key ) if result['OK']: result = result['Value'] #pylint believes it is a tuple because it is the only possible return type #it finds in Subprocess.py if result['OK']: #pylint: disable=invalid-sequence-index if HTTP_FLAG: host = socket.getfqdn() url = 'http://%s:%d/%s' % ( host, HTTP_PORT, key ) gRegister.add( key, 1800, result['CachePath'] ) result['HttpKey'] = key result['HttpURL'] = url return result return result return result
def __retrieveAndUploadFile(self, file, outputDict): """ Retrieve, Upload, and remove """ fileName = file inputPath = outputDict['InputPath'] inputFCName = outputDict['InputFC'] inBytes = 0 if inputFCName == 'LocalDisk': inFile = file file = os.path.join(inputPath, file) else: inputFC = FileCatalog([inputFCName]) inFile = os.path.join(inputPath, file) replicaDict = inputFC.getReplicas(inFile) if not replicaDict['OK']: self.log.error(replicaDict['Message']) return S_ERROR(fileName) if not inFile in replicaDict['Value']['Successful']: self.log.error(replicaDict['Value']['Failed'][inFile]) return S_ERROR(fileName) seList = replicaDict['Value']['Successful'][inFile].keys() inputSE = StorageElement(seList[0]) self.log.info('Retrieving from %s:' % inputSE.name, inFile) # ret = inputSE.getFile( inFile ) # lcg_util binding prevent multithreading, use subprocess instead res = pythonCall(2 * 3600, inputSE.getFile, inFile) if not res['OK']: self.log.error(res['Message']) return S_ERROR(fileName) ret = res['Value'] if not ret['OK']: self.log.error(ret['Message']) return S_ERROR(fileName) if not inFile in ret['Value']['Successful']: self.log.error(ret['Value']['Failed'][inFile]) return S_ERROR(fileName) if os.path.isfile(file): inBytes = os.stat(file)[6] outputPath = outputDict['OutputPath'] outputFCName = outputDict['OutputFC'] replicaManager = ReplicaManager() outFile = os.path.join(outputPath, os.path.basename(file)) transferOK = False for outputSEName in List.fromChar(outputDict['OutputSE'], ","): outputSE = StorageElement(outputSEName) self.log.info('Trying to upload to %s:' % outputSE.name, outFile) # ret = replicaManager.putAndRegister( outFile, os.path.realpath( file ), outputSE.name, catalog=outputFCName ) # lcg_util binding prevent multithreading, use subprocess instead result = pythonCall(2 * 3600, replicaManager.putAndRegister, outFile, os.path.realpath(file), outputSE.name, catalog=outputFCName) if result['OK'] and result['Value']['OK']: if outFile in result['Value']['Value']['Successful']: transferOK = True break else: self.log.error(result['Value']['Value']['Failed'][outFile]) else: if result['OK']: self.log.error(result['Value']['Message']) else: self.log.error(result['Message']) if not transferOK: return S_ERROR(fileName) if result['OK'] or not inputFCName == 'LocalDisk': os.unlink(file) if not result['OK']: self.log.error(ret['Message']) return S_ERROR(fileName) self.log.info("Finished transferring %s [%s bytes]" % (inFile, inBytes)) self.__okTransferredFiles += 1 self.__okTransferredBytes += inBytes if inputFCName == 'LocalDisk': return S_OK(fileName) # Now the file is on final SE/FC, remove from input SE/FC for se in seList: se = StorageElement(se) self.log.info('Removing from %s:' % se.name, inFile) se.removeFile(inFile) inputFC.removeFile(inFile) return S_OK(fileName)
def __retrieveAndUploadFile( self, file, outputDict ): """ Retrieve, Upload, and remove """ fileName = file inputPath = outputDict['InputPath'] inputFCName = outputDict['InputFC'] inBytes = 0 if inputFCName == 'LocalDisk': inFile = file file = os.path.join( inputPath, file ) else: inputFC = FileCatalog( [inputFCName] ) inFile = os.path.join( inputPath, file ) replicaDict = inputFC.getReplicas( inFile ) if not replicaDict['OK']: self.log.error( replicaDict['Message'] ) return S_ERROR( fileName ) if not inFile in replicaDict['Value']['Successful']: self.log.error( replicaDict['Value']['Failed'][inFile] ) return S_ERROR( fileName ) seList = replicaDict['Value']['Successful'][inFile].keys() inputSE = StorageElement( seList[0] ) self.log.info( 'Retrieving from %s:' % inputSE.name, inFile ) # ret = inputSE.getFile( inFile ) # lcg_util binding prevent multithreading, use subprocess instead res = pythonCall( 2 * 3600, inputSE.getFile, inFile ) if not res['OK']: self.log.error( res['Message'] ) return S_ERROR( fileName ) ret = res['Value'] if not ret['OK']: self.log.error( ret['Message'] ) return S_ERROR( fileName ) if not inFile in ret['Value']['Successful']: self.log.error( ret['Value']['Failed'][inFile] ) return S_ERROR( fileName ) if os.path.isfile( file ): inBytes = os.stat( file )[6] outputPath = outputDict['OutputPath'] outputFCName = outputDict['OutputFC'] replicaManager = ReplicaManager() outFile = os.path.join( outputPath, os.path.basename( file ) ) transferOK = False for outputSEName in List.fromChar( outputDict['OutputSE'], "," ): outputSE = StorageElement( outputSEName ) self.log.info( 'Trying to upload to %s:' % outputSE.name, outFile ) # ret = replicaManager.putAndRegister( outFile, os.path.realpath( file ), outputSE.name, catalog=outputFCName ) # lcg_util binding prevent multithreading, use subprocess instead result = pythonCall( 2 * 3600, replicaManager.putAndRegister, outFile, os.path.realpath( file ), outputSE.name, catalog = outputFCName ) if result['OK'] and result['Value']['OK']: if outFile in result['Value']['Value']['Successful']: transferOK = True break else: self.log.error( result['Value']['Value']['Failed'][outFile] ) else: if result['OK']: self.log.error( result['Value']['Message'] ) else: self.log.error( result['Message'] ) if not transferOK: return S_ERROR( fileName ) if result['OK'] or not inputFCName == 'LocalDisk': os.unlink( file ) if not result['OK']: self.log.error( ret['Message'] ) return S_ERROR( fileName ) self.log.info( "Finished transferring %s [%s bytes]" % ( inFile, inBytes ) ) self.__okTransferredFiles += 1 self.__okTransferredBytes += inBytes if inputFCName == 'LocalDisk': return S_OK( fileName ) # Now the file is on final SE/FC, remove from input SE/FC for se in seList: se = StorageElement( se ) self.log.info( 'Removing from %s:' % se.name, inFile ) se.removeFile( inFile ) inputFC.removeFile( inFile ) return S_OK( fileName )
break except Exception, x: gLogger.exception( 'Exception while calling LFC Mirror service' ) result = S_ERROR( 'Exception while calling LFC Mirror service ' + str( x ) ) i += 1 count += 1 # Return environment to the master LFC instance os.environ['LFC_HOST'] = self.master_host # Call the master LFC if all the mirrors failed if not result['OK']: try: result = S_OK() if userDN: resAuth = pythonCall( self.timeout, self.lfc.setAuthorizationId, userDN ) if not resAuth['OK']: result = S_ERROR( 'Failed to set user authorization' ) if result['OK']: method = getattr( self.lfc, self.call ) resMeth = method( *parms, **kws ) if not resMeth['OK']: result = S_ERROR( 'Timout calling ' + self.call + " method" ) else: result = resMeth except Exception, x: gLogger.exception( 'Exception while calling LFC Master service' ) result = S_ERROR( 'Exception while calling LFC Master service ' + str( x ) ) return result