def __putFile( self, src_file, dest_url ): res = pfnparse( src_file ) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: localCache = True srcSEURL = srcDict['Protocol'] + '://' + srcDict['Host'] + ':' + srcDict['Port'] + srcDict['WSUrl'] transferClient = TransferClient( srcSEURL ) res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) ) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists( src_file ): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) sourceSize = getSize( src_file ) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) transferClient = TransferClient( self.url ) res = transferClient.sendFile( src_file, dest_url, token = self.checkSum ) if localCache: os.unlink( src_file ) if res['OK']: return S_OK( sourceSize ) else: return res
def web_getPlotImg(self): """ Get plot image """ callback = {} if 'file' not in self.request.arguments: callback = {"success":"false", "error":"Maybe you forgot the file?"} self.finish( callback ) return plotImageFile = str( self.request.arguments[ 'file' ][0] ) if plotImageFile.find( ".png" ) < -1: callback = {"success":"false", "error":"Not a valid image!"} self.finish( callback ) return transferClient = TransferClient( "Accounting/ReportGenerator" ) tempFile = tempfile.TemporaryFile() retVal = transferClient.receiveFile( tempFile, plotImageFile ) if not retVal[ 'OK' ]: callback = {"success":"false", "error":retVal[ 'Message' ]} self.finish( callback ) return tempFile.seek( 0 ) data = tempFile.read() self.set_header( 'Content-type', 'image/png' ) self.set_header( 'Content-Disposition', 'attachment; filename="%s.png"' % md5( plotImageFile ).hexdigest() ) self.set_header( 'Content-Length', len( data ) ) self.set_header( 'Content-Transfer-Encoding', 'Binary' ) #self.set_header( 'Cache-Control', "no-cache, no-store, must-revalidate, max-age=0" ) #self.set_header( 'Pragma', "no-cache" ) #self.set_header( 'Expires', ( datetime.datetime.utcnow() - datetime.timedelta( minutes = -10 ) ).strftime( "%d %b %Y %H:%M:%S GMT" ) ) self.finish( data )
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: localCache = True srcSEURL = srcDict['Protocol'] + '://' + srcDict[ 'Host'] + ':' + srcDict['Port'] + srcDict['WSUrl'] transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join(srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def getSandbox(self, jobID, output_dir=''): """ Get the job complete sandbox """ # Get the list of files in the sandbox sandbox_status = RPCClient('WorkloadManagement/%sSandbox' % self.sandbox_type, timeout=120) result = sandbox_status.getFileNames(jobID) if not result['OK']: return S_ERROR('Failed to get the list of file names') fileList = result['Value'] cwd = os.getcwd() if output_dir: os.chdir(os.path.realpath(output_dir)) error_files = [] decRes = 1 for f in fileList: sname = ` jobID ` + "::" + f sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type) result = sandbox.receiveFile(f, sname) if not result['OK']: error_files.append(f) else: if f.find('__Sandbox__.tar') != -1 or f.find( '__Sandbox__.tgz') != -1: if f.find('.bz') != -1: decRes = os.system('tar xjf ' + f) elif f.find('.gz') != -1 or f.find('.tgz') != -1: decRes = os.system('tar xzf ' + f) else: decRes = os.system('tar xf ' + f) os.remove(f) if decRes != 0: return S_ERROR("Could not decompress sandbox") if output_dir: os.chdir(cwd) if error_files: result = S_ERROR('Failed to download all the files') result['FailedFiles'] = error_files else: result = S_OK(fileList) # Set job retrieved flag jobState = RPCClient('WorkloadManagement/JobStateUpdate', timeout=120) jobState.setJobFlag(jobID, 'RetrievedFlag') return result
def test_getPlot(self): tempFile = tempfile.TemporaryFile() transferClient = TransferClient('Monitoring/Monitoring') params = ('WMSHistory', 'NumberOfJobs', datetime(2016, 3, 16, 12, 30, 0, 0), datetime(2016, 3, 17, 19, 29, 0, 0), { 'grouping': ['Site'] }, 'Site', {}) result = self.client.generateDelayedPlot(*params) self.assertTrue(result['OK']) result = transferClient.receiveFile(tempFile, result['Value']['plot']) self.assertTrue(result['OK'])
def test_getPlot(self): tempFile = tempfile.TemporaryFile() transferClient = TransferClient('Monitoring/Monitoring') params = ( 'WMSHistory', 'NumberOfJobs', datetime( 2016, 3, 16, 12, 30, 0, 0), datetime( 2016, 3, 17, 19, 29, 0, 0), { 'grouping': ['Site']}, 'Site', {}) result = self.client.generateDelayedPlot(*params) self.assertTrue(result['OK']) result = transferClient.receiveFile(tempFile, result['Value']['plot']) self.assertTrue(result['OK'])
def getSandbox(self,jobID,output_dir=''): """ Get the job complete sandbox """ # Get the list of files in the sandbox sandbox_status = RPCClient('WorkloadManagement/%sSandbox' % self.sandbox_type,timeout=120) result = sandbox_status.getFileNames(jobID) if not result['OK']: return S_ERROR('Failed to get the list of file names') fileList = result['Value'] cwd = os.getcwd() if output_dir: os.chdir(os.path.realpath(output_dir)) error_files = [] decRes = 1 for f in fileList: sname = `jobID`+"::"+f sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type) result = sandbox.receiveFile(f,sname) if not result['OK']: error_files.append(f) else: if f.find('__Sandbox__.tar') != -1 or f.find('__Sandbox__.tgz') != -1 : if f.find('.bz') != -1: decRes = os.system('tar xjf '+f) elif f.find('.gz') != -1 or f.find('.tgz') != -1: decRes = os.system('tar xzf '+f) else: decRes = os.system('tar xf '+f) os.remove(f) if decRes != 0: return S_ERROR( "Could not decompress sandbox" ) if output_dir: os.chdir(cwd) if error_files: result = S_ERROR('Failed to download all the files') result['FailedFiles'] = error_files else: result = S_OK(fileList) # Set job retrieved flag jobState = RPCClient('WorkloadManagement/JobStateUpdate',timeout=120) jobState.setJobFlag(jobID,'RetrievedFlag') return result
def getSEDump(self, seName, outputFilename): """ Dump the content of an SE in the given file. The file contains a list of [lfn,checksum,size] dumped as csv, separated by '|' :param seName: name of the StorageElement :param outputFilename: path to the file where to dump it :returns: result from the TransferClient """ dfc = TransferClient(self.serverURL) return dfc.receiveFile(outputFilename, seName)
def __getFile( self, src_url, dest_file ): transferClient = TransferClient( self.url ) res = transferClient.receiveFile( dest_file, src_url, token = self.checkSum ) if not res['OK']: return res if not os.path.exists( dest_file ): errStr = "DIPStorage.__getFile: The destination local file does not exist." gLogger.error( errStr, dest_file ) return S_ERROR( errStr ) destSize = getSize( dest_file ) if destSize == -1: errStr = "DIPStorage.__getFile: Failed to get the local file size." gLogger.error( errStr, dest_file ) return S_ERROR( errStr ) return S_OK( destSize )
def test_getPlot(self): tempFile = tempfile.TemporaryFile() transferClient = TransferClient("Monitoring/Monitoring") params = ( "WMSHistory", "NumberOfJobs", datetime(2016, 3, 16, 12, 30, 0, 0), datetime(2016, 3, 17, 19, 29, 0, 0), {"grouping": ["Site"]}, "Site", {}, ) result = self.client.generateDelayedPlot(*params) self.assert_(result["OK"]) result = transferClient.receiveFile(tempFile, result["Value"]["plot"]) self.assert_(result["OK"])
def getFile(self, path, localPath=False): res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] failed = {} successful = {} client = RPCClient(self.url) # Make sure transferClient uses the same ProxyStorage instance. # Only the this one holds the file we want to transfer. transferClient = TransferClient(client.serviceURL) for src_url in urls.keys(): res = client.prepareFile(self.name, src_url) if not res['OK']: gLogger.error( "ProxyStorage.getFile: Failed to prepare file on remote server.", res['Message']) failed[src_url] = res['Message'] else: fileName = os.path.basename(src_url) if localPath: dest_file = "%s/%s" % (localPath, fileName) else: dest_file = "%s/%s" % (os.getcwd(), fileName) res = transferClient.receiveFile(dest_file, 'getFile/%s' % fileName) if not res['OK']: gLogger.error( "ProxyStorage.getFile: Failed to recieve file from proxy server.", res['Message']) failed[src_url] = res['Message'] elif not os.path.exists(dest_file): errStr = "ProxyStorage.getFile: The destination local file does not exist." gLogger.error(errStr, dest_file) failed[src_url] = errStr else: destSize = getSize(dest_file) if destSize == -1: errStr = "ProxyStorage.getFile: Failed to get the local file size." gLogger.error(errStr, dest_file) failed[src_url] = errStr else: successful[src_url] = destSize resDict = {'Failed': failed, 'Successful': successful} return S_OK(resDict)
def getFiles(self, in_dict): """ It returns a list of files for a given condition. :param dict in_dict: contains a given conditions :return: list of files """ in_dict = dict(in_dict) bkk = TransferClient('Bookkeeping/BookkeepingManager') in_dict['MethodName'] = 'getFiles' params = JEncoder.dumps(in_dict) file_name = tempfile.NamedTemporaryFile() retVal = bkk.receiveFile(file_name.name, params) if not retVal['OK']: return retVal else: value = JEncoder.load(open(file_name.name)) file_name.close() return value
def getFilesWithMetadata(in_dict): """ It is used for retrieving a files with meta data for a given condition. :param dict in_dict: It can contains the following conditions:'ConfigName', 'ConfigVersion', 'ConditionDescription', 'EventType', 'ProcessingPass','Production','RunNumber', 'FileType', DataQuality, StartDate, EndDate :return: files with meta data associated """ in_dict = dict(in_dict) bkk = TransferClient('Bookkeeping/BookkeepingManager') params = JEncoder.dumps(in_dict) file_name = tempfile.NamedTemporaryFile() retVal = bkk.receiveFile(file_name.name, params) if not retVal['OK']: return retVal else: value = JEncoder.load(open(file_name.name)) file_name.close() return S_OK(value)
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part serviceDict = dict(srcDict) serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part serviceDict = dict(srcDict) serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join(srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def getFile( self, path, localPath = False ): res = checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] failed = {} successful = {} client = RPCClient( self.url ) # Make sure transferClient uses the same ProxyStorage instance. # Only the this one holds the file we want to transfer. transferClient = TransferClient( client.serviceURL ) for src_url in urls.keys(): res = client.prepareFile( self.name, src_url ) if not res['OK']: gLogger.error( "ProxyStorage.getFile: Failed to prepare file on remote server.", res['Message'] ) failed[src_url] = res['Message'] else: fileName = os.path.basename( src_url ) if localPath: dest_file = "%s/%s" % ( localPath, fileName ) else: dest_file = "%s/%s" % ( os.getcwd(), fileName ) res = transferClient.receiveFile( dest_file, 'getFile/%s' % fileName ) if not res['OK']: gLogger.error( "ProxyStorage.getFile: Failed to recieve file from proxy server.", res['Message'] ) failed[src_url] = res['Message'] elif not os.path.exists( dest_file ): errStr = "ProxyStorage.getFile: The destination local file does not exist." gLogger.error( errStr, dest_file ) failed[src_url] = errStr else: destSize = getSize( dest_file ) if destSize == -1: errStr = "ProxyStorage.getFile: Failed to get the local file size." gLogger.error( errStr, dest_file ) failed[src_url] = errStr else: successful[src_url] = destSize resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )