def __putFile( self, src_file, dest_url ):
res = pfnparse( src_file )
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
localCache = True
srcSEURL = srcDict['Protocol'] + '://' + srcDict['Host'] + ':' + srcDict['Port'] + srcDict['WSUrl']
transferClient = TransferClient( srcSEURL )
res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) )
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists( src_file ):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
sourceSize = getSize( src_file )
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
transferClient = TransferClient( self.url )
res = transferClient.sendFile( src_file, dest_url, token = self.checkSum )
if localCache:
os.unlink( src_file )
if res['OK']:
return S_OK( sourceSize )
else:
return res
def web_getPlotImg(self):
"""
Get plot image
"""
callback = {}
if 'file' not in self.request.arguments:
callback = {"success":"false", "error":"Maybe you forgot the file?"}
self.finish( callback )
return
plotImageFile = str( self.request.arguments[ 'file' ][0] )
if plotImageFile.find( ".png" ) < -1:
callback = {"success":"false", "error":"Not a valid image!"}
self.finish( callback )
return
transferClient = TransferClient( "Accounting/ReportGenerator" )
tempFile = tempfile.TemporaryFile()
retVal = transferClient.receiveFile( tempFile, plotImageFile )
if not retVal[ 'OK' ]:
callback = {"success":"false", "error":retVal[ 'Message' ]}
self.finish( callback )
return
tempFile.seek( 0 )
data = tempFile.read()
self.set_header( 'Content-type', 'image/png' )
self.set_header( 'Content-Disposition', 'attachment; filename="%s.png"' % md5( plotImageFile ).hexdigest() )
self.set_header( 'Content-Length', len( data ) )
self.set_header( 'Content-Transfer-Encoding', 'Binary' )
#self.set_header( 'Cache-Control', "no-cache, no-store, must-revalidate, max-age=0" )
#self.set_header( 'Pragma', "no-cache" )
#self.set_header( 'Expires', ( datetime.datetime.utcnow() - datetime.timedelta( minutes = -10 ) ).strftime( "%d %b %Y %H:%M:%S GMT" ) )
self.finish( data )
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
localCache = True
srcSEURL = srcDict['Protocol'] + '://' + srcDict[
'Host'] + ':' + srcDict['Port'] + srcDict['WSUrl']
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'],
os.path.join(srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def getSandbox(self, jobID, output_dir=''):
""" Get the job complete sandbox
"""
# Get the list of files in the sandbox
sandbox_status = RPCClient('WorkloadManagement/%sSandbox' %
self.sandbox_type,
timeout=120)
result = sandbox_status.getFileNames(jobID)
if not result['OK']:
return S_ERROR('Failed to get the list of file names')
fileList = result['Value']
cwd = os.getcwd()
if output_dir:
os.chdir(os.path.realpath(output_dir))
error_files = []
decRes = 1
for f in fileList:
sname = ` jobID ` + "::" + f
sandbox = TransferClient('WorkloadManagement/%sSandbox' %
self.sandbox_type)
result = sandbox.receiveFile(f, sname)
if not result['OK']:
error_files.append(f)
else:
if f.find('__Sandbox__.tar') != -1 or f.find(
'__Sandbox__.tgz') != -1:
if f.find('.bz') != -1:
decRes = os.system('tar xjf ' + f)
elif f.find('.gz') != -1 or f.find('.tgz') != -1:
decRes = os.system('tar xzf ' + f)
else:
decRes = os.system('tar xf ' + f)
os.remove(f)
if decRes != 0:
return S_ERROR("Could not decompress sandbox")
if output_dir:
os.chdir(cwd)
if error_files:
result = S_ERROR('Failed to download all the files')
result['FailedFiles'] = error_files
else:
result = S_OK(fileList)
# Set job retrieved flag
jobState = RPCClient('WorkloadManagement/JobStateUpdate',
timeout=120)
jobState.setJobFlag(jobID, 'RetrievedFlag')
return result
def test_getPlot(self):
tempFile = tempfile.TemporaryFile()
transferClient = TransferClient('Monitoring/Monitoring')
params = ('WMSHistory', 'NumberOfJobs',
datetime(2016, 3, 16, 12, 30, 0,
0), datetime(2016, 3, 17, 19, 29, 0, 0), {
'grouping': ['Site']
}, 'Site', {})
result = self.client.generateDelayedPlot(*params)
self.assertTrue(result['OK'])
result = transferClient.receiveFile(tempFile, result['Value']['plot'])
self.assertTrue(result['OK'])
def test_getPlot(self):
tempFile = tempfile.TemporaryFile()
transferClient = TransferClient('Monitoring/Monitoring')
params = (
'WMSHistory', 'NumberOfJobs', datetime(
2016, 3, 16, 12, 30, 0, 0), datetime(
2016, 3, 17, 19, 29, 0, 0), {
'grouping': ['Site']}, 'Site', {})
result = self.client.generateDelayedPlot(*params)
self.assertTrue(result['OK'])
result = transferClient.receiveFile(tempFile, result['Value']['plot'])
self.assertTrue(result['OK'])
def getSandbox(self,jobID,output_dir=''):
""" Get the job complete sandbox
"""
# Get the list of files in the sandbox
sandbox_status = RPCClient('WorkloadManagement/%sSandbox' % self.sandbox_type,timeout=120)
result = sandbox_status.getFileNames(jobID)
if not result['OK']:
return S_ERROR('Failed to get the list of file names')
fileList = result['Value']
cwd = os.getcwd()
if output_dir:
os.chdir(os.path.realpath(output_dir))
error_files = []
decRes = 1
for f in fileList:
sname = `jobID`+"::"+f
sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type)
result = sandbox.receiveFile(f,sname)
if not result['OK']:
error_files.append(f)
else:
if f.find('__Sandbox__.tar') != -1 or f.find('__Sandbox__.tgz') != -1 :
if f.find('.bz') != -1:
decRes = os.system('tar xjf '+f)
elif f.find('.gz') != -1 or f.find('.tgz') != -1:
decRes = os.system('tar xzf '+f)
else:
decRes = os.system('tar xf '+f)
os.remove(f)
if decRes != 0:
return S_ERROR( "Could not decompress sandbox" )
if output_dir:
os.chdir(cwd)
if error_files:
result = S_ERROR('Failed to download all the files')
result['FailedFiles'] = error_files
else:
result = S_OK(fileList)
# Set job retrieved flag
jobState = RPCClient('WorkloadManagement/JobStateUpdate',timeout=120)
jobState.setJobFlag(jobID,'RetrievedFlag')
return result
def getSEDump(self, seName, outputFilename):
"""
Dump the content of an SE in the given file.
The file contains a list of [lfn,checksum,size] dumped as csv,
separated by '|'
:param seName: name of the StorageElement
:param outputFilename: path to the file where to dump it
:returns: result from the TransferClient
"""
dfc = TransferClient(self.serverURL)
return dfc.receiveFile(outputFilename, seName)
def getSEDump(self, seName, outputFilename):
"""
Dump the content of an SE in the given file.
The file contains a list of [lfn,checksum,size] dumped as csv,
separated by '|'
:param seName: name of the StorageElement
:param outputFilename: path to the file where to dump it
:returns: result from the TransferClient
"""
dfc = TransferClient(self.serverURL)
return dfc.receiveFile(outputFilename, seName)
def __getFile( self, src_url, dest_file ):
transferClient = TransferClient( self.url )
res = transferClient.receiveFile( dest_file, src_url, token = self.checkSum )
if not res['OK']:
return res
if not os.path.exists( dest_file ):
errStr = "DIPStorage.__getFile: The destination local file does not exist."
gLogger.error( errStr, dest_file )
return S_ERROR( errStr )
destSize = getSize( dest_file )
if destSize == -1:
errStr = "DIPStorage.__getFile: Failed to get the local file size."
gLogger.error( errStr, dest_file )
return S_ERROR( errStr )
return S_OK( destSize )
def __getFile( self, src_url, dest_file ):
transferClient = TransferClient( self.url )
res = transferClient.receiveFile( dest_file, src_url, token = self.checkSum )
if not res['OK']:
return res
if not os.path.exists( dest_file ):
errStr = "DIPStorage.__getFile: The destination local file does not exist."
gLogger.error( errStr, dest_file )
return S_ERROR( errStr )
destSize = getSize( dest_file )
if destSize == -1:
errStr = "DIPStorage.__getFile: Failed to get the local file size."
gLogger.error( errStr, dest_file )
return S_ERROR( errStr )
return S_OK( destSize )
def test_getPlot(self):
tempFile = tempfile.TemporaryFile()
transferClient = TransferClient("Monitoring/Monitoring")
params = (
"WMSHistory",
"NumberOfJobs",
datetime(2016, 3, 16, 12, 30, 0, 0),
datetime(2016, 3, 17, 19, 29, 0, 0),
{"grouping": ["Site"]},
"Site",
{},
)
result = self.client.generateDelayedPlot(*params)
self.assert_(result["OK"])
result = transferClient.receiveFile(tempFile, result["Value"]["plot"])
self.assert_(result["OK"])
def getFile(self, path, localPath=False):
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient(self.url)
# Make sure transferClient uses the same ProxyStorage instance.
# Only the this one holds the file we want to transfer.
transferClient = TransferClient(client.serviceURL)
for src_url in urls.keys():
res = client.prepareFile(self.name, src_url)
if not res['OK']:
gLogger.error(
"ProxyStorage.getFile: Failed to prepare file on remote server.",
res['Message'])
failed[src_url] = res['Message']
else:
fileName = os.path.basename(src_url)
if localPath:
dest_file = "%s/%s" % (localPath, fileName)
else:
dest_file = "%s/%s" % (os.getcwd(), fileName)
res = transferClient.receiveFile(dest_file,
'getFile/%s' % fileName)
if not res['OK']:
gLogger.error(
"ProxyStorage.getFile: Failed to recieve file from proxy server.",
res['Message'])
failed[src_url] = res['Message']
elif not os.path.exists(dest_file):
errStr = "ProxyStorage.getFile: The destination local file does not exist."
gLogger.error(errStr, dest_file)
failed[src_url] = errStr
else:
destSize = getSize(dest_file)
if destSize == -1:
errStr = "ProxyStorage.getFile: Failed to get the local file size."
gLogger.error(errStr, dest_file)
failed[src_url] = errStr
else:
successful[src_url] = destSize
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getFiles(self, in_dict):
"""
It returns a list of files for a given condition.
:param dict in_dict: contains a given conditions
:return: list of files
"""
in_dict = dict(in_dict)
bkk = TransferClient('Bookkeeping/BookkeepingManager')
in_dict['MethodName'] = 'getFiles'
params = JEncoder.dumps(in_dict)
file_name = tempfile.NamedTemporaryFile()
retVal = bkk.receiveFile(file_name.name, params)
if not retVal['OK']:
return retVal
else:
value = JEncoder.load(open(file_name.name))
file_name.close()
return value
def getFilesWithMetadata(in_dict):
"""
It is used for retrieving a files with meta data for a given condition.
:param dict in_dict: It can contains the following conditions:'ConfigName',
'ConfigVersion', 'ConditionDescription', 'EventType',
'ProcessingPass','Production','RunNumber', 'FileType', DataQuality, StartDate, EndDate
:return: files with meta data associated
"""
in_dict = dict(in_dict)
bkk = TransferClient('Bookkeeping/BookkeepingManager')
params = JEncoder.dumps(in_dict)
file_name = tempfile.NamedTemporaryFile()
retVal = bkk.receiveFile(file_name.name, params)
if not retVal['OK']:
return retVal
else:
value = JEncoder.load(open(file_name.name))
file_name.close()
return S_OK(value)
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'], os.path.join(
srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'],
os.path.join(srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def getFile( self, path, localPath = False ):
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient( self.url )
# Make sure transferClient uses the same ProxyStorage instance.
# Only the this one holds the file we want to transfer.
transferClient = TransferClient( client.serviceURL )
for src_url in urls.keys():
res = client.prepareFile( self.name, src_url )
if not res['OK']:
gLogger.error( "ProxyStorage.getFile: Failed to prepare file on remote server.", res['Message'] )
failed[src_url] = res['Message']
else:
fileName = os.path.basename( src_url )
if localPath:
dest_file = "%s/%s" % ( localPath, fileName )
else:
dest_file = "%s/%s" % ( os.getcwd(), fileName )
res = transferClient.receiveFile( dest_file, 'getFile/%s' % fileName )
if not res['OK']:
gLogger.error( "ProxyStorage.getFile: Failed to recieve file from proxy server.", res['Message'] )
failed[src_url] = res['Message']
elif not os.path.exists( dest_file ):
errStr = "ProxyStorage.getFile: The destination local file does not exist."
gLogger.error( errStr, dest_file )
failed[src_url] = errStr
else:
destSize = getSize( dest_file )
if destSize == -1:
errStr = "ProxyStorage.getFile: Failed to get the local file size."
gLogger.error( errStr, dest_file )
failed[src_url] = errStr
else:
successful[src_url] = destSize
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
