def getDirectory(self, path, localPath=False):
"""Get a local copy in the current directory of a physical file specified by its path"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
failed = {}
successful = {}
gLogger.debug("DIPStorage.getDirectory: Attempting to get local copies of %s directories." % len(urls))
transferClient = TransferClient(self.url)
for src_dir in urls:
if localPath:
dest_dir = localPath
else:
dest_dir = os.getcwd()
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
res = transferClient.receiveBulk(dest_dir, src_dir)
if res["OK"]:
gLogger.debug("DIPStorage.getDirectory: Successfully got local copy of %s" % src_dir)
successful[src_dir] = {"Files": 0, "Size": 0}
else:
gLogger.error("DIPStorage.getDirectory: Failed to get entire directory.", src_dir)
failed[src_dir] = res["Message"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def getSEDump(self, seName, outputFilename):
"""
Dump the content of an SE in the given file.
The file contains a list of [lfn,checksum,size] dumped as csv,
separated by '|'
:param seName: name of the StorageElement
:param outputFilename: path to the file where to dump it
:returns: result from the TransferClient
"""
dfc = TransferClient(self.serverURL)
return dfc.receiveFile(outputFilename, seName)
def putFile(self, path, sourceSize=0):
client = Client(url=self.url)
if sourceSize:
gLogger.debug(
"ProxyStorage.putFile: The client has provided the source file size implying\
a replication is requested.")
return client.callProxyMethod(self.name, "putFile", [path],
{"sourceSize": sourceSize})
gLogger.debug(
"ProxyStorage.putFile: No source size was provided therefore a simple put will be performed."
)
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
failed = {}
successful = {}
# make sure transferClient uses the same ProxyStorage instance we uploaded the file to
transferClient = TransferClient(client.serviceURL)
for dest_url, src_file in urls.items():
fileName = os.path.basename(dest_url)
res = transferClient.sendFile(src_file, "putFile/%s" % fileName)
if not res["OK"]:
gLogger.error(
"ProxyStorage.putFile: Failed to send file to proxy server.",
res["Message"])
failed[dest_url] = res["Message"]
else:
res = client.uploadFile(self.name, dest_url)
if not res["OK"]:
gLogger.error(
"ProxyStorage.putFile: Failed to upload file to storage element from proxy server.",
res["Message"],
)
failed[dest_url] = res["Message"]
else:
res = self.__executeOperation(dest_url, "getFileSize")
if not res["OK"]:
gLogger.error(
"ProxyStorage.putFile: Failed to determine destination file size.",
res["Message"])
failed[dest_url] = res["Message"]
else:
successful[dest_url] = res["Value"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def __getFile(self, src_url, dest_file):
transferClient = TransferClient(self.url)
res = transferClient.receiveFile(dest_file, src_url, token=self.checkSum)
if not res["OK"]:
return res
if not os.path.exists(dest_file):
errStr = "DIPStorage.__getFile: The destination local file does not exist."
gLogger.error(errStr, dest_file)
return S_ERROR(errStr)
destSize = getSize(dest_file)
if destSize == -1:
errStr = "DIPStorage.__getFile: Failed to get the local file size."
gLogger.error(errStr, dest_file)
return S_ERROR(errStr)
return S_OK(destSize)
def getFile(self, path, localPath=False):
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
failed = {}
successful = {}
client = Client(url=self.url)
# Make sure transferClient uses the same ProxyStorage instance.
# Only the this one holds the file we want to transfer.
transferClient = TransferClient(client.serviceURL)
for src_url in urls.keys():
res = client.prepareFile(self.name, src_url)
if not res["OK"]:
gLogger.error(
"ProxyStorage.getFile: Failed to prepare file on remote server.",
res["Message"])
failed[src_url] = res["Message"]
else:
fileName = os.path.basename(src_url)
if localPath:
dest_file = "%s/%s" % (localPath, fileName)
else:
dest_file = "%s/%s" % (os.getcwd(), fileName)
res = transferClient.receiveFile(dest_file,
"getFile/%s" % fileName)
if not res["OK"]:
gLogger.error(
"ProxyStorage.getFile: Failed to recieve file from proxy server.",
res["Message"])
failed[src_url] = res["Message"]
elif not os.path.exists(dest_file):
errStr = "ProxyStorage.getFile: The destination local file does not exist."
gLogger.error(errStr, dest_file)
failed[src_url] = errStr
else:
destSize = getSize(dest_file)
if destSize == -1:
errStr = "ProxyStorage.getFile: Failed to get the local file size."
gLogger.error(errStr, dest_file)
failed[src_url] = errStr
else:
successful[src_url] = destSize
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def getSEDump(self, seNames, outputFilename):
"""
Dump the content of SEs in the given file.
The file contains a list of [SEName, lfn,checksum,size] dumped as csv,
separated by '|'
:param seName: list of StorageElement names
:param outputFilename: path to the file where to dump it
:returns: result from the TransferClient
"""
if isinstance(seNames, six.string_types):
seNames = seNames.split(",")
seNames = json.dumps(seNames)
dfc = TransferClient(self.serverURL, timeout=3600)
return dfc.receiveFile(outputFilename, seNames)
def putDirectory(self, path):
"""Put a local directory to the physical storage together with all its files and subdirectories."""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
gLogger.debug("DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len(urls))
transferClient = TransferClient(self.url)
for destDir, sourceDir in urls.items():
tmpList = os.listdir(sourceDir)
sourceFiles = ["%s/%s" % (sourceDir, x) for x in tmpList]
res = transferClient.sendBulk(sourceFiles, destDir)
if res["OK"]:
successful[destDir] = {"Files": 0, "Size": 0}
else:
failed[destDir] = res["Message"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res["OK"]:
return res
localCache = False
srcDict = res["Value"]
if srcDict["Protocol"] in ["dips", "dip"]:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict["Path"] = "/".join(srcDict["Path"].split("/")[:3])
serviceDict["FileName"] = ""
res = pfnunparse(serviceDict)
if not res["OK"]:
return res
srcSEURL = res["Value"]
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(srcDict["FileName"], os.path.join(srcDict["Path"], srcDict["FileName"]))
if not res["OK"]:
return res
src_file = srcDict["FileName"]
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res["OK"]:
return S_OK(sourceSize)
return res