def getDirectory(self, path, localPath=False): """Get a local copy in the current directory of a physical file specified by its path""" res = checkArgumentFormat(path) if not res["OK"]: return res urls = res["Value"] failed = {} successful = {} gLogger.debug("DIPStorage.getDirectory: Attempting to get local copies of %s directories." % len(urls)) transferClient = TransferClient(self.url) for src_dir in urls: if localPath: dest_dir = localPath else: dest_dir = os.getcwd() if not os.path.exists(dest_dir): os.mkdir(dest_dir) res = transferClient.receiveBulk(dest_dir, src_dir) if res["OK"]: gLogger.debug("DIPStorage.getDirectory: Successfully got local copy of %s" % src_dir) successful[src_dir] = {"Files": 0, "Size": 0} else: gLogger.error("DIPStorage.getDirectory: Failed to get entire directory.", src_dir) failed[src_dir] = res["Message"] resDict = {"Failed": failed, "Successful": successful} return S_OK(resDict)
def __getTransferClient(self): """Get transfer client :return: TransferClient() """ if self.transferClient: return self.transferClient return TransferClient("Framework/BundleDelivery", skipCACheck=skipCACheck(), httpsClient=BundleDeliveryJSONClient)
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res["OK"]: return res localCache = False srcDict = res["Value"] if srcDict["Protocol"] in ["dips", "dip"]: # Make the service URL from the file URL by stripping off the file part serviceDict = dict(srcDict) serviceDict["Path"] = "/".join(srcDict["Path"].split("/")[:3]) serviceDict["FileName"] = "" res = pfnunparse(serviceDict) if not res["OK"]: return res srcSEURL = res["Value"] localCache = True transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile(srcDict["FileName"], os.path.join(srcDict["Path"], srcDict["FileName"])) if not res["OK"]: return res src_file = srcDict["FileName"] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res["OK"]: return S_OK(sourceSize) return res
def getSEDump(self, seName, outputFilename): """ Dump the content of an SE in the given file. The file contains a list of [lfn,checksum,size] dumped as csv, separated by '|' :param seName: name of the StorageElement :param outputFilename: path to the file where to dump it :returns: result from the TransferClient """ dfc = TransferClient(self.serverURL) return dfc.receiveFile(outputFilename, seName)
def putFile(self, path, sourceSize=0): client = Client(url=self.url) if sourceSize: gLogger.debug( "ProxyStorage.putFile: The client has provided the source file size implying\ a replication is requested.") return client.callProxyMethod(self.name, "putFile", [path], {"sourceSize": sourceSize}) gLogger.debug( "ProxyStorage.putFile: No source size was provided therefore a simple put will be performed." ) res = checkArgumentFormat(path) if not res["OK"]: return res urls = res["Value"] failed = {} successful = {} # make sure transferClient uses the same ProxyStorage instance we uploaded the file to transferClient = TransferClient(client.serviceURL) for dest_url, src_file in urls.items(): fileName = os.path.basename(dest_url) res = transferClient.sendFile(src_file, "putFile/%s" % fileName) if not res["OK"]: gLogger.error( "ProxyStorage.putFile: Failed to send file to proxy server.", res["Message"]) failed[dest_url] = res["Message"] else: res = client.uploadFile(self.name, dest_url) if not res["OK"]: gLogger.error( "ProxyStorage.putFile: Failed to upload file to storage element from proxy server.", res["Message"], ) failed[dest_url] = res["Message"] else: res = self.__executeOperation(dest_url, "getFileSize") if not res["OK"]: gLogger.error( "ProxyStorage.putFile: Failed to determine destination file size.", res["Message"]) failed[dest_url] = res["Message"] else: successful[dest_url] = res["Value"] resDict = {"Failed": failed, "Successful": successful} return S_OK(resDict)
def __getFile(self, src_url, dest_file): transferClient = TransferClient(self.url) res = transferClient.receiveFile(dest_file, src_url, token=self.checkSum) if not res["OK"]: return res if not os.path.exists(dest_file): errStr = "DIPStorage.__getFile: The destination local file does not exist." gLogger.error(errStr, dest_file) return S_ERROR(errStr) destSize = getSize(dest_file) if destSize == -1: errStr = "DIPStorage.__getFile: Failed to get the local file size." gLogger.error(errStr, dest_file) return S_ERROR(errStr) return S_OK(destSize)
def getFile(self, path, localPath=False): res = checkArgumentFormat(path) if not res["OK"]: return res urls = res["Value"] failed = {} successful = {} client = Client(url=self.url) # Make sure transferClient uses the same ProxyStorage instance. # Only the this one holds the file we want to transfer. transferClient = TransferClient(client.serviceURL) for src_url in urls.keys(): res = client.prepareFile(self.name, src_url) if not res["OK"]: gLogger.error( "ProxyStorage.getFile: Failed to prepare file on remote server.", res["Message"]) failed[src_url] = res["Message"] else: fileName = os.path.basename(src_url) if localPath: dest_file = "%s/%s" % (localPath, fileName) else: dest_file = "%s/%s" % (os.getcwd(), fileName) res = transferClient.receiveFile(dest_file, "getFile/%s" % fileName) if not res["OK"]: gLogger.error( "ProxyStorage.getFile: Failed to recieve file from proxy server.", res["Message"]) failed[src_url] = res["Message"] elif not os.path.exists(dest_file): errStr = "ProxyStorage.getFile: The destination local file does not exist." gLogger.error(errStr, dest_file) failed[src_url] = errStr else: destSize = getSize(dest_file) if destSize == -1: errStr = "ProxyStorage.getFile: Failed to get the local file size." gLogger.error(errStr, dest_file) failed[src_url] = errStr else: successful[src_url] = destSize resDict = {"Failed": failed, "Successful": successful} return S_OK(resDict)
def getSEDump(self, seNames, outputFilename): """ Dump the content of SEs in the given file. The file contains a list of [SEName, lfn,checksum,size] dumped as csv, separated by '|' :param seName: list of StorageElement names :param outputFilename: path to the file where to dump it :returns: result from the TransferClient """ if isinstance(seNames, six.string_types): seNames = seNames.split(",") seNames = json.dumps(seNames) dfc = TransferClient(self.serverURL, timeout=3600) return dfc.receiveFile(outputFilename, seNames)
def putDirectory(self, path): """Put a local directory to the physical storage together with all its files and subdirectories.""" res = checkArgumentFormat(path) if not res["OK"]: return res urls = res["Value"] successful = {} failed = {} gLogger.debug("DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len(urls)) transferClient = TransferClient(self.url) for destDir, sourceDir in urls.items(): tmpList = os.listdir(sourceDir) sourceFiles = ["%s/%s" % (sourceDir, x) for x in tmpList] res = transferClient.sendBulk(sourceFiles, destDir) if res["OK"]: successful[destDir] = {"Files": 0, "Size": 0} else: failed[destDir] = res["Message"] resDict = {"Failed": failed, "Successful": successful} return S_OK(resDict)
def __getTransferClient(self): if not self.transferClient: return TransferClient("Accounting/ReportGenerator") else: return self.transferClient
def __getTransferClient(self): if self.transferClient: return self.transferClient return TransferClient(self.serviceName)
def __getTransferClient(self): """Get RPC client for TransferClient""" if self.__transferClient: return self.__transferClient else: return TransferClient(self.__serviceName, **self.__kwargs)