def storeBundle(self, url, remoteDirPath, remoteStashPrefix="A", userName=None, password=None):
""" Store a copy of the bundled search dependencies remotely -
Args:
url (str): URL string for the destination host (e.g. sftp://myserver.net or None for a local file)
remoteDirPath (str): remote directory path on the remote resource
remoteStashPrefix (str, optional): optional label preppended to the stashed dependency bundle artifact (default='A')
userName (str, optional): optional access information. Defaults to None.
password (str, optional): optional access information. Defaults to None.
Returns:
bool: True for success or False otherwise
"""
try:
ok = False
fn = self.__makeBundleFileName(self.__baseBundleFileName, remoteStashPrefix=remoteStashPrefix)
if url and url.startswith("sftp://"):
sftpU = SftpUtil()
hostName = url[7:]
ok = sftpU.connect(hostName, userName, pw=password, port=22)
if ok:
remotePath = os.path.join("/", remoteDirPath, fn)
ok = sftpU.put(self.__localStashTarFilePath, remotePath)
elif not url:
fileU = FileUtil()
remotePath = os.path.join(remoteDirPath, fn)
ok = fileU.put(self.__localStashTarFilePath, remotePath)
else:
logger.error("Unsupported stash protocol %r", url)
return ok
except Exception as e:
logger.exception("For %r %r failing with %s", url, remoteDirPath, str(e))
return False
def testSftpOpsPublic(self):
"""Test case - connection and ops to public server"""
try:
sftpU = SftpUtil()
ok = sftpU.connect(self.__hostName, self.__userName, pw=self.__password, port=self.__hostPort)
self.assertTrue(ok)
fL = sftpU.listdir("/pub/example")
logger.info("listdir: %r", fL)
self.assertGreater(len(fL), 2)
#
ok = sftpU.get("/pub/example/readme.txt", os.path.join(self.__workPath, "readme.txt"))
self.assertTrue(ok)
#
result = sftpU.stat("/pub/example")
logger.info("stat: %r", result)
#
# Read-only public server - expecting a failure here
ok = sftpU.put(os.path.join(self.__workPath, "readme.txt"), "/pub/example/readme.txt")
self.assertFalse(ok)
#
sftpU.close()
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testSftpLocalWriteRead(self):
"""Test case - transfer -
"""
try:
userName = ""
pw = ""
hostName = ""
sftpU = SftpUtil()
ok = sftpU.connect(hostName, userName, pw=pw, port=22)
self.assertTrue(ok)
remoteDirPath = os.path.join("4-coastal", "OE")
localFilePath = os.environ.get("OE_LICENSE", "oe_license.txt")
remoteFilePath = os.path.join(remoteDirPath, "oe_license.txt")
ok = sftpU.mkdir(remoteDirPath)
ok = sftpU.put(localFilePath, remoteFilePath)
sftpU.get(remoteFilePath,
os.path.join(self.__workPath, "oe_license.txt"))
#
result = sftpU.listdir(remoteDirPath)
logger.info("listdir: %r", result)
ok = sftpU.close()
self.assertEqual(ok, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testSftpLocal(self):
"""Test case - connection to a local private server -"""
try:
userName = ""
pw = ""
hostName = ""
sftpU = SftpUtil()
ok = sftpU.connect(hostName, userName, pw=pw, port=22)
self.assertTrue(ok)
fL = sftpU.listdir("4-coastal")
logger.info("listdir: %r", fL)
self.assertGreater(len(fL), 2)
ok = sftpU.close()
self.assertEqual(ok, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def restoreDependencies(self,
url,
dirPath,
bundleLabel="A",
userName=None,
pw=None):
"""Restore bundled dependencies from remote storage and unbundle these in the
current local cache directory.
Args:
url (str): remote URL
dirPath (str): remote directory path on the
bundleLabel (str, optional): optional label preppended to the stashed dependency bundle artifact (default='A')
userName (str, optional): optional access information. Defaults to None.
password (str, optional): optional access information. Defaults to None.
"""
try:
ok = False
fileU = FileUtil()
fn = self.__makeBundleFileName(self.__dependFileName,
bundleLabel=bundleLabel)
if not url:
remotePath = os.path.join(dirPath, fn)
ok = fileU.get(remotePath, self.__dependTarFilePath)
elif url and url.startswith("http://"):
remotePath = url + os.path.join("/", dirPath, fn)
ok = fileU.get(remotePath, self.__dependTarFilePath)
elif url and url.startswith("sftp://"):
sftpU = SftpUtil()
ok = sftpU.connect(url[7:], userName, pw=pw, port=22)
if ok:
remotePath = os.path.join(dirPath, fn)
ok = sftpU.get(remotePath, self.__dependTarFilePath)
else:
logger.error("Unsupported protocol %r", url)
if ok:
ok = fileU.unbundleTarfile(self.__dependTarFilePath,
dirPath=self.__cachePath)
return ok
except Exception as e:
logger.exception("For %r %r Failing with %s", url, dirPath, str(e))
ok = False
return ok
def fetchBundle(self, localRestoreDirPath, url, remoteDirPath, remoteStashPrefix="A", userName=None, password=None):
"""Restore bundled dependencies from remote storage and unbundle these in the
current local cache directory.
Args:
localRestoreDirPath (str): local restore path
url (str): remote URL
remoteDirPath (str): remote directory path on the remote resource
remoteStashPrefix (str, optional): optional label preppended to the stashed dependency bundle artifact (default='A')
userName (str, optional): optional access information. Defaults to None.
password (str, optional): optional access information. Defaults to None.
"""
try:
ok = False
fileU = FileUtil()
fn = self.__makeBundleFileName(self.__baseBundleFileName, remoteStashPrefix=remoteStashPrefix)
if not url:
remotePath = os.path.join(remoteDirPath, fn)
if fileU.exists(remotePath):
ok = fileU.get(remotePath, self.__localStashTarFilePath)
else:
ok = False
logger.warning("Missing bundle file %r", remotePath)
elif url and (url.startswith("http://") or url.startswith("https://")):
remotePath = url + os.path.join("/", remoteDirPath, fn)
ok = fileU.get(remotePath, self.__localStashTarFilePath)
elif url and url.startswith("sftp://"):
sftpU = SftpUtil()
ok = sftpU.connect(url[7:], userName, pw=password, port=22)
if ok:
remotePath = os.path.join(remoteDirPath, fn)
ok = sftpU.get(remotePath, self.__localStashTarFilePath)
else:
logger.error("Unsupported protocol %r", url)
if ok:
ok = fileU.unbundleTarfile(self.__localStashTarFilePath, dirPath=localRestoreDirPath)
return ok
except Exception as e:
logger.exception("For %r %r Failing with %s", url, remoteDirPath, str(e))
ok = False
return ok
def testSftpLocalTransferOps(self):
"""Test case - transfer and remove files and directories -"""
try:
userName = ""
pw = ""
hostName = ""
sftpU = SftpUtil()
ok = sftpU.connect(hostName, userName, pw=pw, port=22)
self.assertTrue(ok)
testDirPath = os.path.join("4-coastal", "test")
testFilePath1 = os.path.join(testDirPath, "TEST-FILE-1.DAT")
testFilePath2 = os.path.join(testDirPath, "TEST-FILE-2.DAT")
ok = sftpU.mkdir(testDirPath)
ok = sftpU.put(self.__testLocalFilePath, testFilePath1)
ok = sftpU.put(self.__testLocalFilePath, testFilePath2)
#
sftpU.get(testFilePath1, os.path.join(self.__workPath, "TEST-FILE-1.DAT"))
sftpU.get(testFilePath2, os.path.join(self.__workPath, "TEST-FILE-2.DAT"))
#
result = sftpU.listdir(testDirPath)
logger.info("listdir: %r", result)
ok = sftpU.remove(testFilePath1)
ok = sftpU.remove(testFilePath2)
#
result = sftpU.listdir(testDirPath)
logger.debug("listdir: %r", result)
#
ok = sftpU.rmdir(testDirPath)
result = sftpU.listdir("4-coastal")
logger.info("listdir after remove: %r", result)
ok = sftpU.close()
self.assertEqual(ok, True)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()