def setUp(self):
unittest.TestCase.setUp(self)
self._tmpdir = TempDir(suffix="aux_tempdir")
self._tmpenv = TempEnv()
self._exiting_dir = os.path.join(self._tmpdir.path(), "ThisDir")
os.mkdir(self._exiting_dir)
self._tmpenv["ELEMENTS_AUX_PATH"] = self._tmpdir.path()
self._test_files = []
self._test_files.append(
AuxiliaryTest.TestFile(os.path.join(self._tmpdir.path(), "file1"),
"That content no replacement"))
self._test_files.append(
AuxiliaryTest.TestFile(
os.path.join(self._tmpdir.path(), "tata", "file1"),
"This content no replacement"))
self._test_files.append(
AuxiliaryTest.TestFile(
os.path.join(self._tmpdir.path(), "tata", "tutu", "file1"),
"This content no replacement"))
self._test_files.append(
AuxiliaryTest.TestFile(os.path.join(self._tmpdir.path(), "file2"),
"That content %(bla)s %(blu)s"))
self._test_files.append(
AuxiliaryTest.TestFile(
os.path.join(self._tmpdir.path(), "tata", "file2"),
"This content %(bla)s %(blu)s"))
self._test_files.append(
AuxiliaryTest.TestFile(
os.path.join(self._tmpdir.path(), "tata", "tutu", "file2"),
"This content %(bla)s %(blu)s"))
def setUp(self):
unittest.TestCase.setUp(self)
self.samplebasename = "toto"
self.tmpdir = TempDir(suffix="tempdir", prefix=self.samplebasename)
self.m_top_dir = TempDir()
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(), "work")
def testDestruction(self):
mydir = TempDir(suffix="tempdir", prefix="mydir")
mydirname = mydir.getName()
self.assertTrue(os.path.exists(mydirname))
self.assertTrue(os.path.isdir(mydirname))
del mydir
# the temporary directory should have been removed
self.assertFalse(os.path.exists(mydirname))
class TestDataSynchronizerMaker(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.m_top_dir = TempDir(prefix="DataSync_test")
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(),
"workspace")
def tearDown(self):
unittest.TestCase.tearDown(self)
del self.m_top_dir
def checkSynchronizerCreation(self, connection_config,
expected_synchronizer):
connection = ConnectionConfiguration(connection_config)
dependency = DependencyConfiguration(theDependencyConfig(),
connection.local_root)
synchronizer = createSynchronizer(connection, dependency)
assert type(synchronizer), expected_synchronizer
def testSynchronizerCreation(self):
webdav_configs = {
theWebdavFrConfig(): WebdavSynchronizer,
}
irods_configs = {theIrodsFrConfig(): IrodsSynchronizer}
if not IrodsSynchronizer.irodsIsInstalled():
irods_configs = {}
configs = webdav_configs.copy()
configs.update(irods_configs)
for config, expected in configs.items():
self.checkSynchronizerCreation(config, expected)
class TestDataSynchronizer(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.m_top_dir = TempDir(prefix="DataSync_test")
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(), "workspace")
def tearDown(self):
unittest.TestCase.tearDown(self)
del self.m_top_dir
def testDownloadFailure(self):
mock = MockDataSynchronizer()
with py.test.raises(DownloadFailed):
mock.downloadAllFiles()
def testOverwritingPolicy(self):
the_filename = "SayThisFileHasAlreadyBeenDownloaded.txt"
with open(the_filename, "w") as f:
f.write("Some contents")
mock_overwrite = MockDataSynchronizer()
mock_no_overwrite = MockDataSynchronizer(
connection=theNoOverwriteConfig())
assert mock_overwrite.fileAlreadyExists(the_filename)
assert mock_overwrite.fileShouldBeWritten(the_filename)
assert mock_no_overwrite.fileAlreadyExists(the_filename)
assert not mock_no_overwrite.fileShouldBeWritten(the_filename)
os.remove(the_filename)
class TestDataSyncUtils(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.m_top_dir = TempDir(prefix="DataSync_test")
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(),
"workspace")
def tearDown(self):
unittest.TestCase.tearDown(self)
del self.m_top_dir
def test_configFileFound(self):
rel_path_to_some_file = "ElementsServices/testdata/test_file_list.txt"
abs_path_to_some_file = DataSyncUtils.dataSyncConfFilePath(
rel_path_to_some_file)
assert os.path.isfile(abs_path_to_some_file)
def test_runCommandAndCaptureOutErr(self):
the_message = "Hello, Antoine!"
out, err = DataSyncUtils.runCommandAndCaptureOutErr("echo " +
the_message)
out = out.replace("\n", "")
err = err.replace("\n", "")
assert out == the_message, "Command output misread: " + out + " != " + the_message
assert not err
def test_createLocalDirOf(self):
the_dir = "/tmp/somewhere/some_temporary_test_data"
the_filename = "some_test_file.txt"
the_path = DataSyncUtils.concatenatePaths(
[DataSyncUtils.localWorkspacePrefix(), the_dir, the_filename])
DataSyncUtils.createLocalDirOf(the_path)
assert os.path.isdir(
DataSyncUtils.concatenatePaths(
[DataSyncUtils.localWorkspacePrefix(), the_dir]))
os.rmdir(
DataSyncUtils.concatenatePaths(
[DataSyncUtils.localWorkspacePrefix(), the_dir]))
def test_localWorkspacePrefix(self):
prefix_ev = DataSyncUtils.environmentVariable('NOPREFIX')
workspace_ev = DataSyncUtils.environmentVariable('WORKSPACE')
prefix = DataSyncUtils.localWorkspacePrefix()
if prefix_ev == '':
assert prefix == workspace_ev
else:
assert prefix == ''
def checkConcatenatePaths(self, chunks, expected):
found = DataSyncUtils.concatenatePaths(chunks)
assert found == expected
def test_concatenatePaths(self):
self.checkConcatenatePaths(["", "/tmp"], "/tmp")
self.checkConcatenatePaths(["", "/tmp/"], "/tmp")
self.checkConcatenatePaths(["", "/tmp", "file.fits"], "/tmp/file.fits")
self.checkConcatenatePaths(["", "/tmp/", "/data/file.fits"],
"/tmp/data/file.fits")
self.checkConcatenatePaths(["/prefix", "/tmp"], "/prefix/tmp")
class TestDependencyConfiguration(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.m_top_dir = TempDir(prefix="DataSync_test")
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(),
"workspace")
def tearDown(self):
unittest.TestCase.tearDown(self)
del self.m_top_dir
def test_parseLineWithoutAlias(self):
distant_root = "distant/"
local_root = "local/"
config = DependencyConfiguration(distant_root, local_root)
file = "test/file.fits"
assert not config.lineHasAlias(file)
config.parseLineWithoutAlias(file)
assert config.getDependencyCount() == 1
assert config.getDistantPaths()[0] == distant_root + file
abs_local_file = concatenatePaths(
[localWorkspacePrefix(), local_root, file])
assert config.getLocalPaths()[0] == abs_local_file
assert config.getDistantPathOf(abs_local_file) == distant_root + file
def test_parseLineWithAlias(self):
distant_root = "distant/"
local_root = "local/"
config = DependencyConfiguration(distant_root, local_root)
distant = "test/file_v1.fits"
local = "test/file.fits"
line = distant + config.getAliasSeparator() + local
assert config.lineHasAlias(line)
config.parseLineWithAlias(line)
assert config.getDependencyCount() == 1
assert config.getDistantPaths()[0] == distant_root + distant
abs_local_file = concatenatePaths(
[localWorkspacePrefix(), local_root, local])
assert config.getLocalPaths()[0] == abs_local_file
assert config.getDistantPathOf(
abs_local_file) == distant_root + distant
def test_confDependencies(self):
the_config_file = getConfigurationPath(theDependencyConfig())
dependencies = DependencyConfiguration("", thePrefixedLocalWorkspace(),
the_config_file)
parsed_files = dependencies.getLocalPaths()
assert len(parsed_files) == len(
theLocalFiles()), "Input cardinal is different from read one"
the_local_abs_paths = [
concatenatePaths([thePrefixedLocalWorkspace(), f])
for f in theLocalFiles()
]
for f in the_local_abs_paths:
assert f in parsed_files, "Input file not read"
for f in parsed_files:
assert f in the_local_abs_paths, "File not in input list"
def testKeepTmpDir(self):
os.environ["KEEPTEMPDIR"] = "1"
with TempDir() as td:
td_path = td.path()
self.assertTrue(os.path.exists(td_path))
self.assertTrue(os.path.exists(td_path))
rmtree(td_path)
self.assertFalse(os.path.exists(td_path))
del os.environ["KEEPTEMPDIR"]
class TestIrodsIsInstalled(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.m_top_dir = TempDir(prefix="DataSync_test")
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(),
"workspace")
def tearDown(self):
unittest.TestCase.tearDown(self)
del self.m_top_dir
def test_irodsIsInstalled(self):
assert IrodsSynchronizer.irodsIsInstalled()
class TestConnectionConfiguration(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.m_top_dir = TempDir(prefix="DataSync_test")
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(),
"workspace")
def tearDown(self):
unittest.TestCase.tearDown(self)
del self.m_top_dir
def checkHostParsing(self, host, names):
config = ConnectionConfiguration()
for name in names:
config.parseHost(name)
assert config.host == host, "'" + name + "' not parsed as " + host.name
def test_hostParsing(self):
self.checkHostParsing(DataHost.IRODS, ["irods", "iRODS", "IRODS"])
self.checkHostParsing(DataHost.WEBDAV, ["webdav", "WebDAV", "WEBDAV"])
self.checkHostParsing(None, ["blob", "Blob", "BLOB"])
def test_overwritingPolicy(self):
config = ConnectionConfiguration(theWebdavFrConfig())
config_no_overwrite = ConnectionConfiguration(theNoOverwriteConfig())
assert config.overwritingAllowed(), \
"Overwrite should be allowed, but it is not"
assert not config_no_overwrite.overwritingAllowed(), \
"Overwrite should not be allowed, but it is"
def test_webdavFrConfig(self):
config = ConnectionConfiguration(theWebdavFrConfig())
assert config.host == DataHost.WEBDAV, \
"Host shoud be WebDAV, but it is not"
assert config.overwriting_policy, \
"Overwrite should be allowed, but it is not"
assert config.distant_root == "/euclid-fr/ct/mock_test_data", \
"Distant workspace path misread: " + config.distant_root
assert config.local_root == thePrefixedLocalWorkspace(), \
"Local workspace path misread: " + config.local_root
assert config.tries == 8, \
"Wrong number of tries: " + str(config.tries)
class TestDataSync(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.m_top_dir = TempDir(prefix="DataSync_test")
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(),
"workspace")
def tearDown(self):
unittest.TestCase.tearDown(self)
del self.m_top_dir
def checkDownload(self, connection_config):
sync = DataSync(connection_config, theDependencyConfig())
sync.download()
for file in theLocalFiles():
abs_path = sync.absolutePath(file)
assert os.path.isfile(abs_path)
os.remove(abs_path)
def checkFallback(self, fallback_config):
sync = DataSync(aBadConnectionConfig(), theDependencyConfig())
with py.test.raises(Exception):
sync.download()
sync.downloadWithFallback(fallback_config)
for file in theLocalFiles():
abs_path = sync.absolutePath(file)
assert os.path.isfile(abs_path)
os.remove(abs_path)
def test_DataSyncWebdavFr(self):
if webdavIsInstalled():
self.checkDownload(theWebdavFrConfig())
self.checkFallback(theWebdavFrConfig())
def test_DataSyncIrodsFr(self):
if irodsIsInstalled():
self.checkDownload(theIrodsFrConfig())
self.checkFallback(theIrodsFrConfig())
class TestCase(unittest.TestCase):
""" test case for the temporary directory class """
def setUp(self):
unittest.TestCase.setUp(self)
self.samplebasename = "toto"
self.tmpdir = TempDir(suffix="tempdir", prefix=self.samplebasename)
self.m_top_dir = TempDir()
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(), "work")
def tearDown(self):
unittest.TestCase.tearDown(self)
del self.tmpdir
del self.m_top_dir
def testName(self):
name1 = str(self.tmpdir)
name2 = self.tmpdir.getName()
# the next line should print twice the same thing
self.assertEqual(name1, name2)
def testDestruction(self):
mydir = TempDir(suffix="tempdir", prefix="mydir")
mydirname = mydir.getName()
self.assertTrue(os.path.exists(mydirname))
self.assertTrue(os.path.isdir(mydirname))
del mydir
# the temporary directory should have been removed
self.assertFalse(os.path.exists(mydirname))
def testContext(self):
with TempDir() as td:
td_path = td.getName()
self.assertTrue(os.path.exists(td_path))
self.assertFalse(os.path.exists(td_path))
def testFileDestruction(self):
myfile = TempFile()
myfilename = myfile.getName()
self.assertTrue(os.path.exists(myfilename))
self.assertFalse(os.path.isdir(myfilename))
self.assertTrue(os.path.isfile(myfilename))
del myfile
# the temporary directory should have been removed
self.assertFalse(os.path.exists(myfilename))
def testFileContext(self):
with TempFile() as tf:
tf_path = tf.getName()
self.assertTrue(os.path.exists(tf_path))
self.assertFalse(os.path.exists(tf_path))
def testPath(self):
with TempDir() as td:
td_path = td.path()
self.assertTrue(os.path.exists(td_path))
def testEnvWith(self):
os.environ["TESTENVEK"] = "wuk"
with TempEnv() as e:
e["TESTENVEK"] = "bla"
e["TESTENVEK2"] = "bla"
self.assertEqual(os.environ.get("TESTENVEK"), "bla")
# The environment is restored
self.assertEqual(os.environ.get("TESTENVEK"), "wuk")
with TempEnv() as e:
e["TESTENVEK2"] = "bla"
self.assertEqual(os.environ.get("TESTENVEK2"), "bla")
self.assertEqual(os.environ.get("TESTENVEK2", None), None)
def testKeepTmpDir(self):
os.environ["KEEPTEMPDIR"] = "1"
with TempDir() as td:
td_path = td.path()
self.assertTrue(os.path.exists(td_path))
self.assertTrue(os.path.exists(td_path))
rmtree(td_path)
self.assertFalse(os.path.exists(td_path))
del os.environ["KEEPTEMPDIR"]
def testTempEnv2(self):
self.assertEqual(self.m_env["WORKSPACE"],
os.path.join(self.m_top_dir.path(), "work"))
with TempEnv() as local:
self.assertEqual(self.m_env["WORKSPACE"],
os.path.join(self.m_top_dir.path(), "work"))
self.assertEqual(local["WORKSPACE"],
os.path.join(self.m_top_dir.path(), "work"))
local["WORKSPACE"] = "that_work"
self.assertEqual(self.m_env["WORKSPACE"], "that_work")
self.assertEqual(local["WORKSPACE"], "that_work")
self.assertEqual(self.m_env["WORKSPACE"],
os.path.join(self.m_top_dir.path(), "work"))
def setUp(self):
unittest.TestCase.setUp(self)
self.m_top_dir = TempDir(prefix="DataSync_test")
self.m_env = TempEnv()
self.m_env["WORKSPACE"] = os.path.join(self.m_top_dir.path(),
"workspace")
def __init__(self):
m_top_dir = TempDir("DataSync_test")
m_env = TempEnv()
m_env["WORKSPACE"] = os.path.join(m_top_dir.path(), "workspace")
class AuxiliaryTest(unittest.TestCase):
class TestFile(object):
def __init__(self, path, content=""):
self.setPath(path)
self.setContent(content)
def setPath(self, path):
self._path = path
def path(self):
return self._path
def setContent(self, content):
self._content = content
if self._content:
parent_path = os.path.dirname(self._path)
if not os.path.exists(parent_path):
os.makedirs(parent_path)
with open(self._path, "w") as f:
f.write(self._content)
def content(self):
return self._content
def setUp(self):
unittest.TestCase.setUp(self)
self._tmpdir = TempDir(suffix="aux_tempdir")
self._tmpenv = TempEnv()
self._exiting_dir = os.path.join(self._tmpdir.path(), "ThisDir")
os.mkdir(self._exiting_dir)
self._tmpenv["ELEMENTS_AUX_PATH"] = self._tmpdir.path()
self._test_files = []
self._test_files.append(
AuxiliaryTest.TestFile(os.path.join(self._tmpdir.path(), "file1"),
"That content no replacement"))
self._test_files.append(
AuxiliaryTest.TestFile(
os.path.join(self._tmpdir.path(), "tata", "file1"),
"This content no replacement"))
self._test_files.append(
AuxiliaryTest.TestFile(
os.path.join(self._tmpdir.path(), "tata", "tutu", "file1"),
"This content no replacement"))
self._test_files.append(
AuxiliaryTest.TestFile(os.path.join(self._tmpdir.path(), "file2"),
"That content %(bla)s %(blu)s"))
self._test_files.append(
AuxiliaryTest.TestFile(
os.path.join(self._tmpdir.path(), "tata", "file2"),
"This content %(bla)s %(blu)s"))
self._test_files.append(
AuxiliaryTest.TestFile(
os.path.join(self._tmpdir.path(), "tata", "tutu", "file2"),
"This content %(bla)s %(blu)s"))
def tearDown(self):
unittest.TestCase.tearDown(self)
del self._tmpenv
del self._tmpdir
def testAuxPathEnv(self):
self.assertEqual(self._tmpenv["ELEMENTS_AUX_PATH"],
self._tmpdir.path())
self.assertEqual(getAuxiliaryPath("file1"),
os.path.join(self._tmpdir.path(), "file1"))
self.assertEqual(getAuxiliaryPath("tata/file1"),
os.path.join(self._tmpdir.path(), "tata", "file1"))
self.assertEqual(
getAuxiliaryPath("tata/tutu/file1"),
os.path.join(self._tmpdir.path(), "tata", "tutu", "file1"))
self.assertEqual(getAuxiliaryPath("file2"),
os.path.join(self._tmpdir.path(), "file2"))
self.assertEqual(getAuxiliaryPath("tata/file2"),
os.path.join(self._tmpdir.path(), "tata", "file2"))
self.assertEqual(
getAuxiliaryPath("tata/tutu/file2"),
os.path.join(self._tmpdir.path(), "tata", "tutu", "file2"))
def testConfigure(self):
configure("file1", self._exiting_dir)
self.assertTrue(
os.path.exists(os.path.join(self._exiting_dir, "file1")))
configure("file2",
self._exiting_dir,
configuration={
"bla": "foo",
"blu": "bar"
})
target_file = os.path.join(self._exiting_dir, "file2")
self.assertTrue(os.path.exists(target_file))
self.assertEqual(open(target_file).read(), "That content foo bar")
def testContext(self):
with TempDir() as td:
td_path = td.getName()
self.assertTrue(os.path.exists(td_path))
self.assertFalse(os.path.exists(td_path))
def testPath(self):
with TempDir() as td:
td_path = td.path()
self.assertTrue(os.path.exists(td_path))
def setUp(self):
unittest.TestCase.setUp(self)
self._tmpdir_1 = TempDir(suffix="tempdir")
self._tmpdir_2 = TempDir(suffix="tempdir")