def test_init(self):
rem_exists = package.RemoteModelIndex.remote_exists
package.RemoteModelIndex.remote_exists = lambda x: True
try:
remote = package.RemoteModelIndex()
remote.cache = self.tmpdir
finally:
package.RemoteModelIndex.remote_exists = rem_exists
self.assertEqual(remote.org_name, 'macsy-models')
self.assertEqual(remote.base_url, 'https://api.github.com')
self.assertEqual(remote.cache, self.tmpdir)
package.RemoteModelIndex.remote_exists = lambda x: True
try:
remote = package.RemoteModelIndex(org='foo')
remote.cache = self.tmpdir
finally:
package.RemoteModelIndex.remote_exists = rem_exists
self.assertEqual(remote.org_name, 'foo')
package.RemoteModelIndex.remote_exists = lambda x: False
try:
with self.assertRaises(ValueError) as ctx:
package.RemoteModelIndex(org='foo')
finally:
package.RemoteModelIndex.remote_exists = rem_exists
self.assertEqual(str(ctx.exception), "the 'foo' organization does not exist.")
def test_get_metadata(self, mock_urlopen):
rem_exists = package.RemoteModelIndex.remote_exists
list_package_vers = package.RemoteModelIndex.list_package_vers
try:
vers = '0.0'
pack_name = 'foo'
package.RemoteModelIndex.remote_exists = lambda x: True
package.RemoteModelIndex.list_package_vers = lambda x, pack_name: [vers]
remote = package.RemoteModelIndex(org="get_metadata")
remote.cache = self.tmpdir
metadata = remote.get_metadata(pack_name)
self.assertDictEqual(metadata, {"maintainer": {"name": "moi"}})
finally:
package.RemoteModelIndex.remote_exists = rem_exists
package.RemoteModelIndex.list_package_vers = list_package_vers
#################################################
# The remote package is not versioned (tagged) #
#################################################
try:
package.RemoteModelIndex.remote_exists = lambda x: True
package.RemoteModelIndex.list_package_vers = lambda x, pack_name: []
remote = package.RemoteModelIndex(org="get_metadata")
with self.assertRaises(MacsydataError) as ctx:
remote.get_metadata(pack_name)
self.assertEqual(str(ctx.exception),
"No official version available for model 'foo'")
finally:
package.RemoteModelIndex.remote_exists = rem_exists
package.RemoteModelIndex.list_package_vers = list_package_vers
#####################################
# The pack version is not available #
#####################################
try:
package.RemoteModelIndex.remote_exists = lambda x: True
package.RemoteModelIndex.list_package_vers = lambda x, pack_name: ["12"]
remote = package.RemoteModelIndex(org="get_metadata")
with self.assertRaises(RuntimeError) as ctx:
remote.get_metadata(pack_name, vers="1.1")
self.assertEqual(str(ctx.exception),
"The version '1.1' does not exists for model foo.")
finally:
package.RemoteModelIndex.remote_exists = rem_exists
package.RemoteModelIndex.list_package_vers = list_package_vers
def test_list_packages(self, mock_urlopen):
rem_exists = package.RemoteModelIndex.remote_exists
try:
package.RemoteModelIndex.remote_exists = lambda x: True
remote = package.RemoteModelIndex(org="list_packages")
remote.cache = self.tmpdir
self.assertListEqual(remote.list_packages(), ['model_1', 'model_2'])
finally:
package.RemoteModelIndex.remote_exists = rem_exists
def test_url_json(self, mock_urlopen):
rem_exists = package.RemoteModelIndex.remote_exists
package.RemoteModelIndex.remote_exists = lambda x: True
remote = package.RemoteModelIndex(org="nimportnaoik")
remote.cache = self.tmpdir
try:
j = remote._url_json("https://test_url_json/")
self.assertDictEqual(j, {'fake': ['json', 'response']})
finally:
package.RemoteModelIndex.remote_exists = rem_exists
def test_url_json_reach_limit(self, mock_urlopen):
rem_exists = package.RemoteModelIndex.remote_exists
package.RemoteModelIndex.remote_exists = lambda x: True
remote = package.RemoteModelIndex(org="nimportnaoik")
remote.cache = self.tmpdir
try:
with self.assertRaises(MacsyDataLimitError) as ctx:
remote._url_json("https://test_url_json/limit")
self.assertEqual(str(ctx.exception),
"""You reach the maximum number of request per hour to github.
Please wait before to try again.""")
finally:
package.RemoteModelIndex.remote_exists = rem_exists
def test_download(self, mock_urlopen):
rem_exists = package.RemoteModelIndex.remote_exists
try:
package.RemoteModelIndex.remote_exists = lambda x: True
remote = package.RemoteModelIndex(org="package_download")
remote.cache = self.tmpdir
pack_name = "fake"
pack_vers = "1.0"
# ensure that remote.cache does not exists
if os.path.exists(remote.cache):
if os.path.isdir(remote.cache):
shutil.rmtree(remote.cache)
elif os.path.isfile(remote.cache) or os.path.islink(remote.cache):
os.unlink(remote.cache)
arch_path = remote.download(pack_name, pack_vers)
self.assertEqual(os.path.join(remote.cache, remote.org_name, f"{pack_name}-{pack_vers}.tar.gz"),
arch_path)
self.assertFileEqual(arch_path, io.StringIO('fake data ' * 2))
# download again with existing remote.cache and replace old archive
os.unlink(arch_path)
arch_path = remote.download(pack_name, pack_vers)
self.assertFileEqual(arch_path, io.StringIO('fake data ' * 2))
# download again with existing remote.cache and replace old archive
os.unlink(arch_path)
dest = os.path.join(self.tmpdir, 'dest')
os.makedirs(dest)
arch_path = remote.download(pack_name, pack_vers, dest=dest)
self.assertEqual(os.path.join(dest, f'{pack_name}-{pack_vers}.tar.gz'), arch_path)
# remote cache exist and is a file
shutil.rmtree(remote.cache)
open(remote.cache, 'w').close()
try:
with self.assertRaises(NotADirectoryError) as ctx:
remote.download(pack_name, pack_vers)
self.assertEqual(str(ctx.exception),
f"The tmp cache '{remote.cache}' already exists")
finally:
os.unlink(remote.cache)
with self.assertRaises(ValueError) as ctx:
_ = remote.download("bad_pack", "0.2")
self.assertEqual(str(ctx.exception),
"package 'bad_pack-0.2' does not exists on repos 'package_download'")
finally:
package.RemoteModelIndex.remote_exists = rem_exists
def test_list_package_vers(self, mock_urlopen):
rem_exists = package.RemoteModelIndex.remote_exists
try:
package.RemoteModelIndex.remote_exists = lambda x: True
remote = package.RemoteModelIndex(org="list_package_vers")
remote.cache = self.tmpdir
finally:
package.RemoteModelIndex.remote_exists = rem_exists
self.assertListEqual(remote.list_package_vers('model_1'), ['v_1', 'v_2'])
with self.assertRaises(ValueError) as ctx:
_ = remote.list_package_vers('model_2')
self.assertEqual(str(ctx.exception), "package 'model_2' does not exists on repos 'list_package_vers'")
with self.assertRaises(urllib.error.HTTPError) as ctx:
_ = remote.list_package_vers('model_3')
self.assertEqual(str(ctx.exception), "HTTP Error 500: Server Error")
def test_unarchive(self):
def create_pack(dir_, repo, name, vers, key):
pack_name = f"{name}-{vers}"
tar_path = os.path.join(dir_, f"{pack_name}.tar.gz")
with tarfile.open(tar_path, "w:gz") as tar:
tmp_pack = os.path.join(dir_, f"{repo}-{name}-{key}")
os.mkdir(tmp_pack)
for i in range(3):
name = f"file_{i}"
tmp_file = os.path.join(tmp_pack, name)
with open(tmp_file, 'w') as f:
f.write(f"Content of file {i}\n")
tar.add(tmp_pack, arcname=os.path.basename(tmp_pack))
shutil.rmtree(tmp_pack)
return tar_path
pack_name = 'model-toto'
pack_vers = '2.0'
rem_exists = package.RemoteModelIndex.remote_exists
package.RemoteModelIndex.remote_exists = lambda x: True
try:
remote = package.RemoteModelIndex(org="package_unarchive")
arch = create_pack(self.tmpdir, remote.org_name, pack_name, pack_vers, 'e020300')
remote.cache = self.tmpdir
model_path = remote.unarchive_package(arch)
unpacked_path = os.path.join(self.tmpdir, remote.org_name, pack_name, pack_vers, pack_name)
self.assertEqual(model_path, unpacked_path)
self.assertTrue(os.path.exists(unpacked_path))
self.assertTrue(os.path.isdir(unpacked_path))
self.assertListEqual(sorted(glob.glob(f"{unpacked_path}/*")),
sorted([os.path.join(unpacked_path, f"file_{i}") for i in range(3)])
)
# test package is remove before to unarchive a new one
open(os.path.join(unpacked_path, f"file_must_be_removed"), 'w').close()
model_path = remote.unarchive_package(arch)
self.assertListEqual(sorted(glob.glob(f"{unpacked_path}/*")),
sorted([os.path.join(unpacked_path, f"file_{i}") for i in range(3)])
)
finally:
package.RemoteModelIndex.remote_exists = rem_exists
def test_remote_exists(self, mock_urlopen):
remote = package.RemoteModelIndex(org="remote_exists_true")
remote.cache = self.tmpdir
exists = remote.remote_exists()
self.assertTrue(exists)
remote.org_name = "remote_exists_false"
exists = remote.remote_exists()
self.assertFalse(exists)
remote.org_name = "remote_exists_server_error"
with self.assertRaises(urllib.error.HTTPError) as ctx:
remote.remote_exists()
self.assertEqual(str(ctx.exception),
"HTTP Error 500: Server Error")
remote.org_name = "remote_exists_unexpected_error"
with self.assertRaises(urllib.error.HTTPError) as ctx:
remote.remote_exists()
self.assertEqual(str(ctx.exception),
"HTTP Error 204: No Content")