def _get_file_downloader(insecure, cacert):
fd = FileDownloader(insecure=insecure, cacert=cacert)
arch = fd.get_sysinfo()
if arch[1] != 64:
_log.error("IDAES Extensions only supports 64bit Python.")
raise RuntimeError("IDAES Extensions only supports 64bit Python.")
return fd, arch
def test_get_sysinfo(self):
f = FileDownloader()
ans = f.get_sysinfo()
self.assertIs(type(ans), tuple)
self.assertEqual(len(ans), 2)
self.assertTrue(len(ans[0]) > 0)
self.assertTrue(platform.system().lower().startswith(ans[0]))
self.assertFalse(any(c in ans[0] for c in '.-_'))
self.assertIn(ans[1], (32, 64))
def test_get_platform_url(self):
f = FileDownloader()
urlmap = {'bogus_sys': 'bogus'}
with self.assertRaisesRegexp(
RuntimeError, "cannot infer the correct url for platform '.*'"):
f.get_platform_url(urlmap)
urlmap[f.get_sysinfo()[0]] = 'correct'
self.assertEqual(f.get_platform_url(urlmap), 'correct')
def test_get_url(self):
f = FileDownloader()
urlmap = {'bogus_sys': 'bogus'}
with self.assertRaisesRegexp(
RuntimeError, "cannot infer the correct url for platform '.*'"):
f.get_url(urlmap)
urlmap[f.get_sysinfo()[0]] = 'correct'
self.assertEqual(f.get_url(urlmap), 'correct')
def test_get_sysinfo(self):
f = FileDownloader()
ans = f.get_sysinfo()
self.assertIs(type(ans), tuple)
self.assertEqual(len(ans), 2)
self.assertTrue(len(ans[0]) > 0)
self.assertTrue(platform.system().lower().startswith(ans[0]))
self.assertFalse(any(c in ans[0] for c in '.-_'))
self.assertIn(ans[1], (32,64))
def download_binaries(url=None, verbose=False):
"""
Download IDAES solvers and libraries and put them in the right location. Need
to supply either local or url argument.
Args:
url (str): a url to download binary files to install files
Returns:
None
"""
if verbose:
_log.setLevel(idaeslog.DEBUG)
idaes._create_lib_dir()
idaes._create_bin_dir()
solvers_tar = os.path.join(idaes.bin_directory, "idaes-solvers.tar.gz")
libs_tar = os.path.join(idaes.lib_directory, "idaes-lib.tar.gz")
fd = FileDownloader()
arch = fd.get_sysinfo()
if url is not None:
if not url.endswith("/"):
c = "/"
else:
c = ""
solvers_from = c.join(
[url, "idaes-solvers-{}-{}.tar.gz".format(arch[0], arch[1])])
libs_from = c.join(
[url, "idaes-lib-{}-{}.tar.gz".format(arch[0], arch[1])])
_log.debug("URLs \n {}\n {}\n {}".format(url, solvers_from,
libs_from))
_log.debug("Destinations \n {}\n {}".format(solvers_tar, libs_tar))
if arch[0] == 'darwin':
raise Exception('Mac OSX currently unsupported')
fd.set_destination_filename(solvers_tar)
fd.get_binary_file(solvers_from)
fd.set_destination_filename(libs_tar)
fd.get_binary_file(libs_from)
else:
raise Exception("Must provide a location to download binaries")
_log.debug("Extracting files in {}".format(idaes.bin_directory))
with tarfile.open(solvers_tar, 'r') as f:
f.extractall(idaes.bin_directory)
_log.debug("Extracting files in {}".format(idaes.lib_directory))
with tarfile.open(libs_tar, 'r') as f:
f.extractall(idaes.lib_directory)
def test_find_library_system(self):
# Find a system library (before we muck with the PATH)
_args = {'cwd': False, 'include_PATH': False, 'pathlist': []}
if FileDownloader.get_sysinfo()[0] == 'windows':
a = find_library('ntdll', **_args)
b = find_library('ntdll.dll', **_args)
c = find_library('foo\\bar\\ntdll.dll', **_args)
else:
a = find_library('c', **_args)
b = find_library('libc.so', **_args)
c = find_library('foo/bar/libc.so', **_args)
self.assertIsNotNone(a)
self.assertIsNotNone(b)
self.assertIsNotNone(c)
self.assertEqual(a, b)
# find_library could have found libc.so.6
self.assertTrue(c.startswith(a))
# Verify that the library is loadable (they are all the same
# file, so only check one)
_lib = ctypes.cdll.LoadLibrary(a)
self.assertIsNotNone(_lib)
def test_find_library(self):
self.tmpdir = os.path.abspath(tempfile.mkdtemp())
os.chdir(self.tmpdir)
# Find a system library (before we muck with the PATH)
_args = {'cwd': False, 'include_PATH': False, 'pathlist': []}
if FileDownloader.get_sysinfo()[0] == 'windows':
a = find_library('ntdll', **_args)
b = find_library('ntdll.dll', **_args)
c = find_library('foo\\bar\\ntdll.dll', **_args)
else:
a = find_library('c', **_args)
b = find_library('libc.so', **_args)
c = find_library('foo/bar/libc.so', **_args)
self.assertIsNotNone(a)
self.assertIsNotNone(b)
self.assertIsNotNone(c)
self.assertEqual(a, b)
# find_library could have found libc.so.6
self.assertTrue(c.startswith(a))
# Verify that the library is loadable (they are all the same
# file, so only check one)
_lib = ctypes.cdll.LoadLibrary(a)
self.assertIsNotNone(_lib)
envvar.PYOMO_CONFIG_DIR = self.tmpdir
config_libdir = os.path.join(self.tmpdir, 'lib')
os.mkdir(config_libdir)
config_bindir = os.path.join(self.tmpdir, 'bin')
os.mkdir(config_bindir)
ldlibdir_name = 'in_ld_lib'
ldlibdir = os.path.join(self.tmpdir, ldlibdir_name)
os.mkdir(ldlibdir)
os.environ['LD_LIBRARY_PATH'] = os.pathsep + ldlibdir + os.pathsep
pathdir_name = 'in_path'
pathdir = os.path.join(self.tmpdir, pathdir_name)
os.mkdir(pathdir)
os.environ['PATH'] = os.pathsep + pathdir + os.pathsep
libExt = _libExt[_system()][0]
f_in_cwd_ldlib_path = 'f_in_cwd_ldlib_path'
open(os.path.join(self.tmpdir, f_in_cwd_ldlib_path), 'w').close()
open(os.path.join(ldlibdir, f_in_cwd_ldlib_path), 'w').close()
open(os.path.join(pathdir, f_in_cwd_ldlib_path), 'w').close()
f_in_ldlib_extension = 'f_in_ldlib_extension'
open(os.path.join(ldlibdir, f_in_ldlib_extension + libExt),
'w').close()
f_in_path = 'f_in_path'
open(os.path.join(pathdir, f_in_path), 'w').close()
f_in_configlib = 'f_in_configlib'
open(os.path.join(config_libdir, f_in_configlib), 'w').close()
f_in_configbin = 'f_in_configbin'
open(os.path.join(config_bindir, f_in_ldlib_extension), 'w').close()
open(os.path.join(config_bindir, f_in_configbin), 'w').close()
self._check_file(find_library(f_in_cwd_ldlib_path),
os.path.join(self.tmpdir, f_in_cwd_ldlib_path))
self._check_file(os.path.join(ldlibdir, f_in_cwd_ldlib_path),
find_library(f_in_cwd_ldlib_path, cwd=False))
self._check_file(
os.path.join(ldlibdir, f_in_ldlib_extension) + libExt,
find_library(f_in_ldlib_extension))
self._check_file(os.path.join(pathdir, f_in_path),
find_library(f_in_path))
if _system() == 'windows':
self._check_file(os.path.join(pathdir, f_in_path),
find_library(f_in_path, include_PATH=False))
else:
# Note that on Windows, ctypes.util.find_library *always*
# searches the PATH
self.assertIsNone(find_library(f_in_path, include_PATH=False))
self._check_file(
os.path.join(pathdir, f_in_path),
find_library(f_in_path,
pathlist=os.pathsep + pathdir + os.pathsep))
# test an explicit pathlist overrides LD_LIBRARY_PATH
self._check_file(
os.path.join(pathdir, f_in_cwd_ldlib_path),
find_library(f_in_cwd_ldlib_path, cwd=False, pathlist=[pathdir]))
# test that the PYOMO_CONFIG_DIR 'lib' dir is included
self._check_file(os.path.join(config_libdir, f_in_configlib),
find_library(f_in_configlib))
# and the Bin dir
self._check_file(os.path.join(config_bindir, f_in_configbin),
find_library(f_in_configbin))
# ... but only if include_PATH is true
self.assertIsNone(find_library(f_in_configbin, include_PATH=False))
# And none of them if the pathlist is specified
self.assertIsNone(find_library(f_in_configlib, pathlist=pathdir))
self.assertIsNone(find_library(f_in_configbin, pathlist=pathdir))
def download_binaries(release=None, url=None, verbose=False, platform="auto"):
"""
Download IDAES solvers and libraries and put them in the right location. Need
to supply either local or url argument.
Args:
url (str): a url to download binary files to install files
Returns:
None
"""
if verbose:
_log.setLevel(idaeslog.DEBUG)
idaes._create_bin_dir()
solvers_tar = os.path.join(idaes.bin_directory, "idaes-solvers.tar.gz")
libs_tar = os.path.join(idaes.bin_directory, "idaes-lib.tar.gz")
fd = FileDownloader()
arch = fd.get_sysinfo()
if arch[1] != 64:
_log.error("IDAES Extensions currently only supports 64bit Python.")
raise RuntimeError(
"IDAES Extensions currently only supports 64bit Python.")
if platform == "auto":
platform = arch[0]
if platform == "linux":
linux_dist = fd.get_os_version().replace(".", "")
if linux_dist in idaes.config.known_binary_platform:
platform = linux_dist
if platform not in idaes.config.known_binary_platform:
raise Exception("Unknow platform {}".format(platform))
if platform in idaes.config.binary_platform_map:
platform = idaes.config.binary_platform_map[platform]
checksum = {}
if release is not None:
# if a release is specified it takes precedence over a url
url = "/".join([_release_base_url, release])
# if we're downloading an official release check checksum
check_to = os.path.join(idaes.bin_directory,
f"sha256sum_{release}.txt")
check_from = f"https://raw.githubusercontent.com/IDAES/idaes-ext/main/releases/sha256sum_{release}.txt"
_log.debug("Getting release {}\n checksum file {}".format(
release, check_from))
fd.set_destination_filename(check_to)
fd.get_binary_file(check_from)
with open(check_to, 'r') as f:
for i in range(1000):
line = f.readline(1000)
if line == "":
break
line = line.split(sep=" ")
checksum[line[1].strip()] = line[0].strip()
if url is not None:
if not url.endswith("/"):
c = "/"
else:
c = ""
solvers_from = c.join(
[url, "idaes-solvers-{}-{}.tar.gz".format(platform, arch[1])])
libs_from = c.join(
[url, "idaes-lib-{}-{}.tar.gz".format(platform, arch[1])])
_log.debug("URLs \n {}\n {}\n {}".format(url, solvers_from,
libs_from))
_log.debug("Destinations \n {}\n {}".format(solvers_tar, libs_tar))
if platform == 'darwin':
raise Exception('Mac OSX currently unsupported')
fd.set_destination_filename(solvers_tar)
fd.get_binary_file(solvers_from)
fd.set_destination_filename(libs_tar)
fd.get_binary_file(libs_from)
else:
raise Exception("Must provide a location to download binaries")
if checksum:
# if you are downloading a release and not a specific URL verify checksum
fn_s = "idaes-solvers-{}-{}.tar.gz".format(platform, arch[1])
fn_l = "idaes-lib-{}-{}.tar.gz".format(platform, arch[1])
hash_s = _hash(solvers_tar)
hash_l = _hash(libs_tar)
_log.debug("Solvers Hash {}".format(hash_s))
_log.debug("Libs Hash {}".format(hash_l))
if checksum.get(fn_s, "") != hash_s:
raise Exception("Solver files hash does not match expected")
if checksum.get(fn_l, "") != hash_l:
raise Exception("Library files hash does not match expected")
_log.debug("Extracting files in {}".format(idaes.bin_directory))
with tarfile.open(solvers_tar, 'r') as f:
f.extractall(idaes.bin_directory)
_log.debug("Extracting files in {}".format(idaes.bin_directory))
with tarfile.open(libs_tar, 'r') as f:
f.extractall(idaes.bin_directory)
def test_get_os_version(self):
f = FileDownloader()
_os, _ver = f.get_os_version(normalize=False)
_norm = f.get_os_version(normalize=True)
#print(_os,_ver,_norm)
_sys = f.get_sysinfo()[0]
if _sys == 'linux':
dist, dist_ver = re.match('^([^0-9]+)(.*)', _norm).groups()
self.assertNotIn('.', dist_ver)
self.assertGreater(int(dist_ver), 0)
if dist == 'ubuntu':
self.assertEqual(dist_ver, ''.join(_ver.split('.')[:2]))
else:
self.assertEqual(dist_ver, _ver.split('.')[0])
if distro_available:
d, v = f._get_distver_from_distro()
#print(d,v)
self.assertEqual(_os, d)
self.assertEqual(_ver, v)
self.assertTrue(v.replace('.', '').startswith(dist_ver))
if os.path.exists('/etc/redhat-release'):
d, v = f._get_distver_from_redhat_release()
#print(d,v)
self.assertEqual(_os, d)
self.assertEqual(_ver, v)
self.assertTrue(v.replace('.', '').startswith(dist_ver))
if subprocess.run(['lsb_release'],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL).returncode == 0:
d, v = f._get_distver_from_lsb_release()
#print(d,v)
self.assertEqual(_os, d)
self.assertEqual(_ver, v)
self.assertTrue(v.replace('.', '').startswith(dist_ver))
if os.path.exists('/etc/os-release'):
d, v = f._get_distver_from_os_release()
#print(d,v)
self.assertEqual(_os, d)
# Note that (at least on centos), os_release is an
# imprecise version string
self.assertTrue(_ver.startswith(v))
self.assertTrue(v.replace('.', '').startswith(dist_ver))
elif _sys == 'darwin':
dist, dist_ver = re.match('^([^0-9]+)(.*)', _norm).groups()
self.assertEqual(_os, 'macos')
self.assertEqual(dist, 'macos')
self.assertNotIn('.', dist_ver)
self.assertGreater(int(dist_ver), 0)
self.assertEqual(_norm, _os + ''.join(_ver.split('.')[:2]))
elif _sys == 'windows':
self.assertEqual(_os, 'win')
self.assertEqual(_norm, _os + ''.join(_ver.split('.')[:2]))
else:
self.assertEqual(ans, '')
self.assertEqual((_os, _ver), FileDownloader._os_version)
# Exercise the fetch from CACHE
try:
FileDownloader._os_version, tmp \
= ("test", '2'), FileDownloader._os_version
self.assertEqual(f.get_os_version(False), ("test", "2"))
self.assertEqual(f.get_os_version(), "test2")
finally:
FileDownloader._os_version = tmp
def skip(self):
return FileDownloader.get_sysinfo()[0] == 'windows'
