def test_get_test_binary_file(self):
tmpdir = tempfile.mkdtemp()
try:
f = FileDownloader()
# Mock retrieve_url so network connections are not necessary
if six.PY3:
f.retrieve_url = lambda url: bytes("\n", encoding='utf-8')
else:
f.retrieve_url = lambda url: bytes("\n")
# Binary files will preserve line endings
target = os.path.join(tmpdir, 'bin.txt')
f.set_destination_filename(target)
f.get_binary_file(None)
self.assertEqual(os.path.getsize(target), 1)
# Mock retrieve_url so network connections are not necessary
f.retrieve_url = lambda url: "\n"
# Text files will convert line endings to the local platform
target = os.path.join(tmpdir, 'txt.txt')
f.set_destination_filename(target)
f.get_text_file(None)
self.assertEqual(os.path.getsize(target), len(os.linesep))
finally:
shutil.rmtree(tmpdir)
def test_get_files_requires_set_destination(self):
f = FileDownloader()
with self.assertRaisesRegexp(
DeveloperError, 'target file name has not been initialized'):
f.get_binary_file('bogus')
with self.assertRaisesRegexp(
DeveloperError, 'target file name has not been initialized'):
f.get_binary_file_from_zip_archive('bogus', 'bogus')
with self.assertRaisesRegexp(
DeveloperError, 'target file name has not been initialized'):
f.get_gzipped_binary_file('bogus')
def test_get_files_requires_set_destination(self):
f = FileDownloader()
with self.assertRaisesRegexp(
DeveloperError, 'target file name has not been initialized'):
f.get_binary_file('bogus')
with self.assertRaisesRegexp(
DeveloperError, 'target file name has not been initialized'):
f.get_binary_file_from_zip_archive('bogus', 'bogus')
with self.assertRaisesRegexp(
DeveloperError, 'target file name has not been initialized'):
f.get_gzipped_binary_file('bogus')
def download_binaries(url=None, verbose=False):
"""
Download IDAES solvers and libraries and put them in the right location. Need
to supply either local or url argument.
Args:
url (str): a url to download binary files to install files
Returns:
None
"""
if verbose:
_log.setLevel(idaeslog.DEBUG)
idaes._create_lib_dir()
idaes._create_bin_dir()
solvers_tar = os.path.join(idaes.bin_directory, "idaes-solvers.tar.gz")
libs_tar = os.path.join(idaes.lib_directory, "idaes-lib.tar.gz")
fd = FileDownloader()
arch = fd.get_sysinfo()
if url is not None:
if not url.endswith("/"):
c = "/"
else:
c = ""
solvers_from = c.join(
[url, "idaes-solvers-{}-{}.tar.gz".format(arch[0], arch[1])])
libs_from = c.join(
[url, "idaes-lib-{}-{}.tar.gz".format(arch[0], arch[1])])
_log.debug("URLs \n {}\n {}\n {}".format(url, solvers_from,
libs_from))
_log.debug("Destinations \n {}\n {}".format(solvers_tar, libs_tar))
if arch[0] == 'darwin':
raise Exception('Mac OSX currently unsupported')
fd.set_destination_filename(solvers_tar)
fd.get_binary_file(solvers_from)
fd.set_destination_filename(libs_tar)
fd.get_binary_file(libs_from)
else:
raise Exception("Must provide a location to download binaries")
_log.debug("Extracting files in {}".format(idaes.bin_directory))
with tarfile.open(solvers_tar, 'r') as f:
f.extractall(idaes.bin_directory)
_log.debug("Extracting files in {}".format(idaes.lib_directory))
with tarfile.open(libs_tar, 'r') as f:
f.extractall(idaes.lib_directory)
def download_binaries(release=None, url=None, verbose=False, platform="auto"):
"""
Download IDAES solvers and libraries and put them in the right location. Need
to supply either local or url argument.
Args:
url (str): a url to download binary files to install files
Returns:
None
"""
if verbose:
_log.setLevel(idaeslog.DEBUG)
idaes._create_bin_dir()
solvers_tar = os.path.join(idaes.bin_directory, "idaes-solvers.tar.gz")
libs_tar = os.path.join(idaes.bin_directory, "idaes-lib.tar.gz")
fd = FileDownloader()
arch = fd.get_sysinfo()
if arch[1] != 64:
_log.error("IDAES Extensions currently only supports 64bit Python.")
raise RuntimeError(
"IDAES Extensions currently only supports 64bit Python.")
if platform == "auto":
platform = arch[0]
if platform == "linux":
linux_dist = fd.get_os_version().replace(".", "")
if linux_dist in idaes.config.known_binary_platform:
platform = linux_dist
if platform not in idaes.config.known_binary_platform:
raise Exception("Unknow platform {}".format(platform))
if platform in idaes.config.binary_platform_map:
platform = idaes.config.binary_platform_map[platform]
checksum = {}
if release is not None:
# if a release is specified it takes precedence over a url
url = "/".join([_release_base_url, release])
# if we're downloading an official release check checksum
check_to = os.path.join(idaes.bin_directory,
f"sha256sum_{release}.txt")
check_from = f"https://raw.githubusercontent.com/IDAES/idaes-ext/main/releases/sha256sum_{release}.txt"
_log.debug("Getting release {}\n checksum file {}".format(
release, check_from))
fd.set_destination_filename(check_to)
fd.get_binary_file(check_from)
with open(check_to, 'r') as f:
for i in range(1000):
line = f.readline(1000)
if line == "":
break
line = line.split(sep=" ")
checksum[line[1].strip()] = line[0].strip()
if url is not None:
if not url.endswith("/"):
c = "/"
else:
c = ""
solvers_from = c.join(
[url, "idaes-solvers-{}-{}.tar.gz".format(platform, arch[1])])
libs_from = c.join(
[url, "idaes-lib-{}-{}.tar.gz".format(platform, arch[1])])
_log.debug("URLs \n {}\n {}\n {}".format(url, solvers_from,
libs_from))
_log.debug("Destinations \n {}\n {}".format(solvers_tar, libs_tar))
if platform == 'darwin':
raise Exception('Mac OSX currently unsupported')
fd.set_destination_filename(solvers_tar)
fd.get_binary_file(solvers_from)
fd.set_destination_filename(libs_tar)
fd.get_binary_file(libs_from)
else:
raise Exception("Must provide a location to download binaries")
if checksum:
# if you are downloading a release and not a specific URL verify checksum
fn_s = "idaes-solvers-{}-{}.tar.gz".format(platform, arch[1])
fn_l = "idaes-lib-{}-{}.tar.gz".format(platform, arch[1])
hash_s = _hash(solvers_tar)
hash_l = _hash(libs_tar)
_log.debug("Solvers Hash {}".format(hash_s))
_log.debug("Libs Hash {}".format(hash_l))
if checksum.get(fn_s, "") != hash_s:
raise Exception("Solver files hash does not match expected")
if checksum.get(fn_l, "") != hash_l:
raise Exception("Library files hash does not match expected")
_log.debug("Extracting files in {}".format(idaes.bin_directory))
with tarfile.open(solvers_tar, 'r') as f:
f.extractall(idaes.bin_directory)
_log.debug("Extracting files in {}".format(idaes.bin_directory))
with tarfile.open(libs_tar, 'r') as f:
f.extractall(idaes.bin_directory)
