def test_get_test_binary_file(self): tmpdir = tempfile.mkdtemp() try: f = FileDownloader() # Mock retrieve_url so network connections are not necessary if six.PY3: f.retrieve_url = lambda url: bytes("\n", encoding='utf-8') else: f.retrieve_url = lambda url: bytes("\n") # Binary files will preserve line endings target = os.path.join(tmpdir, 'bin.txt') f.set_destination_filename(target) f.get_binary_file(None) self.assertEqual(os.path.getsize(target), 1) # Mock retrieve_url so network connections are not necessary f.retrieve_url = lambda url: "\n" # Text files will convert line endings to the local platform target = os.path.join(tmpdir, 'txt.txt') f.set_destination_filename(target) f.get_text_file(None) self.assertEqual(os.path.getsize(target), len(os.linesep)) finally: shutil.rmtree(tmpdir)
def download_binaries(url=None, verbose=False): """ Download IDAES solvers and libraries and put them in the right location. Need to supply either local or url argument. Args: url (str): a url to download binary files to install files Returns: None """ if verbose: _log.setLevel(idaeslog.DEBUG) idaes._create_lib_dir() idaes._create_bin_dir() solvers_tar = os.path.join(idaes.bin_directory, "idaes-solvers.tar.gz") libs_tar = os.path.join(idaes.lib_directory, "idaes-lib.tar.gz") fd = FileDownloader() arch = fd.get_sysinfo() if url is not None: if not url.endswith("/"): c = "/" else: c = "" solvers_from = c.join( [url, "idaes-solvers-{}-{}.tar.gz".format(arch[0], arch[1])]) libs_from = c.join( [url, "idaes-lib-{}-{}.tar.gz".format(arch[0], arch[1])]) _log.debug("URLs \n {}\n {}\n {}".format(url, solvers_from, libs_from)) _log.debug("Destinations \n {}\n {}".format(solvers_tar, libs_tar)) if arch[0] == 'darwin': raise Exception('Mac OSX currently unsupported') fd.set_destination_filename(solvers_tar) fd.get_binary_file(solvers_from) fd.set_destination_filename(libs_tar) fd.get_binary_file(libs_from) else: raise Exception("Must provide a location to download binaries") _log.debug("Extracting files in {}".format(idaes.bin_directory)) with tarfile.open(solvers_tar, 'r') as f: f.extractall(idaes.bin_directory) _log.debug("Extracting files in {}".format(idaes.lib_directory)) with tarfile.open(libs_tar, 'r') as f: f.extractall(idaes.lib_directory)
def test_set_destination_filename(self): self.tmpdir = os.path.abspath(tempfile.mkdtemp()) f = FileDownloader() self.assertIsNone(f._fname) f.set_destination_filename('foo') self.assertEqual(f._fname, os.path.join(PYOMO_CONFIG_DIR, 'foo')) # By this point, the CONFIG_DIR is guaranteed to have been created self.assertTrue(os.path.isdir(PYOMO_CONFIG_DIR)) f.target = self.tmpdir f.set_destination_filename('foo') target = os.path.join(self.tmpdir, 'foo') self.assertEqual(f._fname, target) self.assertFalse(os.path.exists(target)) f.target = self.tmpdir f.set_destination_filename(os.path.join('foo','bar')) target = os.path.join(self.tmpdir, 'foo', 'bar') self.assertEqual(f._fname, target) self.assertFalse(os.path.exists(target)) target_dir = os.path.join(self.tmpdir, 'foo',) self.assertTrue(os.path.isdir(target_dir))
def download_binaries(release=None, url=None, verbose=False, platform="auto"): """ Download IDAES solvers and libraries and put them in the right location. Need to supply either local or url argument. Args: url (str): a url to download binary files to install files Returns: None """ if verbose: _log.setLevel(idaeslog.DEBUG) idaes._create_bin_dir() solvers_tar = os.path.join(idaes.bin_directory, "idaes-solvers.tar.gz") libs_tar = os.path.join(idaes.bin_directory, "idaes-lib.tar.gz") fd = FileDownloader() arch = fd.get_sysinfo() if arch[1] != 64: _log.error("IDAES Extensions currently only supports 64bit Python.") raise RuntimeError( "IDAES Extensions currently only supports 64bit Python.") if platform == "auto": platform = arch[0] if platform == "linux": linux_dist = fd.get_os_version().replace(".", "") if linux_dist in idaes.config.known_binary_platform: platform = linux_dist if platform not in idaes.config.known_binary_platform: raise Exception("Unknow platform {}".format(platform)) if platform in idaes.config.binary_platform_map: platform = idaes.config.binary_platform_map[platform] checksum = {} if release is not None: # if a release is specified it takes precedence over a url url = "/".join([_release_base_url, release]) # if we're downloading an official release check checksum check_to = os.path.join(idaes.bin_directory, f"sha256sum_{release}.txt") check_from = f"https://raw.githubusercontent.com/IDAES/idaes-ext/main/releases/sha256sum_{release}.txt" _log.debug("Getting release {}\n checksum file {}".format( release, check_from)) fd.set_destination_filename(check_to) fd.get_binary_file(check_from) with open(check_to, 'r') as f: for i in range(1000): line = f.readline(1000) if line == "": break line = line.split(sep=" ") checksum[line[1].strip()] = line[0].strip() if url is not None: if not url.endswith("/"): c = "/" else: c = "" solvers_from = c.join( [url, "idaes-solvers-{}-{}.tar.gz".format(platform, arch[1])]) libs_from = c.join( [url, "idaes-lib-{}-{}.tar.gz".format(platform, arch[1])]) _log.debug("URLs \n {}\n {}\n {}".format(url, solvers_from, libs_from)) _log.debug("Destinations \n {}\n {}".format(solvers_tar, libs_tar)) if platform == 'darwin': raise Exception('Mac OSX currently unsupported') fd.set_destination_filename(solvers_tar) fd.get_binary_file(solvers_from) fd.set_destination_filename(libs_tar) fd.get_binary_file(libs_from) else: raise Exception("Must provide a location to download binaries") if checksum: # if you are downloading a release and not a specific URL verify checksum fn_s = "idaes-solvers-{}-{}.tar.gz".format(platform, arch[1]) fn_l = "idaes-lib-{}-{}.tar.gz".format(platform, arch[1]) hash_s = _hash(solvers_tar) hash_l = _hash(libs_tar) _log.debug("Solvers Hash {}".format(hash_s)) _log.debug("Libs Hash {}".format(hash_l)) if checksum.get(fn_s, "") != hash_s: raise Exception("Solver files hash does not match expected") if checksum.get(fn_l, "") != hash_l: raise Exception("Library files hash does not match expected") _log.debug("Extracting files in {}".format(idaes.bin_directory)) with tarfile.open(solvers_tar, 'r') as f: f.extractall(idaes.bin_directory) _log.debug("Extracting files in {}".format(idaes.bin_directory)) with tarfile.open(libs_tar, 'r') as f: f.extractall(idaes.bin_directory)