def _get_file_downloader(insecure, cacert): fd = FileDownloader(insecure=insecure, cacert=cacert) arch = fd.get_sysinfo() if arch[1] != 64: _log.error("IDAES Extensions only supports 64bit Python.") raise RuntimeError("IDAES Extensions only supports 64bit Python.") return fd, arch
def test_set_destination_filename(self): self.tmpdir = os.path.abspath(tempfile.mkdtemp()) f = FileDownloader() self.assertIsNone(f._fname) f.set_destination_filename('foo') self.assertEqual(f._fname, os.path.join(PYOMO_CONFIG_DIR, 'foo')) # By this point, the CONFIG_DIR is guaranteed to have been created self.assertTrue(os.path.isdir(PYOMO_CONFIG_DIR)) f.target = self.tmpdir f.set_destination_filename('foo') target = os.path.join(self.tmpdir, 'foo') self.assertEqual(f._fname, target) self.assertFalse(os.path.exists(target)) f.target = self.tmpdir f.set_destination_filename(os.path.join('foo', 'bar')) target = os.path.join(self.tmpdir, 'foo', 'bar') self.assertEqual(f._fname, target) self.assertFalse(os.path.exists(target)) target_dir = os.path.join( self.tmpdir, 'foo', ) self.assertTrue(os.path.isdir(target_dir))
def test_get_url(self): f = FileDownloader() urlmap = {'bogus_sys': 'bogus'} with self.assertRaisesRegexp( RuntimeError, "cannot infer the correct url for platform '.*'"): f.get_url(urlmap) urlmap[f.get_sysinfo()[0]] = 'correct' self.assertEqual(f.get_url(urlmap), 'correct')
def test_get_sysinfo(self): f = FileDownloader() ans = f.get_sysinfo() self.assertIs(type(ans), tuple) self.assertEqual(len(ans), 2) self.assertTrue(len(ans[0]) > 0) self.assertTrue(platform.system().lower().startswith(ans[0])) self.assertFalse(any(c in ans[0] for c in '.-_')) self.assertIn(ans[1], (32,64))
def test_get_sysinfo(self): f = FileDownloader() ans = f.get_sysinfo() self.assertIs(type(ans), tuple) self.assertEqual(len(ans), 2) self.assertTrue(len(ans[0]) > 0) self.assertTrue(platform.system().lower().startswith(ans[0])) self.assertFalse(any(c in ans[0] for c in '.-_')) self.assertIn(ans[1], (32, 64))
class GroupDownloader(object): def __init__(self): self.downloader = FileDownloader() def create_parser(self, parser): return self.downloader.create_parser(parser) def call(self, args, unparsed): logger = logging.getLogger('pyomo.common') logger.setLevel(logging.INFO) results = [] result_fmt = "[%s] %s" returncode = 0 self.downloader.cacert = args.cacert self.downloader.insecure = args.insecure for target in DownloadFactory: try: DownloadFactory(target, downloader=self.downloader) result = ' OK ' except SystemExit: result = 'FAIL' returncode = 1 except: result = 'FAIL' returncode = 1 results.append(result_fmt % (result, target)) logger.info("Finished downloading Pyomo extensions.") logger.info("The following extensions were downloaded:\n " + "\n ".join(results)) return returncode
class GroupDownloader(object): def __init__(self): self.downloader = FileDownloader() def create_parser(self, parser): return self.downloader.create_parser(parser) def call(self, args, unparsed): logger = logging.getLogger('pyomo.common') logger.setLevel(logging.INFO) results = [] result_fmt = "[%s] %s" returncode = 0 self.downloader.cacert = args.cacert self.downloader.insecure = args.insecure for target in DownloadFactory: try: DownloadFactory(target, downloader=self.downloader) result = ' OK ' except SystemExit: result = 'FAIL' returncode = 1 except: result = 'FAIL' returncode = 1 results.append(result_fmt % (result, target)) logger.info("Finished downloading Pyomo extensions.") logger.info( "The following extensions were downloaded:\n " + "\n ".join(results)) return returncode
def test_init(self): f = FileDownloader() self.assertFalse(f.insecure) self.assertIsNone(f.cacert) self.assertIsNone(f._fname) f = FileDownloader(True) self.assertTrue(f.insecure) self.assertIsNone(f.cacert) self.assertIsNone(f._fname) f = FileDownloader(True, this_file()) self.assertTrue(f.insecure) self.assertEqual(f.cacert, this_file()) self.assertIsNone(f._fname) with self.assertRaisesRegexp( RuntimeError, "cacert='nonexistant_file_name' does not " "refer to a valid file."): FileDownloader(True, 'nonexistant_file_name')
def test_get_files_requires_set_destination(self): f = FileDownloader() with self.assertRaisesRegexp( DeveloperError, 'target file name has not been initialized'): f.get_binary_file('bogus') with self.assertRaisesRegexp( DeveloperError, 'target file name has not been initialized'): f.get_binary_file_from_zip_archive('bogus', 'bogus') with self.assertRaisesRegexp( DeveloperError, 'target file name has not been initialized'): f.get_gzipped_binary_file('bogus')
class GroupDownloader(object): def __init__(self): self.downloader = FileDownloader() def create_parser(self, parser): return self.downloader.create_parser(parser) def call(self, args, unparsed): logger = logging.getLogger('pyomo.common') original_level = logger.level logger.setLevel(logging.INFO) try: return self._call_impl(args, unparsed, logger) finally: logger.setLevel(original_level) def _call_impl(self, args, unparsed, logger): results = [] result_fmt = "[%s] %s" returncode = 0 self.downloader.cacert = args.cacert self.downloader.insecure = args.insecure for target in DownloadFactory: try: ext = DownloadFactory(target, downloader=self.downloader) if hasattr(ext, 'skip') and ext.skip(): result = 'SKIP' elif hasattr(ext, '__call__'): ext() result = ' OK ' else: # Extension was a simple function and already ran result = ' OK ' except SystemExit: _info = sys.exc_info() _cls = str(_info[0].__name__ if _info[0] is not None else "NoneType") + ": " logger.error(_cls + str(_info[1])) result = 'FAIL' returncode |= 2 except: _info = sys.exc_info() _cls = str(_info[0].__name__ if _info[0] is not None else "NoneType") + ": " logger.error(_cls + str(_info[1])) result = 'FAIL' returncode |= 1 results.append(result_fmt % (result, target)) logger.info("Finished downloading Pyomo extensions.") logger.info( "The following extensions were downloaded:\n " + "\n ".join(results)) return returncode
class GroupDownloader(object): def __init__(self): self.downloader = FileDownloader() def create_parser(self, parser): return self.downloader.create_parser(parser) def call(self, args, unparsed): logging.getLogger('pyomo.common').setLevel(logging.INFO) self.downloader.cacert = args.cacert self.downloader.insecure = args.insecure for target in DownloadFactory: DownloadFactory(target, downloader=self.downloader)
def test_get_platform_url(self): f = FileDownloader() urlmap = {'bogus_sys': 'bogus'} with self.assertRaisesRegexp( RuntimeError, "cannot infer the correct url for platform '.*'"): f.get_platform_url(urlmap) urlmap[f.get_sysinfo()[0]] = 'correct' self.assertEqual(f.get_platform_url(urlmap), 'correct')
def test_set_destination_filename(self): self.tmpdir = os.path.abspath(tempfile.mkdtemp()) f = FileDownloader() self.assertIsNone(f._fname) f.set_destination_filename('foo') self.assertEqual(f._fname, os.path.join(PYOMO_CONFIG_DIR, 'foo')) # By this point, the CONFIG_DIR is guaranteed to have been created self.assertTrue(os.path.isdir(PYOMO_CONFIG_DIR)) f.target = self.tmpdir f.set_destination_filename('foo') target = os.path.join(self.tmpdir, 'foo') self.assertEqual(f._fname, target) self.assertFalse(os.path.exists(target)) f.target = self.tmpdir f.set_destination_filename(os.path.join('foo','bar')) target = os.path.join(self.tmpdir, 'foo', 'bar') self.assertEqual(f._fname, target) self.assertFalse(os.path.exists(target)) target_dir = os.path.join(self.tmpdir, 'foo',) self.assertTrue(os.path.isdir(target_dir))
def test_find_library_system(self): # Find a system library (before we muck with the PATH) _args = {'cwd': False, 'include_PATH': False, 'pathlist': []} if FileDownloader.get_sysinfo()[0] == 'windows': a = find_library('ntdll', **_args) b = find_library('ntdll.dll', **_args) c = find_library('foo\\bar\\ntdll.dll', **_args) else: a = find_library('c', **_args) b = find_library('libc.so', **_args) c = find_library('foo/bar/libc.so', **_args) self.assertIsNotNone(a) self.assertIsNotNone(b) self.assertIsNotNone(c) self.assertEqual(a, b) # find_library could have found libc.so.6 self.assertTrue(c.startswith(a)) # Verify that the library is loadable (they are all the same # file, so only check one) _lib = ctypes.cdll.LoadLibrary(a) self.assertIsNotNone(_lib)
def download_binaries(url=None, verbose=False): """ Download IDAES solvers and libraries and put them in the right location. Need to supply either local or url argument. Args: url (str): a url to download binary files to install files Returns: None """ if verbose: _log.setLevel(idaeslog.DEBUG) idaes._create_lib_dir() idaes._create_bin_dir() solvers_tar = os.path.join(idaes.bin_directory, "idaes-solvers.tar.gz") libs_tar = os.path.join(idaes.lib_directory, "idaes-lib.tar.gz") fd = FileDownloader() arch = fd.get_sysinfo() if url is not None: if not url.endswith("/"): c = "/" else: c = "" solvers_from = c.join( [url, "idaes-solvers-{}-{}.tar.gz".format(arch[0], arch[1])]) libs_from = c.join( [url, "idaes-lib-{}-{}.tar.gz".format(arch[0], arch[1])]) _log.debug("URLs \n {}\n {}\n {}".format(url, solvers_from, libs_from)) _log.debug("Destinations \n {}\n {}".format(solvers_tar, libs_tar)) if arch[0] == 'darwin': raise Exception('Mac OSX currently unsupported') fd.set_destination_filename(solvers_tar) fd.get_binary_file(solvers_from) fd.set_destination_filename(libs_tar) fd.get_binary_file(libs_from) else: raise Exception("Must provide a location to download binaries") _log.debug("Extracting files in {}".format(idaes.bin_directory)) with tarfile.open(solvers_tar, 'r') as f: f.extractall(idaes.bin_directory) _log.debug("Extracting files in {}".format(idaes.lib_directory)) with tarfile.open(libs_tar, 'r') as f: f.extractall(idaes.lib_directory)
def test_find_library(self): self.tmpdir = os.path.abspath(tempfile.mkdtemp()) os.chdir(self.tmpdir) # Find a system library (before we muck with the PATH) _args = {'cwd': False, 'include_PATH': False, 'pathlist': []} if FileDownloader.get_sysinfo()[0] == 'windows': a = find_library('ntdll', **_args) b = find_library('ntdll.dll', **_args) c = find_library('foo\\bar\\ntdll.dll', **_args) else: a = find_library('c', **_args) b = find_library('libc.so', **_args) c = find_library('foo/bar/libc.so', **_args) self.assertIsNotNone(a) self.assertIsNotNone(b) self.assertIsNotNone(c) self.assertEqual(a, b) # find_library could have found libc.so.6 self.assertTrue(c.startswith(a)) # Verify that the library is loadable (they are all the same # file, so only check one) _lib = ctypes.cdll.LoadLibrary(a) self.assertIsNotNone(_lib) envvar.PYOMO_CONFIG_DIR = self.tmpdir config_libdir = os.path.join(self.tmpdir, 'lib') os.mkdir(config_libdir) config_bindir = os.path.join(self.tmpdir, 'bin') os.mkdir(config_bindir) ldlibdir_name = 'in_ld_lib' ldlibdir = os.path.join(self.tmpdir, ldlibdir_name) os.mkdir(ldlibdir) os.environ['LD_LIBRARY_PATH'] = os.pathsep + ldlibdir + os.pathsep pathdir_name = 'in_path' pathdir = os.path.join(self.tmpdir, pathdir_name) os.mkdir(pathdir) os.environ['PATH'] = os.pathsep + pathdir + os.pathsep libExt = _libExt[_system()][0] f_in_cwd_ldlib_path = 'f_in_cwd_ldlib_path' open(os.path.join(self.tmpdir, f_in_cwd_ldlib_path), 'w').close() open(os.path.join(ldlibdir, f_in_cwd_ldlib_path), 'w').close() open(os.path.join(pathdir, f_in_cwd_ldlib_path), 'w').close() f_in_ldlib_extension = 'f_in_ldlib_extension' open(os.path.join(ldlibdir, f_in_ldlib_extension + libExt), 'w').close() f_in_path = 'f_in_path' open(os.path.join(pathdir, f_in_path), 'w').close() f_in_configlib = 'f_in_configlib' open(os.path.join(config_libdir, f_in_configlib), 'w').close() f_in_configbin = 'f_in_configbin' open(os.path.join(config_bindir, f_in_ldlib_extension), 'w').close() open(os.path.join(config_bindir, f_in_configbin), 'w').close() self._check_file(find_library(f_in_cwd_ldlib_path), os.path.join(self.tmpdir, f_in_cwd_ldlib_path)) self._check_file(os.path.join(ldlibdir, f_in_cwd_ldlib_path), find_library(f_in_cwd_ldlib_path, cwd=False)) self._check_file( os.path.join(ldlibdir, f_in_ldlib_extension) + libExt, find_library(f_in_ldlib_extension)) self._check_file(os.path.join(pathdir, f_in_path), find_library(f_in_path)) if _system() == 'windows': self._check_file(os.path.join(pathdir, f_in_path), find_library(f_in_path, include_PATH=False)) else: # Note that on Windows, ctypes.util.find_library *always* # searches the PATH self.assertIsNone(find_library(f_in_path, include_PATH=False)) self._check_file( os.path.join(pathdir, f_in_path), find_library(f_in_path, pathlist=os.pathsep + pathdir + os.pathsep)) # test an explicit pathlist overrides LD_LIBRARY_PATH self._check_file( os.path.join(pathdir, f_in_cwd_ldlib_path), find_library(f_in_cwd_ldlib_path, cwd=False, pathlist=[pathdir])) # test that the PYOMO_CONFIG_DIR 'lib' dir is included self._check_file(os.path.join(config_libdir, f_in_configlib), find_library(f_in_configlib)) # and the Bin dir self._check_file(os.path.join(config_bindir, f_in_configbin), find_library(f_in_configbin)) # ... but only if include_PATH is true self.assertIsNone(find_library(f_in_configbin, include_PATH=False)) # And none of them if the pathlist is specified self.assertIsNone(find_library(f_in_configlib, pathlist=pathdir)) self.assertIsNone(find_library(f_in_configbin, pathlist=pathdir))
def download_binaries(release=None, url=None, verbose=False, platform="auto"): """ Download IDAES solvers and libraries and put them in the right location. Need to supply either local or url argument. Args: url (str): a url to download binary files to install files Returns: None """ if verbose: _log.setLevel(idaeslog.DEBUG) idaes._create_bin_dir() solvers_tar = os.path.join(idaes.bin_directory, "idaes-solvers.tar.gz") libs_tar = os.path.join(idaes.bin_directory, "idaes-lib.tar.gz") fd = FileDownloader() arch = fd.get_sysinfo() if arch[1] != 64: _log.error("IDAES Extensions currently only supports 64bit Python.") raise RuntimeError( "IDAES Extensions currently only supports 64bit Python.") if platform == "auto": platform = arch[0] if platform == "linux": linux_dist = fd.get_os_version().replace(".", "") if linux_dist in idaes.config.known_binary_platform: platform = linux_dist if platform not in idaes.config.known_binary_platform: raise Exception("Unknow platform {}".format(platform)) if platform in idaes.config.binary_platform_map: platform = idaes.config.binary_platform_map[platform] checksum = {} if release is not None: # if a release is specified it takes precedence over a url url = "/".join([_release_base_url, release]) # if we're downloading an official release check checksum check_to = os.path.join(idaes.bin_directory, f"sha256sum_{release}.txt") check_from = f"https://raw.githubusercontent.com/IDAES/idaes-ext/main/releases/sha256sum_{release}.txt" _log.debug("Getting release {}\n checksum file {}".format( release, check_from)) fd.set_destination_filename(check_to) fd.get_binary_file(check_from) with open(check_to, 'r') as f: for i in range(1000): line = f.readline(1000) if line == "": break line = line.split(sep=" ") checksum[line[1].strip()] = line[0].strip() if url is not None: if not url.endswith("/"): c = "/" else: c = "" solvers_from = c.join( [url, "idaes-solvers-{}-{}.tar.gz".format(platform, arch[1])]) libs_from = c.join( [url, "idaes-lib-{}-{}.tar.gz".format(platform, arch[1])]) _log.debug("URLs \n {}\n {}\n {}".format(url, solvers_from, libs_from)) _log.debug("Destinations \n {}\n {}".format(solvers_tar, libs_tar)) if platform == 'darwin': raise Exception('Mac OSX currently unsupported') fd.set_destination_filename(solvers_tar) fd.get_binary_file(solvers_from) fd.set_destination_filename(libs_tar) fd.get_binary_file(libs_from) else: raise Exception("Must provide a location to download binaries") if checksum: # if you are downloading a release and not a specific URL verify checksum fn_s = "idaes-solvers-{}-{}.tar.gz".format(platform, arch[1]) fn_l = "idaes-lib-{}-{}.tar.gz".format(platform, arch[1]) hash_s = _hash(solvers_tar) hash_l = _hash(libs_tar) _log.debug("Solvers Hash {}".format(hash_s)) _log.debug("Libs Hash {}".format(hash_l)) if checksum.get(fn_s, "") != hash_s: raise Exception("Solver files hash does not match expected") if checksum.get(fn_l, "") != hash_l: raise Exception("Library files hash does not match expected") _log.debug("Extracting files in {}".format(idaes.bin_directory)) with tarfile.open(solvers_tar, 'r') as f: f.extractall(idaes.bin_directory) _log.debug("Extracting files in {}".format(idaes.bin_directory)) with tarfile.open(libs_tar, 'r') as f: f.extractall(idaes.bin_directory)
def skip(self): return FileDownloader.get_sysinfo()[0] == 'windows'
def main(argv): downloader = FileDownloader() downloader.parse_args(argv) get_gsl(downloader)
def test_parse(self): f = FileDownloader() f.parse_args([]) self.assertFalse(f.insecure) self.assertIsNone(f.cacert) self.assertIsNone(f.target) f = FileDownloader() f.parse_args(['--insecure']) self.assertTrue(f.insecure) self.assertIsNone(f.cacert) self.assertIsNone(f.target) f = FileDownloader() f.parse_args(['--insecure', '--cacert', this_file()]) self.assertTrue(f.insecure) self.assertEqual(f.cacert, this_file()) self.assertIsNone(f.target) f = FileDownloader() f.parse_args(['--insecure', 'bar', '--cacert', this_file()]) self.assertTrue(f.insecure) self.assertEqual(f.cacert, this_file()) self.assertEqual(f.target, 'bar') f = FileDownloader() with capture_output() as io: with self.assertRaises(SystemExit): f.parse_args(['--cacert']) self.assertIn('argument --cacert: expected one argument', io.getvalue()) f = FileDownloader() with capture_output() as io: with self.assertRaises(SystemExit): f.parse_args(['--cacert', '--insecure']) self.assertIn('argument --cacert: expected one argument', io.getvalue()) f = FileDownloader() with self.assertRaisesRegexp( RuntimeError, "--cacert='nonexistant_file_name' does " "not refer to a valid file"): f.parse_args(['--cacert', 'nonexistant_file_name']) f = FileDownloader() with capture_output() as io: with self.assertRaises(SystemExit): f.parse_args(['--foo']) self.assertIn('error: unrecognized arguments: --foo', io.getvalue())
def __init__(self): self.downloader = FileDownloader()
def test_get_test_binary_file(self): tmpdir = tempfile.mkdtemp() try: f = FileDownloader() # Mock retrieve_url so network connections are not necessary if six.PY3: f.retrieve_url = lambda url: bytes("\n", encoding='utf-8') else: f.retrieve_url = lambda url: str("\n") # Binary files will preserve line endings target = os.path.join(tmpdir, 'bin.txt') f.set_destination_filename(target) f.get_binary_file(None) self.assertEqual(os.path.getsize(target), 1) # Text files will convert line endings to the local platform target = os.path.join(tmpdir, 'txt.txt') f.set_destination_filename(target) f.get_text_file(None) self.assertEqual(os.path.getsize(target), len(os.linesep)) finally: shutil.rmtree(tmpdir)
def test_get_os_version(self): f = FileDownloader() _os, _ver = f.get_os_version(normalize=False) _norm = f.get_os_version(normalize=True) #print(_os,_ver,_norm) _sys = f.get_sysinfo()[0] if _sys == 'linux': dist, dist_ver = re.match('^([^0-9]+)(.*)', _norm).groups() self.assertNotIn('.', dist_ver) self.assertGreater(int(dist_ver), 0) if dist == 'ubuntu': self.assertEqual(dist_ver, ''.join(_ver.split('.')[:2])) else: self.assertEqual(dist_ver, _ver.split('.')[0]) if distro_available: d, v = f._get_distver_from_distro() #print(d,v) self.assertEqual(_os, d) self.assertEqual(_ver, v) self.assertTrue(v.replace('.', '').startswith(dist_ver)) if os.path.exists('/etc/redhat-release'): d, v = f._get_distver_from_redhat_release() #print(d,v) self.assertEqual(_os, d) self.assertEqual(_ver, v) self.assertTrue(v.replace('.', '').startswith(dist_ver)) if subprocess.run(['lsb_release'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0: d, v = f._get_distver_from_lsb_release() #print(d,v) self.assertEqual(_os, d) self.assertEqual(_ver, v) self.assertTrue(v.replace('.', '').startswith(dist_ver)) if os.path.exists('/etc/os-release'): d, v = f._get_distver_from_os_release() #print(d,v) self.assertEqual(_os, d) # Note that (at least on centos), os_release is an # imprecise version string self.assertTrue(_ver.startswith(v)) self.assertTrue(v.replace('.', '').startswith(dist_ver)) elif _sys == 'darwin': dist, dist_ver = re.match('^([^0-9]+)(.*)', _norm).groups() self.assertEqual(_os, 'macos') self.assertEqual(dist, 'macos') self.assertNotIn('.', dist_ver) self.assertGreater(int(dist_ver), 0) self.assertEqual(_norm, _os + ''.join(_ver.split('.')[:2])) elif _sys == 'windows': self.assertEqual(_os, 'win') self.assertEqual(_norm, _os + ''.join(_ver.split('.')[:2])) else: self.assertEqual(ans, '') self.assertEqual((_os, _ver), FileDownloader._os_version) # Exercise the fetch from CACHE try: FileDownloader._os_version, tmp \ = ("test", '2'), FileDownloader._os_version self.assertEqual(f.get_os_version(False), ("test", "2")) self.assertEqual(f.get_os_version(), "test2") finally: FileDownloader._os_version = tmp