def installer(qt_archive, base_dir, command): name = qt_archive.name url = qt_archive.url archive = qt_archive.archive start_time = time.perf_counter() logger = getLogger('aqt') logger.info("Downloading {}...".format(name)) logger.debug("Download URL: {}".format(url)) session = requests.Session() retry = Retry(connect=5, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) try: r = session.get(url, allow_redirects=False, stream=True) if r.status_code == 302: newurl = altlink(r.url, r.headers['Location'], logger=logger) logger.info('Redirected URL: {}'.format(newurl)) r = session.get(newurl, stream=True) except requests.exceptions.ConnectionError as e: logger.error("Connection error: %s" % e.args) raise e else: try: with open(archive, 'wb') as fd: for chunk in r.iter_content(chunk_size=8196): fd.write(chunk) fd.flush() if command is None: with py7zr.SevenZipFile(archive, 'r') as szf: szf.extractall(path=base_dir) except Exception as e: exc = sys.exc_info() logger.error("Download error: %s" % exc[1]) raise e else: if command is not None: if base_dir is not None: command_args = [ command, 'x', '-aoa', '-bd', '-y', '-o{}'.format(base_dir), archive ] else: command_args = [command, 'x', '-aoa', '-bd', '-y', archive] try: proc = subprocess.run(command_args, stdout=subprocess.PIPE, check=True) logger.debug(proc.stdout) except subprocess.CalledProcessError as cpe: logger.error("Extraction error: %d" % cpe.returncode) if cpe.stdout is not None: logger.error(cpe.stdout) if cpe.stderr is not None: logger.error(cpe.stderr) raise cpe os.unlink(archive) logger.info("Finish installation of {} in {}".format( archive, time.perf_counter() - start_time))
def retrieve_archive(self, package, path=None, command=None): archive = package.archive url = package.url self.logger.info("-Downloading {}...".format(url)) try: r = requests.get(url, allow_redirects=False, stream=True) if r.status_code == 302: newurl = altlink(r.url) # newurl = r.headers['Location'] self.logger.info('Redirected to new URL: {}'.format(newurl)) r = requests.get(newurl, stream=True) except requests.exceptions.ConnectionError as e: self.logger.warning("Caught download error: %s" % e.args) return False else: with open(archive, 'wb') as fd: for chunk in r.iter_content(chunk_size=8196): fd.write(chunk) self.logger.info("-Extracting {}...".format(archive)) if sys.version_info > (3, 5): if not py7zr.is_7zfile(archive): raise BadPackageFile if command is None: py7zr.SevenZipFile(archive).extractall(path=path) else: if path is not None: run([command, 'x', '-aoa', '-bd', '-y', '-o{}'.format(path), archive]) else: run([command, 'x', '-aoa', '-bd', '-y', archive]) os.unlink(archive) return True
def retrieve_archive(self, package: QtPackage): archive = package.archive url = package.url cwd = os.getcwd() self.logger.info("Downloading {}...".format(url)) try: r = requests.get(url, allow_redirects=False, stream=True) if r.status_code == 302: newurl = altlink(r.url) self.logger.info('Redirected to new URL: {}'.format(newurl)) r = requests.get(newurl, stream=True) except requests.exceptions.ConnectionError as e: self.logger.error("Connection error: %s" % e.args) raise e else: try: with open(archive, 'wb') as fd: for chunk in r.iter_content(chunk_size=8196): fd.write(chunk) fd.flush() if self.command is None: with open(archive, 'rb') as fd: self.extract_archive(fd) except Exception as e: exc = sys.exc_info() self.logger.error("Download error: %s" % exc[1]) raise e else: if self.command is not None: self.extract_archive_ext(archive) # work around for extractor bug that may change current working directory os.chdir(cwd) os.unlink(archive) self.logger.info("Finish installation of {} in {}".format( archive, time.process_time()))
def retrieve_archive(self, package: QtPackage): archive = package.archive url = package.url name = package.name start_time = time.perf_counter() self.logger.info("Downloading {}...".format(name)) self.logger.debug("Download URL: {}".format(url)) session = requests.Session() retry = Retry(connect=5, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) try: r = session.get(url, allow_redirects=False, stream=True) if r.status_code == 302: newurl = altlink(r.url, r.headers['Location'], logger=self.logger) self.logger.info('Redirected URL: {}'.format(newurl)) r = session.get(newurl, stream=True) except requests.exceptions.ConnectionError as e: self.logger.error("Connection error: %s" % e.args) raise e else: try: with open(archive, 'wb') as fd: for chunk in r.iter_content(chunk_size=8196): fd.write(chunk) fd.flush() if self.command is None: with open(archive, 'rb') as fd: self.extract_archive(fd) except Exception as e: exc = sys.exc_info() self.logger.error("Download error: %s" % exc[1]) raise e else: if self.command is not None: self.extract_archive_ext(archive) os.unlink(archive) self.logger.info("Finish installation of {} in {}".format( archive, time.perf_counter() - start_time))
def test_helper_altlink(monkeypatch): class Message: headers = {"content-type": "text/plain", "length": 300} text = """<?xml version="1.0" encoding="UTF-8"?> <metalink xmlns="urn:ietf:params:xml:ns:metalink"> <generator>MirrorBrain/2.17.0</generator> <origin dynamic="true">http://download.example.io/boo.7z.meta4</origin> <published>2020-03-04T01:11:48Z</published> <publisher> <name>Example Project</name> <url>https://download.example.io</url> </publisher> <file name="boo.7z"> <size>651</size> <hash type="md5">d49eba3937fb063caa48769e8f28377c</hash> <hash type="sha-1">25d3a33d00c1e5880679a17fd4b8b831134cfa6f</hash> <hash type="sha-256">37e50248cf061109e2cb92105cd2c36a6e271701d6d4a72c4e73c6d82aad790a</hash> <pieces length="262144" type="sha-1"> <hash>bec628a149ed24a3a9b83747776ecca5a1fad11c</hash> <hash>98b1dee3f741de51167a9428b0560cd2d1f4d945</hash> <hash>8717a0cb3d14c1958de5981635c9b90b146da165</hash> <hash>78cd2ae3ae37ca7c080a56a2b34eb33ec44a9ef1</hash> </pieces> <url location="cn" priority="1">http://mirrors.geekpie.club/boo.7z</url> <url location="jp" priority="2">http://ftp.jaist.ac.jp/pub/boo.7z</url> <url location="jp" priority="3">http://ftp.yz.yamagata-u.ac.jp/pub/boo.7z</url> </file> </metalink> """ def mock_return(url): return Message() monkeypatch.setattr(helper, "_get_meta", mock_return) url = "http://foo.baz/qtproject/boo.7z" alt = "http://mirrors.geekpie.club/boo.7z" newurl = helper.altlink(url, alt) assert newurl.startswith("http://ftp.jaist.ac.jp/")
def installer(qt_archive, base_dir, command, keep=False, response_timeout=30): """ Installer function to download archive files and extract it. It is called through multiprocessing.Pool() """ name = qt_archive.name url = qt_archive.url archive = qt_archive.archive start_time = time.perf_counter() logger = getLogger("aqt") logger.info("Downloading {}...".format(name)) logger.debug("Download URL: {}".format(url)) timeout = (3.5, response_timeout) with requests.Session() as session: retry = Retry(connect=5, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) try: r = session.get(url, allow_redirects=False, stream=True, timeout=timeout) if r.status_code == 302: newurl = altlink(r.url, r.headers["Location"], logger=logger) logger.info("Redirected URL: {}".format(newurl)) r = session.get(newurl, stream=True, timeout=timeout) except requests.exceptions.ConnectionError as e: logger.error("Connection error: %s" % e.args) raise e except requests.exceptions.Timeout as e: logger.error("Connection timeout: %s" % e.args) raise e else: try: with open(archive, "wb") as fd: for chunk in r.iter_content(chunk_size=8196): fd.write(chunk) fd.flush() if command is None: with py7zr.SevenZipFile(archive, "r") as szf: szf.extractall(path=base_dir) except Exception as e: exc = sys.exc_info() logger.error("Download error: %s" % exc[1]) raise e else: if command is not None: if base_dir is not None: command_args = [ command, "x", "-aoa", "-bd", "-y", "-o{}".format(base_dir), archive, ] else: command_args = [command, "x", "-aoa", "-bd", "-y", archive] try: proc = subprocess.run( command_args, stdout=subprocess.PIPE, check=True ) logger.debug(proc.stdout) except subprocess.CalledProcessError as cpe: logger.error("Extraction error: %d" % cpe.returncode) if cpe.stdout is not None: logger.error(cpe.stdout) if cpe.stderr is not None: logger.error(cpe.stderr) raise cpe if not keep: os.unlink(archive) logger.info( "Finished installation of {} in {}".format( archive, time.perf_counter() - start_time ) )