def installer(qt_archive, base_dir, command):
name = qt_archive.name
url = qt_archive.url
archive = qt_archive.archive
start_time = time.perf_counter()
logger = getLogger('aqt')
logger.info("Downloading {}...".format(name))
logger.debug("Download URL: {}".format(url))
session = requests.Session()
retry = Retry(connect=5, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
try:
r = session.get(url, allow_redirects=False, stream=True)
if r.status_code == 302:
newurl = altlink(r.url, r.headers['Location'], logger=logger)
logger.info('Redirected URL: {}'.format(newurl))
r = session.get(newurl, stream=True)
except requests.exceptions.ConnectionError as e:
logger.error("Connection error: %s" % e.args)
raise e
else:
try:
with open(archive, 'wb') as fd:
for chunk in r.iter_content(chunk_size=8196):
fd.write(chunk)
fd.flush()
if command is None:
with py7zr.SevenZipFile(archive, 'r') as szf:
szf.extractall(path=base_dir)
except Exception as e:
exc = sys.exc_info()
logger.error("Download error: %s" % exc[1])
raise e
else:
if command is not None:
if base_dir is not None:
command_args = [
command, 'x', '-aoa', '-bd', '-y',
'-o{}'.format(base_dir), archive
]
else:
command_args = [command, 'x', '-aoa', '-bd', '-y', archive]
try:
proc = subprocess.run(command_args,
stdout=subprocess.PIPE,
check=True)
logger.debug(proc.stdout)
except subprocess.CalledProcessError as cpe:
logger.error("Extraction error: %d" % cpe.returncode)
if cpe.stdout is not None:
logger.error(cpe.stdout)
if cpe.stderr is not None:
logger.error(cpe.stderr)
raise cpe
os.unlink(archive)
logger.info("Finish installation of {} in {}".format(
archive,
time.perf_counter() - start_time))
def retrieve_archive(self, package, path=None, command=None):
archive = package.archive
url = package.url
self.logger.info("-Downloading {}...".format(url))
try:
r = requests.get(url, allow_redirects=False, stream=True)
if r.status_code == 302:
newurl = altlink(r.url)
# newurl = r.headers['Location']
self.logger.info('Redirected to new URL: {}'.format(newurl))
r = requests.get(newurl, stream=True)
except requests.exceptions.ConnectionError as e:
self.logger.warning("Caught download error: %s" % e.args)
return False
else:
with open(archive, 'wb') as fd:
for chunk in r.iter_content(chunk_size=8196):
fd.write(chunk)
self.logger.info("-Extracting {}...".format(archive))
if sys.version_info > (3, 5):
if not py7zr.is_7zfile(archive):
raise BadPackageFile
if command is None:
py7zr.SevenZipFile(archive).extractall(path=path)
else:
if path is not None:
run([command, 'x', '-aoa', '-bd', '-y', '-o{}'.format(path), archive])
else:
run([command, 'x', '-aoa', '-bd', '-y', archive])
os.unlink(archive)
return True
def retrieve_archive(self, package: QtPackage):
archive = package.archive
url = package.url
cwd = os.getcwd()
self.logger.info("Downloading {}...".format(url))
try:
r = requests.get(url, allow_redirects=False, stream=True)
if r.status_code == 302:
newurl = altlink(r.url)
self.logger.info('Redirected to new URL: {}'.format(newurl))
r = requests.get(newurl, stream=True)
except requests.exceptions.ConnectionError as e:
self.logger.error("Connection error: %s" % e.args)
raise e
else:
try:
with open(archive, 'wb') as fd:
for chunk in r.iter_content(chunk_size=8196):
fd.write(chunk)
fd.flush()
if self.command is None:
with open(archive, 'rb') as fd:
self.extract_archive(fd)
except Exception as e:
exc = sys.exc_info()
self.logger.error("Download error: %s" % exc[1])
raise e
else:
if self.command is not None:
self.extract_archive_ext(archive)
# work around for extractor bug that may change current working directory
os.chdir(cwd)
os.unlink(archive)
self.logger.info("Finish installation of {} in {}".format(
archive, time.process_time()))
def retrieve_archive(self, package: QtPackage):
archive = package.archive
url = package.url
name = package.name
start_time = time.perf_counter()
self.logger.info("Downloading {}...".format(name))
self.logger.debug("Download URL: {}".format(url))
session = requests.Session()
retry = Retry(connect=5, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
try:
r = session.get(url, allow_redirects=False, stream=True)
if r.status_code == 302:
newurl = altlink(r.url,
r.headers['Location'],
logger=self.logger)
self.logger.info('Redirected URL: {}'.format(newurl))
r = session.get(newurl, stream=True)
except requests.exceptions.ConnectionError as e:
self.logger.error("Connection error: %s" % e.args)
raise e
else:
try:
with open(archive, 'wb') as fd:
for chunk in r.iter_content(chunk_size=8196):
fd.write(chunk)
fd.flush()
if self.command is None:
with open(archive, 'rb') as fd:
self.extract_archive(fd)
except Exception as e:
exc = sys.exc_info()
self.logger.error("Download error: %s" % exc[1])
raise e
else:
if self.command is not None:
self.extract_archive_ext(archive)
os.unlink(archive)
self.logger.info("Finish installation of {} in {}".format(
archive,
time.perf_counter() - start_time))
def test_helper_altlink(monkeypatch):
class Message:
headers = {"content-type": "text/plain", "length": 300}
text = """<?xml version="1.0" encoding="UTF-8"?>
<metalink xmlns="urn:ietf:params:xml:ns:metalink">
<generator>MirrorBrain/2.17.0</generator>
<origin dynamic="true">http://download.example.io/boo.7z.meta4</origin>
<published>2020-03-04T01:11:48Z</published>
<publisher>
<name>Example Project</name>
<url>https://download.example.io</url>
</publisher>
<file name="boo.7z">
<size>651</size>
<hash type="md5">d49eba3937fb063caa48769e8f28377c</hash>
<hash type="sha-1">25d3a33d00c1e5880679a17fd4b8b831134cfa6f</hash>
<hash type="sha-256">37e50248cf061109e2cb92105cd2c36a6e271701d6d4a72c4e73c6d82aad790a</hash>
<pieces length="262144" type="sha-1">
<hash>bec628a149ed24a3a9b83747776ecca5a1fad11c</hash>
<hash>98b1dee3f741de51167a9428b0560cd2d1f4d945</hash>
<hash>8717a0cb3d14c1958de5981635c9b90b146da165</hash>
<hash>78cd2ae3ae37ca7c080a56a2b34eb33ec44a9ef1</hash>
</pieces>
<url location="cn" priority="1">http://mirrors.geekpie.club/boo.7z</url>
<url location="jp" priority="2">http://ftp.jaist.ac.jp/pub/boo.7z</url>
<url location="jp" priority="3">http://ftp.yz.yamagata-u.ac.jp/pub/boo.7z</url>
</file>
</metalink>
"""
def mock_return(url):
return Message()
monkeypatch.setattr(helper, "_get_meta", mock_return)
url = "http://foo.baz/qtproject/boo.7z"
alt = "http://mirrors.geekpie.club/boo.7z"
newurl = helper.altlink(url, alt)
assert newurl.startswith("http://ftp.jaist.ac.jp/")
def installer(qt_archive, base_dir, command, keep=False, response_timeout=30):
"""
Installer function to download archive files and extract it.
It is called through multiprocessing.Pool()
"""
name = qt_archive.name
url = qt_archive.url
archive = qt_archive.archive
start_time = time.perf_counter()
logger = getLogger("aqt")
logger.info("Downloading {}...".format(name))
logger.debug("Download URL: {}".format(url))
timeout = (3.5, response_timeout)
with requests.Session() as session:
retry = Retry(connect=5, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
try:
r = session.get(url, allow_redirects=False, stream=True, timeout=timeout)
if r.status_code == 302:
newurl = altlink(r.url, r.headers["Location"], logger=logger)
logger.info("Redirected URL: {}".format(newurl))
r = session.get(newurl, stream=True, timeout=timeout)
except requests.exceptions.ConnectionError as e:
logger.error("Connection error: %s" % e.args)
raise e
except requests.exceptions.Timeout as e:
logger.error("Connection timeout: %s" % e.args)
raise e
else:
try:
with open(archive, "wb") as fd:
for chunk in r.iter_content(chunk_size=8196):
fd.write(chunk)
fd.flush()
if command is None:
with py7zr.SevenZipFile(archive, "r") as szf:
szf.extractall(path=base_dir)
except Exception as e:
exc = sys.exc_info()
logger.error("Download error: %s" % exc[1])
raise e
else:
if command is not None:
if base_dir is not None:
command_args = [
command,
"x",
"-aoa",
"-bd",
"-y",
"-o{}".format(base_dir),
archive,
]
else:
command_args = [command, "x", "-aoa", "-bd", "-y", archive]
try:
proc = subprocess.run(
command_args, stdout=subprocess.PIPE, check=True
)
logger.debug(proc.stdout)
except subprocess.CalledProcessError as cpe:
logger.error("Extraction error: %d" % cpe.returncode)
if cpe.stdout is not None:
logger.error(cpe.stdout)
if cpe.stderr is not None:
logger.error(cpe.stderr)
raise cpe
if not keep:
os.unlink(archive)
logger.info(
"Finished installation of {} in {}".format(
archive, time.perf_counter() - start_time
)
)
