def win_unarchive_compressed_tar_pipe(archive_fpath, dest_folder, logger):
""" uncompress tar.[bz2|gz] on windows in a single pass """
# 7z extraction to stdout
uncompress = subprocess.Popen([szip_exe, "x", "-so", archive_fpath],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**startup_info_args())
logger.std("Call: " + str(uncompress.args))
# 7x tar extraction using stdin (piping other process' stdout)
untar = subprocess.Popen(
[szip_exe, "x", "-si", "-ttar", "-o{}".format(dest_folder)],
stdin=uncompress.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**startup_info_args())
logger.std("Call: " + str(untar.args))
uncompress.wait()
untar.wait()
for line in untar.stdout.readlines():
logger.raw_std(line.decode("utf-8", "ignore"))
assert untar.returncode == 0
def get_iterator():
lines = subprocess.check_output(["wmic", "diskdrive"], **startup_info_args()).decode('utf-8').splitlines()
column = {}
matches = re.finditer(r"(\w+\W+)", lines[0])
for match in matches:
column[lines[0][match.start():match.end()].strip()] = match
devices = []
lines.pop(0)
for line in filter(lambda l: len(l) is not 0, lines):
size = extract_field(column["Size"], line)
formatted_size = human_readable_size(size)
devices.append({
"caption": extract_field(column["Caption"], line),
"description": extract_field(column["Description"], line),
"device": extract_field(column["DeviceID"], line),
"media_type": extract_field(column["MediaType"], line),
"model": extract_field(column["Model"], line),
"name": extract_field(column["Name"], line),
"size": size,
"formatted_size": formatted_size,
})
return filter(lambda d: d["media_type"] != "Fixed hard disk media" and d["size"] is not '', devices)
def download_file(url, fpath, logger, checksum=None, debug=False):
""" download an URL into a named path and reports progress to logger
download is externalized to aria2c binary and progress extracted
from periodic summary
downloads are resumed if possible
supports metalink. if link is metalink, downloads both then replace
actual target (aria2 doesn't allow setting target for metalink
as it can be multiple files) """
output_dir, fname = os.path.split(fpath)
args = [aria2_exe, "--dir={}".format(output_dir), "--out={}".format(fname)]
args += [
"--connect-timeout=60",
"--max-file-not-found=5",
"--max-tries=5",
"--retry-wait=60",
"--timeout=60",
"--follow-metalink=mem",
"--allow-overwrite=true",
"--always-resume=false",
"--max-resume-failure-tries=1",
"--auto-file-renaming=false",
"--download-result=full",
"--log-level=error",
"--console-log-level=error",
"--summary-interval=1", # display a line with progress every X seconds
"--human-readable={}".format(str(logger.on_tty).lower()),
"--ca-certificate={}".format(
os.path.join(data_dir, "ca-certificates.crt")),
]
# zip* files might send incorrect gzip header and get auto-extracted by aria2
if not fname.endswith("zip"):
args += ["--http-accept-gzip=true"]
args += [url]
aria2c = subprocess.Popen(args,
stdout=subprocess.PIPE,
universal_newlines=True,
**startup_info_args())
if debug:
logger.std(" ".join(args))
if not logger.on_tty:
logger.ascii_progressbar(0, 100)
metalink_target = None
for line in iter(aria2c.stdout.readline, ""):
line = line.strip()
# [#915371 5996544B/90241109B(6%) CN:4 DL:1704260B ETA:49s]
if line.startswith("[#") and line.endswith("]"): # quick check, no re
if logger.on_tty:
logger.flash(line + " ")
else:
try:
downloaded_size, total_size = [
int(x) for x in list(
re.search(r"\s([0-9]+)B\/([0-9]+)B",
line).groups())
]
except Exception:
downloaded_size, total_size = 1, -1
logger.ascii_progressbar(downloaded_size, total_size)
# parse filename from progress report
if (metalink_target is None and line.startswith("FILE:")
and "[MEMORY]" not in line):
metalink_target = os.path.join(
output_dir, os.path.basename(line.split(":")[-1].strip()))
# parse metalink filename from results summary (if not caught before)
if metalink_target is None and "|OK |" in line and "[MEMORY]" not in line:
metalink_target = os.path.join(
output_dir, os.path.basename(line.split("|", 4)[-1].strip()))
if aria2c.poll() is None:
try:
aria2c.wait(timeout=2)
except subprocess.TimeoutError:
aria2c.terminate()
logger.std("") # clear last \r
if not aria2c.returncode == 0:
return RequestedFile.from_failure(
url,
fpath,
ValueError("aria2c returned {}".format(aria2c.returncode)),
checksum,
)
if metalink_target is not None and metalink_target != fpath:
logger.std("mv {src} {dst}".format(src=metalink_target, dst=fpath))
time.sleep(5)
os.replace(metalink_target, fpath)
return RequestedFile.from_download(url, fpath, os.path.getsize(fpath))