def win_unarchive_compressed_tar_pipe(archive_fpath, dest_folder, logger): """ uncompress tar.[bz2|gz] on windows in a single pass """ # 7z extraction to stdout uncompress = subprocess.Popen([szip_exe, "x", "-so", archive_fpath], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **startup_info_args()) logger.std("Call: " + str(uncompress.args)) # 7x tar extraction using stdin (piping other process' stdout) untar = subprocess.Popen( [szip_exe, "x", "-si", "-ttar", "-o{}".format(dest_folder)], stdin=uncompress.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **startup_info_args()) logger.std("Call: " + str(untar.args)) uncompress.wait() untar.wait() for line in untar.stdout.readlines(): logger.raw_std(line.decode("utf-8", "ignore")) assert untar.returncode == 0
def get_iterator(): lines = subprocess.check_output(["wmic", "diskdrive"], **startup_info_args()).decode('utf-8').splitlines() column = {} matches = re.finditer(r"(\w+\W+)", lines[0]) for match in matches: column[lines[0][match.start():match.end()].strip()] = match devices = [] lines.pop(0) for line in filter(lambda l: len(l) is not 0, lines): size = extract_field(column["Size"], line) formatted_size = human_readable_size(size) devices.append({ "caption": extract_field(column["Caption"], line), "description": extract_field(column["Description"], line), "device": extract_field(column["DeviceID"], line), "media_type": extract_field(column["MediaType"], line), "model": extract_field(column["Model"], line), "name": extract_field(column["Name"], line), "size": size, "formatted_size": formatted_size, }) return filter(lambda d: d["media_type"] != "Fixed hard disk media" and d["size"] is not '', devices)
def download_file(url, fpath, logger, checksum=None, debug=False): """ download an URL into a named path and reports progress to logger download is externalized to aria2c binary and progress extracted from periodic summary downloads are resumed if possible supports metalink. if link is metalink, downloads both then replace actual target (aria2 doesn't allow setting target for metalink as it can be multiple files) """ output_dir, fname = os.path.split(fpath) args = [aria2_exe, "--dir={}".format(output_dir), "--out={}".format(fname)] args += [ "--connect-timeout=60", "--max-file-not-found=5", "--max-tries=5", "--retry-wait=60", "--timeout=60", "--follow-metalink=mem", "--allow-overwrite=true", "--always-resume=false", "--max-resume-failure-tries=1", "--auto-file-renaming=false", "--download-result=full", "--log-level=error", "--console-log-level=error", "--summary-interval=1", # display a line with progress every X seconds "--human-readable={}".format(str(logger.on_tty).lower()), "--ca-certificate={}".format( os.path.join(data_dir, "ca-certificates.crt")), ] # zip* files might send incorrect gzip header and get auto-extracted by aria2 if not fname.endswith("zip"): args += ["--http-accept-gzip=true"] args += [url] aria2c = subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True, **startup_info_args()) if debug: logger.std(" ".join(args)) if not logger.on_tty: logger.ascii_progressbar(0, 100) metalink_target = None for line in iter(aria2c.stdout.readline, ""): line = line.strip() # [#915371 5996544B/90241109B(6%) CN:4 DL:1704260B ETA:49s] if line.startswith("[#") and line.endswith("]"): # quick check, no re if logger.on_tty: logger.flash(line + " ") else: try: downloaded_size, total_size = [ int(x) for x in list( re.search(r"\s([0-9]+)B\/([0-9]+)B", line).groups()) ] except Exception: downloaded_size, total_size = 1, -1 logger.ascii_progressbar(downloaded_size, total_size) # parse filename from progress report if (metalink_target is None and line.startswith("FILE:") and "[MEMORY]" not in line): metalink_target = os.path.join( output_dir, os.path.basename(line.split(":")[-1].strip())) # parse metalink filename from results summary (if not caught before) if metalink_target is None and "|OK |" in line and "[MEMORY]" not in line: metalink_target = os.path.join( output_dir, os.path.basename(line.split("|", 4)[-1].strip())) if aria2c.poll() is None: try: aria2c.wait(timeout=2) except subprocess.TimeoutError: aria2c.terminate() logger.std("") # clear last \r if not aria2c.returncode == 0: return RequestedFile.from_failure( url, fpath, ValueError("aria2c returned {}".format(aria2c.returncode)), checksum, ) if metalink_target is not None and metalink_target != fpath: logger.std("mv {src} {dst}".format(src=metalink_target, dst=fpath)) time.sleep(5) os.replace(metalink_target, fpath) return RequestedFile.from_download(url, fpath, os.path.getsize(fpath))