def reset_cache(logger, build_folder, cache_folder, **kwargs): """ wipe out the cache folder, optionnaly keeping latest master """ logger.step("Reseting cache folder: {}".format(cache_folder)) cache_size, free_space = display_cache_and_free_space( logger, build_folder, cache_folder ) logger.std("-------------") if kwargs.get("keep_master"): tmp_master_fpath = None master = get_content("hotspot_master_image") master_fpath = os.path.join(cache_folder, master["name"]) if ( os.path.exists(master_fpath) and get_checksum(master_fpath) == master["checksum"] ): # latest master to be moved temporarly to build-dir tmp_master_fpath = os.path.join( build_folder, ".__tmp--{}".format(master["name"]) ) logger.std("Keeping your latest master aside: {}".format(master["name"])) try: shutil.move(master_fpath, tmp_master_fpath) except Exception as exp: logger.err("Unable to move your latest master into build-dir. Exiting.") return 1 logger.std("Removing cache...", end="") try: shutil.rmtree(cache_folder) except Exception as exp: logger.err("FAILED ({}).".format(exp)) else: logger.succ("OK.") logger.std("Recreating cache placeholder.") cache_folder = get_cache(build_folder) if kwargs.get("keep_master"): logger.std("Restoring your latest master.") try: shutil.move(tmp_master_fpath, master_fpath) except Exception as exp: logger.err("Unable to move back your master file into fresh cache.") if tmp_master_fpath is not None: logger.err("Please find your master at: {}".format(tmp_master_fpath)) return 1 logger.std("-------------") display_cache_and_free_space( logger, build_folder, cache_folder, cache_size, free_space ) return 0
def content_is_cached(content, cache_folder, check_sum=False): """ whether a content is already present in cache """ content_fpath = os.path.join(cache_folder, content.get("name")) if not os.path.exists(content_fpath) or os.path.getsize( content_fpath) != content.get("archive_size"): return False if check_sum: return get_checksum(content_fpath) == content.get("checksum") return True
def package_is_latest_version(fpath, fname, logger): """ whether a package (ZIM or ZIP) is in the current catalog """ for catalog in get_catalogs(logger): for (package_id, package) in catalog["all"].items(): package.update({"ext": "zip" if package["type"] != "zim" else "zim"}) package.update({"langid": package.get("langid") or package_id}) rfname = "package_{langid}-{version}.{ext}".format(**package) if rfname == fname and get_checksum(fpath) == package["sha256sum"]: return "{ext}: {fname}".format( fname=package["langid"], ext=package["ext"].upper() ) return False
def refresh_yum_repo(mirror, data, mirror_url, ts): """ Refresh package metadata for a yum-style rpm mirror and add the packages to the mirror """ primary_url, checksum, checksum_type = get_primary_url(mirror_url, data) if not primary_url: mirror.fail() return res = get_url(primary_url) mirror.last_access_ok = response_is_valid(res) if not mirror.last_access_ok: mirror.fail() return data = download_url(res, 'Downloading repo info (2/2):') if data is None: mirror.fail() return computed_checksum = get_checksum(data, Checksum[checksum_type]) if not mirror_checksum_is_valid(computed_checksum, checksum, mirror): return if mirror.file_checksum == checksum: text = 'Mirror checksum has not changed, ' text += 'not refreshing package metadata' warning_message.send(sender=None, text=text) return mirror.file_checksum = checksum if hasattr(settings, 'MAX_MIRRORS') and \ isinstance(settings.MAX_MIRRORS, int): max_mirrors = settings.MAX_MIRRORS # only refresh X mirrors, where X = max_mirrors checksum_q = Q(mirrorlist=False, refresh=True, timestamp=ts, file_checksum=checksum) have_checksum = mirror.repo.mirror_set.filter(checksum_q).count() if have_checksum >= max_mirrors: text = '{0!s} mirrors already have this '.format(max_mirrors) text += 'checksum, ignoring refresh to save time' info_message.send(sender=None, text=text) else: packages = extract_yum_packages(data, primary_url) if packages: update_mirror_packages(mirror, packages)
def download_if_missing(url, fpath, logger, checksum=None): """ returns local file if existing and matching sum otherwise download """ # file already downloaded if checksum and os.path.exists(fpath): logger.std("calculating sum for {}...".format(fpath), "") if get_checksum(fpath) == checksum: logger.std("MATCH.") return RequestedFile.from_disk(url, fpath, checksum) logger.std("MISMATCH.") elif os.path.exists(fpath): return RequestedFile.from_disk(url, fpath) return download_file(url, fpath, logger, checksum, debug=True)
def is_latest_version(fpath, fname, logger): """ whether the filename is a usable content """ if fname.startswith("package_"): return package_is_latest_version(fpath, fname, logger) for key, content in CONTENTS.items(): if not content["name"] == fname: continue if get_checksum(fpath) == content["checksum"]: return key return False
def refresh_arch_repo(repo): """ Refresh all mirrors of an arch linux repo """ if hasattr(settings, 'MAX_MIRRORS') and \ isinstance(settings.MAX_MIRRORS, int): max_mirrors = settings.MAX_MIRRORS fname = '{0!s}/{1!s}.db'.format(repo.arch, repo.repo_id) ts = datetime.now().replace(microsecond=0) for i, mirror in enumerate(repo.mirror_set.filter(refresh=True)): res = find_mirror_url(mirror.url, [fname]) mirror.last_access_ok = response_is_valid(res) if mirror.last_access_ok: if i >= max_mirrors: text = '{0!s} mirrors already refreshed, '.format(max_mirrors) text += ' not refreshing {0!s}'.format(mirror.url) warning_message.send(sender=None, text=text) continue mirror_url = res.url text = 'Found arch repo - {0!s}'.format(mirror_url) info_message.send(sender=None, text=text) data = download_url(res, 'Downloading repo info:') if data is None: mirror.fail() return computed_checksum = get_checksum(data, Checksum.sha1) if mirror.file_checksum == computed_checksum: text = 'Mirror checksum has not changed, ' text += 'not refreshing package metadata' warning_message.send(sender=None, text=text) else: packages = extract_arch_packages(data) mirror.last_access_ok = True mirror.timestamp = ts update_mirror_packages(mirror, packages) mirror.file_checksum = computed_checksum packages.clear() else: mirror.fail() mirror.save()
def refresh_deb_repo(repo): """ Refresh a debian repo. Checks for the Packages* files to determine what the mirror urls are and then downloads and extracts packages from those files. """ formats = ['Packages.bz2', 'Packages.gz', 'Packages'] if lzma is not None: formats.insert(0, 'Packages.xz') ts = datetime.now().replace(microsecond=0) for mirror in repo.mirror_set.filter(refresh=True): res = find_mirror_url(mirror.url, formats) mirror.last_access_ok = response_is_valid(res) if mirror.last_access_ok: mirror_url = res.url text = 'Found deb repo - {0!s}'.format(mirror_url) info_message.send(sender=None, text=text) data = download_url(res, 'Downloading repo info:') if data is None: mirror.fail() return computed_checksum = get_checksum(data, Checksum.sha1) if mirror.file_checksum == computed_checksum: text = 'Mirror checksum has not changed, ' text += 'not refreshing package metadata' warning_message.send(sender=None, text=text) else: packages = extract_deb_packages(data, mirror_url) mirror.last_access_ok = True mirror.timestamp = ts update_mirror_packages(mirror, packages) mirror.file_checksum = computed_checksum packages.clear() else: mirror.fail() mirror.save()
if log_filename == "stdout": log_file = sys.stdout else: try: log_file = open(log_filename, 'w') except IOError: exit("Unable to open " + log_filename + ".") next_acknum = 0 # Receive first packet packet, addr = recv_sock.recvfrom(576) source_port, dest_port, seqnum, acknum, header_length, \ ack, final, window_size, contents = util.unpack(packet) checksum = util.get_checksum(packet) packet_valid = checksum == 0 and next_acknum == acknum if packet_valid: recv_file.write(contents) next_acknum += 1 log = str(datetime.datetime.now()) + " " + str(source_port) + " " + str( dest_port) + " " + str(seqnum) + " " + str(acknum) log_file.write(log + "\n") # Establish ack socket connection ack_sock.connect((sender_ip, sender_port)) out_port = ack_sock.getsockname()[1] ack_segment = util.make_packet(out_port, sender_port, seqnum, acknum, packet_valid, False, 1, "")
def verified(self): return self.present and (get_checksum(self.fpath) == self.checksum or self.checksum is None)
if log_filename == "stdout": log_file = sys.stdout else: try: log_file = open(log_filename, 'w') except IOError: exit("Unable to open " + log_filename + ".") next_acknum = 0 # Receive first packet packet, addr = recv_sock.recvfrom(576) source_port, dest_port, seqnum, acknum, header_length, \ ack, final, window_size, contents = util.unpack(packet) checksum = util.get_checksum(packet) packet_valid = checksum == 0 and next_acknum == acknum if packet_valid: recv_file.write(contents) next_acknum += 1 log = str(datetime.datetime.now()) + " " + str(source_port) + " " + str(dest_port) + " " + str(seqnum) + " " + str( acknum) log_file.write(log + "\n") # Establish ack socket connection ack_sock.connect((sender_ip, sender_port)) out_port = ack_sock.getsockname()[1] ack_segment = util.make_packet(out_port, sender_port, seqnum, acknum, packet_valid,