def _get_schema_from_web(self, kind, version): url = self._get_request_url(kind, version) logger.debug("Validation: fetching schema from {}".format(url)) try: content, _ = make_request(url) except requests.exceptions.HTTPError: raise RequestUnsuccessfulError( "Validation: schema failed to fetch from {}" "\nThe specified version '{}' or kind '{}' may not be supported" .format(url, version, kind)) logger.debug("Validation: schema fetched from {}".format(url)) return yaml.safe_load(content)
def fetch_http_source(source, save_path, item_type): """downloads a http[s] file from source and saves into save_path(which is a file)""" if os.path.exists(save_path): os.remove(save_path) logger.debug("Removed %s", save_path) logger.info("%s %s: fetching now", item_type, source) content, content_type = make_request(source) logger.info("%s %s: successfully fetched", item_type, source) if content is not None: with open(save_path, "wb") as f: f.write(content) logger.debug("Cached to %s", save_path) return content_type logger.warning("%s %s: failed to fetch", item_type, source) return None
def fetch_http_source(source, save_dir): """downloads a http[s] file from source and saves into save_dir""" logger.info("Dependency {} : fetching now".format(source)) content, content_type = make_request(source) logger.info("Dependency {} : successfully fetched".format(source)) if content is not None: basename = os.path.basename(source) # to avoid collisions between basename(source) path_hash = hashlib.sha256( os.path.dirname(source).encode()).hexdigest()[:8] full_save_path = os.path.join(save_dir, path_hash + basename) with open(full_save_path, "wb") as f: f.write(content) return content_type else: logger.warning("Dependency {} : failed to fetch".format(source)) return None