def fetch(self): """fetch normal tarballs""" CraftCore.log.debug("ArchiveSource.fetch called") filenames = self.localFileNames() if (self.noFetch): CraftCore.log.debug("skipping fetch (--offline)") return True if self.subinfo.hasTarget(): if self.__checkFilesPresent(filenames): CraftCore.log.debug("files and digests available, no need to download files") return True if self.subinfo.target(): # compat for scripts that provide multiple files files = zip(self.subinfo.target(), self.subinfo.archiveName()) if isinstance(self.subinfo.target(), list) else [(self.subinfo.target(), self.subinfo.archiveName()[0])] for url, fileName in files: if not GetFiles.getFile(url, self.__downloadDir, fileName): CraftCore.log.debug("failed to download files") return False if self.subinfo.hasTargetDigestUrls(): if isinstance(self.subinfo.targetDigestUrl(), tuple): url, alg = self.subinfo.targetDigestUrl() return GetFiles.getFile(url[0], self.__downloadDir, self.subinfo.archiveName()[0] + CraftHash.HashAlgorithm.fileEndings().get(alg)) else: for url in self.subinfo.targetDigestUrl(): if not GetFiles.getFile(url, self.__downloadDir): return False else: CraftCore.log.debug("no digestUrls present") return True
def postInstall(self): if CraftCore.compiler.isWindows: self.schemeDir = os.path.join(self.installDir(), 'bin', 'data', 'color-schemes') else: self.schemeDir = os.path.join(self.installDir(), 'share', 'color-schemes') for scheme in ['Breeze', 'BreezeDark', 'BreezeHighContrast', 'BreezeLight']: GetFiles.getFile('https://cgit.kde.org/breeze.git/plain/colors/'+scheme+'.colors', self.schemeDir) for scheme in ['RustedBronze']: GetFiles.getFile('https://raw.githubusercontent.com/Bartoloni/RustedBronze/master/'+scheme+'.colors', self.schemeDir) return True
def postInstall(self): if CraftCore.compiler.isWindows: self.schemeDir = os.path.join(self.installDir(), 'bin', 'data', 'color-schemes') else: self.schemeDir = os.path.join(self.installDir(), 'share', 'color-schemes') for scheme in [ 'Breeze', 'BreezeDark', 'BreezeHighContrast', 'BreezeLight' ]: GetFiles.getFile( 'https://cgit.kde.org/breeze.git/plain/colors/' + scheme + '.colors', self.schemeDir) for scheme in [ 'Honeycomb', 'Norway', 'ObsidianCoast', 'Oxygen', 'OxygenCold', 'Steel', 'WontonSoup', 'Zion', 'ZionReversed' ]: GetFiles.getFile( 'https://cgit.kde.org/plasma-desktop.git/plain/kcms/colors/schemes/' + scheme + '.colors', self.schemeDir) return True
def __fetchFromArchiveCache(self, downloadRetriesLeft : int=3): for url, files in self._getFileInfoFromArchiveCache(): self.__downloadDir.mkdir(parents=True, exist_ok=True) for entry in files: if entry.version != self.buildTarget: continue if not GetFiles.getFile(utils.urljoin(url, entry.fileName), self.__archiveDir, entry.fileName): self.__retry(downloadRetriesLeft, self.__fetchFromArchiveCache) if not CraftHash.checkFilesDigests(self.__archiveDir, [entry.fileName], digests=entry.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): return self.__retry(downloadRetriesLeft, lambda downloadRetriesLeft: utils.deleteFile(self.__archiveDir / entry.fileName) and self.__fetchFromArchiveCache(downloadRetriesLeft)) if self.__checkFilesPresent(self.localFileNames()): return True return False
def cacheJsonFromUrl(self, url, timeout=10) -> object: CraftCore.log.debug(f"Fetch Json: {url}") if not url in self._jsonCache: if os.path.isfile(url): with open(url, "rt", encoding="UTF-8") as jsonFile: # don't cache local manifest return json.loads(jsonFile.read()) else: with tempfile.TemporaryDirectory() as tmp: if not GetFiles.getFile(url, tmp, "manifest.json", quiet=True): # TODO: provide the error code and only cache 404... self._jsonCache[url] = {} return {} with open(os.path.join(tmp, "manifest.json"), "rt", encoding="UTF-8") as jsonFile: self._jsonCache[url] = json.loads(jsonFile.read()) return self._jsonCache.get(url, {})
def fetchBinary(self, downloadRetriesLeft=3) -> bool: if self.subinfo.options.package.disableBinaryCache: return False for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.") if url == self.cacheLocation(): fileUrl = f"{url}/manifest.json" if os.path.exists(fileUrl): with open(fileUrl, "rt", encoding="UTF-8") as f: manifest = CraftManifest.fromJson(json.load(f)) else: continue else: manifest = CraftManifest.fromJson( CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) fileEntry = manifest.get(str(self)).files files = [] for f in fileEntry: if f.version == self.version: files.append(f) if not files: CraftCore.log.info( f"Could not find {self}={self.version} in {url}") continue latest = files[0] if not self.subinfo.options.dynamic.compatible( latest.config, latest.configHash): CraftCore.log.info( "Failed to restore package, configuration missmatch") CraftCore.debug.debug_line() CraftCore.log.info("Cached config: {}".format(", ".join( f"{k}={v}" for k, v in latest.config.items()))) CraftCore.log.info( f"Local config: {self.subinfo.options.dynamic}") CraftCore.debug.debug_line() # try next cache continue # if we are creating the cache, a rebuild on a failed fetch would be suboptimal createingCache = CraftCore.settings.getboolean( "Packager", "CreateCache", False) if url != self.cacheLocation(): downloadFolder = self.cacheLocation( os.path.join(CraftCore.standardDirs.downloadDir(), "cache")) else: downloadFolder = self.cacheLocation() localArchiveAbsPath = OsUtils.toNativePath( os.path.join(downloadFolder, latest.fileName)) localArchivePath, localArchiveName = os.path.split( localArchiveAbsPath) if url != self.cacheLocation(): if not os.path.exists(localArchiveAbsPath): os.makedirs(localArchivePath, exist_ok=True) fileName = latest.fileName if CraftCore.compiler.isWindows: fileName = fileName.replace("\\", "/") fUrl = f"{url}/{fileName}" # try it up to 3 times retries = 3 while True: if GetFiles.getFile(fUrl, localArchivePath, localArchiveName): break msg = f"Failed to fetch {fUrl}" retries -= 1 if not retries: if createingCache: raise BlueprintException(msg, self.package) else: CraftCore.log.warning(msg) return False elif not os.path.isfile(localArchiveAbsPath): continue if not CraftHash.checkFilesDigests( localArchivePath, [localArchiveName], digests=latest.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): msg = f"Hash did not match, {localArchiveName} might be corrupted" CraftCore.log.warning(msg) if downloadRetriesLeft and CraftChoicePrompt.promptForChoice( "Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes"): return utils.deleteFile( localArchiveAbsPath) and self.fetchBinary( downloadRetriesLeft=downloadRetriesLeft - 1) if createingCache: raise BlueprintException(msg, self.package) return False self.subinfo.buildPrefix = latest.buildPrefix self.subinfo.isCachedBuild = True if not (self.cleanImage() and utils.unpackFile( localArchivePath, localArchiveName, self.imageDir()) and self.internalPostInstall() and self.postInstall() and self.qmerge() and self.internalPostQmerge() and self.postQmerge()): return False return True return False
def getFile(url, destdir, filename='') -> bool: from Utils import GetFiles return GetFiles.getFile(url, destdir, filename=filename)
def getFiles(urls, destdir, suffix='', filenames=''): from Utils import GetFiles return GetFiles.getFiles(urls, destdir, suffix=suffix, filenames=filenames)
def fetchBinary(self, downloadRetriesLeft=3) -> bool: if self.subinfo.options.package.disableBinaryCache: return False for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.") if url == self.cacheLocation(): fileUrl = f"{url}/manifest.json" if os.path.exists(fileUrl): with open(fileUrl, "rt", encoding="UTF-8") as f: manifest = CraftManifest.fromJson(json.load(f)) else: continue else: manifest = CraftManifest.fromJson( CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) fileEntry = manifest.get(str(self)).files files = [] for f in fileEntry: if f.version == self.version: files.append(f) latest = None if not files: CraftCore.log.debug( f"Could not find {self}={self.version} in {url}") continue latest = files[0] if url != self.cacheLocation(): downloadFolder = self.cacheLocation( os.path.join(CraftCore.standardDirs.downloadDir(), "cache")) else: downloadFolder = self.cacheLocation() localArchiveAbsPath = OsUtils.toNativePath( os.path.join(downloadFolder, latest.fileName)) localArchivePath, localArchiveName = os.path.split( localArchiveAbsPath) if url != self.cacheLocation(): if not os.path.exists(localArchiveAbsPath): os.makedirs(localArchivePath, exist_ok=True) fUrl = f"{url}/{latest.fileName}" if not GetFiles.getFile(fUrl, localArchivePath, localArchiveName): CraftCore.log.warning(f"Failed to fetch {fUrl}") return False elif not os.path.isfile(localArchiveAbsPath): continue if not CraftHash.checkFilesDigests( localArchivePath, [localArchiveName], digests=latest.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): CraftCore.log.warning( f"Hash did not match, {localArchiveName} might be corrupted" ) if downloadRetriesLeft and CraftChoicePrompt.promptForChoice( "Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes"): return utils.deleteFile( localArchiveAbsPath) and self.fetchBinary( downloadRetriesLeft=downloadRetriesLeft - 1) return False self.subinfo.buildPrefix = latest.buildPrefix self.subinfo.isCachedBuild = True if not (self.cleanImage() and utils.unpackFile( localArchivePath, localArchiveName, self.imageDir()) and self.internalPostInstall() and self.postInstall() and self.qmerge() and self.internalPostQmerge() and self.postQmerge()): return False return True return False