def fetch(self):
"""fetch normal tarballs"""
CraftCore.log.debug("ArchiveSource.fetch called")
filenames = self.localFileNames()
if (self.noFetch):
CraftCore.log.debug("skipping fetch (--offline)")
return True
if self.subinfo.hasTarget():
if self.__checkFilesPresent(filenames):
CraftCore.log.debug("files and digests available, no need to download files")
return True
if self.subinfo.target():
# compat for scripts that provide multiple files
files = zip(self.subinfo.target(), self.subinfo.archiveName()) if isinstance(self.subinfo.target(), list) else [(self.subinfo.target(), self.subinfo.archiveName()[0])]
for url, fileName in files:
if not GetFiles.getFile(url, self.__downloadDir, fileName):
CraftCore.log.debug("failed to download files")
return False
if self.subinfo.hasTargetDigestUrls():
if isinstance(self.subinfo.targetDigestUrl(), tuple):
url, alg = self.subinfo.targetDigestUrl()
return GetFiles.getFile(url[0], self.__downloadDir, self.subinfo.archiveName()[0] + CraftHash.HashAlgorithm.fileEndings().get(alg))
else:
for url in self.subinfo.targetDigestUrl():
if not GetFiles.getFile(url, self.__downloadDir):
return False
else:
CraftCore.log.debug("no digestUrls present")
return True
def postInstall(self):
if CraftCore.compiler.isWindows:
self.schemeDir = os.path.join(self.installDir(), 'bin', 'data', 'color-schemes')
else:
self.schemeDir = os.path.join(self.installDir(), 'share', 'color-schemes')
for scheme in ['Breeze', 'BreezeDark', 'BreezeHighContrast', 'BreezeLight']:
GetFiles.getFile('https://cgit.kde.org/breeze.git/plain/colors/'+scheme+'.colors', self.schemeDir)
for scheme in ['RustedBronze']:
GetFiles.getFile('https://raw.githubusercontent.com/Bartoloni/RustedBronze/master/'+scheme+'.colors', self.schemeDir)
return True
def postInstall(self):
if CraftCore.compiler.isWindows:
self.schemeDir = os.path.join(self.installDir(), 'bin', 'data',
'color-schemes')
else:
self.schemeDir = os.path.join(self.installDir(), 'share',
'color-schemes')
for scheme in [
'Breeze', 'BreezeDark', 'BreezeHighContrast', 'BreezeLight'
]:
GetFiles.getFile(
'https://cgit.kde.org/breeze.git/plain/colors/' + scheme +
'.colors', self.schemeDir)
for scheme in [
'Honeycomb', 'Norway', 'ObsidianCoast', 'Oxygen', 'OxygenCold',
'Steel', 'WontonSoup', 'Zion', 'ZionReversed'
]:
GetFiles.getFile(
'https://cgit.kde.org/plasma-desktop.git/plain/kcms/colors/schemes/'
+ scheme + '.colors', self.schemeDir)
return True
def __fetchFromArchiveCache(self, downloadRetriesLeft : int=3):
for url, files in self._getFileInfoFromArchiveCache():
self.__downloadDir.mkdir(parents=True, exist_ok=True)
for entry in files:
if entry.version != self.buildTarget:
continue
if not GetFiles.getFile(utils.urljoin(url, entry.fileName), self.__archiveDir, entry.fileName):
self.__retry(downloadRetriesLeft, self.__fetchFromArchiveCache)
if not CraftHash.checkFilesDigests(self.__archiveDir, [entry.fileName],
digests=entry.checksum,
digestAlgorithm=CraftHash.HashAlgorithm.SHA256):
return self.__retry(downloadRetriesLeft,
lambda downloadRetriesLeft: utils.deleteFile(self.__archiveDir / entry.fileName) and self.__fetchFromArchiveCache(downloadRetriesLeft))
if self.__checkFilesPresent(self.localFileNames()):
return True
return False
def cacheJsonFromUrl(self, url, timeout=10) -> object:
CraftCore.log.debug(f"Fetch Json: {url}")
if not url in self._jsonCache:
if os.path.isfile(url):
with open(url, "rt", encoding="UTF-8") as jsonFile:
# don't cache local manifest
return json.loads(jsonFile.read())
else:
with tempfile.TemporaryDirectory() as tmp:
if not GetFiles.getFile(url, tmp, "manifest.json", quiet=True):
# TODO: provide the error code and only cache 404...
self._jsonCache[url] = {}
return {}
with open(os.path.join(tmp, "manifest.json"), "rt", encoding="UTF-8") as jsonFile:
self._jsonCache[url] = json.loads(jsonFile.read())
return self._jsonCache.get(url, {})
def fetchBinary(self, downloadRetriesLeft=3) -> bool:
if self.subinfo.options.package.disableBinaryCache:
return False
for url in [self.cacheLocation()] + self.cacheRepositoryUrls():
CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.")
if url == self.cacheLocation():
fileUrl = f"{url}/manifest.json"
if os.path.exists(fileUrl):
with open(fileUrl, "rt", encoding="UTF-8") as f:
manifest = CraftManifest.fromJson(json.load(f))
else:
continue
else:
manifest = CraftManifest.fromJson(
CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json"))
fileEntry = manifest.get(str(self)).files
files = []
for f in fileEntry:
if f.version == self.version:
files.append(f)
if not files:
CraftCore.log.info(
f"Could not find {self}={self.version} in {url}")
continue
latest = files[0]
if not self.subinfo.options.dynamic.compatible(
latest.config, latest.configHash):
CraftCore.log.info(
"Failed to restore package, configuration missmatch")
CraftCore.debug.debug_line()
CraftCore.log.info("Cached config: {}".format(", ".join(
f"{k}={v}" for k, v in latest.config.items())))
CraftCore.log.info(
f"Local config: {self.subinfo.options.dynamic}")
CraftCore.debug.debug_line()
# try next cache
continue
# if we are creating the cache, a rebuild on a failed fetch would be suboptimal
createingCache = CraftCore.settings.getboolean(
"Packager", "CreateCache", False)
if url != self.cacheLocation():
downloadFolder = self.cacheLocation(
os.path.join(CraftCore.standardDirs.downloadDir(),
"cache"))
else:
downloadFolder = self.cacheLocation()
localArchiveAbsPath = OsUtils.toNativePath(
os.path.join(downloadFolder, latest.fileName))
localArchivePath, localArchiveName = os.path.split(
localArchiveAbsPath)
if url != self.cacheLocation():
if not os.path.exists(localArchiveAbsPath):
os.makedirs(localArchivePath, exist_ok=True)
fileName = latest.fileName
if CraftCore.compiler.isWindows:
fileName = fileName.replace("\\", "/")
fUrl = f"{url}/{fileName}"
# try it up to 3 times
retries = 3
while True:
if GetFiles.getFile(fUrl, localArchivePath,
localArchiveName):
break
msg = f"Failed to fetch {fUrl}"
retries -= 1
if not retries:
if createingCache:
raise BlueprintException(msg, self.package)
else:
CraftCore.log.warning(msg)
return False
elif not os.path.isfile(localArchiveAbsPath):
continue
if not CraftHash.checkFilesDigests(
localArchivePath, [localArchiveName],
digests=latest.checksum,
digestAlgorithm=CraftHash.HashAlgorithm.SHA256):
msg = f"Hash did not match, {localArchiveName} might be corrupted"
CraftCore.log.warning(msg)
if downloadRetriesLeft and CraftChoicePrompt.promptForChoice(
"Do you want to delete the files and redownload them?",
[("Yes", True), ("No", False)],
default="Yes"):
return utils.deleteFile(
localArchiveAbsPath) and self.fetchBinary(
downloadRetriesLeft=downloadRetriesLeft - 1)
if createingCache:
raise BlueprintException(msg, self.package)
return False
self.subinfo.buildPrefix = latest.buildPrefix
self.subinfo.isCachedBuild = True
if not (self.cleanImage() and utils.unpackFile(
localArchivePath, localArchiveName, self.imageDir())
and self.internalPostInstall() and self.postInstall()
and self.qmerge() and self.internalPostQmerge()
and self.postQmerge()):
return False
return True
return False
def getFile(url, destdir, filename='') -> bool:
from Utils import GetFiles
return GetFiles.getFile(url, destdir, filename=filename)
def fetchBinary(self, downloadRetriesLeft=3) -> bool:
if self.subinfo.options.package.disableBinaryCache:
return False
for url in [self.cacheLocation()] + self.cacheRepositoryUrls():
CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.")
if url == self.cacheLocation():
fileUrl = f"{url}/manifest.json"
if os.path.exists(fileUrl):
with open(fileUrl, "rt", encoding="UTF-8") as f:
manifest = CraftManifest.fromJson(json.load(f))
else:
continue
else:
manifest = CraftManifest.fromJson(
CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json"))
fileEntry = manifest.get(str(self)).files
files = []
for f in fileEntry:
if f.version == self.version:
files.append(f)
latest = None
if not files:
CraftCore.log.debug(
f"Could not find {self}={self.version} in {url}")
continue
latest = files[0]
if url != self.cacheLocation():
downloadFolder = self.cacheLocation(
os.path.join(CraftCore.standardDirs.downloadDir(),
"cache"))
else:
downloadFolder = self.cacheLocation()
localArchiveAbsPath = OsUtils.toNativePath(
os.path.join(downloadFolder, latest.fileName))
localArchivePath, localArchiveName = os.path.split(
localArchiveAbsPath)
if url != self.cacheLocation():
if not os.path.exists(localArchiveAbsPath):
os.makedirs(localArchivePath, exist_ok=True)
fUrl = f"{url}/{latest.fileName}"
if not GetFiles.getFile(fUrl, localArchivePath,
localArchiveName):
CraftCore.log.warning(f"Failed to fetch {fUrl}")
return False
elif not os.path.isfile(localArchiveAbsPath):
continue
if not CraftHash.checkFilesDigests(
localArchivePath, [localArchiveName],
digests=latest.checksum,
digestAlgorithm=CraftHash.HashAlgorithm.SHA256):
CraftCore.log.warning(
f"Hash did not match, {localArchiveName} might be corrupted"
)
if downloadRetriesLeft and CraftChoicePrompt.promptForChoice(
"Do you want to delete the files and redownload them?",
[("Yes", True), ("No", False)],
default="Yes"):
return utils.deleteFile(
localArchiveAbsPath) and self.fetchBinary(
downloadRetriesLeft=downloadRetriesLeft - 1)
return False
self.subinfo.buildPrefix = latest.buildPrefix
self.subinfo.isCachedBuild = True
if not (self.cleanImage() and utils.unpackFile(
localArchivePath, localArchiveName, self.imageDir())
and self.internalPostInstall() and self.postInstall()
and self.qmerge() and self.internalPostQmerge()
and self.postQmerge()):
return False
return True
return False