def obtainKIDSPatchFileBySha1(filePath, sha1Sum, cacheDir):
assert cacheDir and os.path.exists(cacheDir)
rootDir = os.path.dirname(filePath)
externalFileName = generateExternalDataFileName(sha1Sum)
externalFile = os.path.join(rootDir, externalFileName)
logger.info("Checking %s" % externalFile)
if os.path.exists(externalFile):
if generateSha1Sum(externalFile) == sha1Sum:
return (True, externalFile)
else:
os.remove(externalFile)
""" try to find the file in the cache dir """
externalFile = os.path.join(cacheDir, externalFileName.replace('_','/'))
logger.info("Checking %s" % externalFile)
if os.path.exists(externalFile):
if generateSha1Sum(externalFile) == sha1Sum:
return (True, externalFile)
else:
os.remove(externalFile)
""" make sure cacheDir has the right layout """
rootDir = os.path.dirname(externalFile)
if not os.path.exists(rootDir):
os.makedirs(rootDir)
""" download from remote """
extDownloader = ExternalDataDownloader()
logger.info("Downloading from remote link")
result = extDownloader.downloadExternalDataByHash(sha1Sum, externalFile)
if not result:
logger.error("Downloading from remote failed")
if os.path.exists(externalFile):
os.remove(externalFile)
externalFile = None
logger.info("%s, %s" % (result, externalFile))
return (result, externalFile)
def obtainKIDSBuildFileBySha1(filePath, sha1Sum, cacheDir):
assert cacheDir and os.path.exists(cacheDir)
rootDir = os.path.dirname(filePath)
externalFileName = generateExternalDataFileName(sha1Sum)
externalFile = os.path.join(rootDir, externalFileName)
logger.info("Checking %s" % externalFile)
if os.path.exists(externalFile):
if generateSha1Sum(externalFile) == sha1Sum:
return (True, externalFile)
else:
os.remove(externalFile)
""" try to find the file in the cache dir """
externalFile = os.path.join(cacheDir, externalFileName.replace('_','/'))
logger.info("Checking %s" % externalFile)
if os.path.exists(externalFile):
if generateSha1Sum(externalFile) == sha1Sum:
return (True, externalFile)
else:
os.remove(externalFile)
""" make sure cacheDir has the right layout """
rootDir = os.path.dirname(externalFile)
if not os.path.exists(rootDir):
os.makedirs(rootDir)
""" download from remote """
extDownloader = ExternalDataDownloader()
logger.info("Downloading from remote link")
result = extDownloader.downloadExternalDataByHash(sha1Sum, externalFile)
if not result:
logger.error("Downloading from remote failed")
if os.path.exists(externalFile):
os.remove(externalFile)
externalFile = None
logger.info("%s, %s" % (result, externalFile))
return (result, externalFile)
def generateSha1SumForPatchInfo(patchInfo):
if patchInfo.kidsSha1 is None:
patchInfo.kidsSha1 = generateSha1Sum(patchInfo.kidsFilePath)
if patchInfo.kidsInfoPath:
patchInfo.kidsInfoSha1 = generateSha1Sum(patchInfo.kidsInfoPath)
if patchInfo.otherKidsInfoList:
for item in patchInfo.otherKidsInfoList:
if item[0]:
item[1] = generateSha1Sum(item[0])
def generateSha1SumForPatchInfo(patchInfo):
if patchInfo.kidsSha1 is None:
patchInfo.kidsSha1 = generateSha1Sum(patchInfo.kidsFilePath)
if patchInfo.kidsInfoPath:
patchInfo.kidsInfoSha1 = generateSha1Sum(patchInfo.kidsInfoPath)
if patchInfo.otherKidsInfoList:
for item in patchInfo.otherKidsInfoList:
if item[0]:
item[1] = generateSha1Sum(item[0])
def downloadExternalDataByHash(self, sha1Sum, fileToSave):
dwnUrl = "%s/%s" % (self._siteUrl, sha1Sum)
if not self.downloadExternalDataDirectly(dwnUrl, fileToSave):
return False
""" verify the sha1sum of downloaded file """
sha1SumDwn = generateSha1Sum(fileToSave)
if sha1Sum == sha1SumDwn:
return True
logger.error("sha1Sum mismatch %s:%s" % (sha1Sum, sha1SumDwn))
os.remove(fileToSave)
def downloadExternalDataByHash(self, sha1Sum, fileToSave):
dwnUrl = "%s/%s" % (self._siteUrl, sha1Sum)
if not self.downloadExternalDataDirectly(dwnUrl, fileToSave):
return False
""" verify the sha1sum of downloaded file """
sha1SumDwn = generateSha1Sum(fileToSave)
if sha1Sum == sha1SumDwn:
return True
logger.error("sha1Sum mismatch %s:%s" % (sha1Sum, sha1SumDwn))
os.remove(fileToSave)
def downloadAllKIDSSha1File(topDir, cacheDir):
from ConvertToExternalData import isValidKIDSBuildSha1Suffix
from ConvertToExternalData import readSha1SumFromSha1File
import shutil
initConsoleLogging()
absCurDir = os.path.abspath(topDir)
for (root, dirs, files) in os.walk(absCurDir):
for f in files:
if not isValidKIDSBuildSha1Suffix(f):
continue
filePath = os.path.join(root, f)
sha1Sum = readSha1SumFromSha1File(filePath)
result, extFilePath = obtainKIDSBuildFileBySha1(filePath, sha1Sum, cacheDir)
if result:
destFile = filePath[:filePath.rfind('.')]
if os.path.exists(destFile) and generateSha1Sum(destFile) == sha1Sum:
logger.info("%s is already current" % destFile)
continue
logger.info("%s => %s" % (extFilePath, destFile))
shutil.copyfile(extFilePath, destFile)
