def obtainKIDSPatchFileBySha1(filePath, sha1Sum, cacheDir):
  assert cacheDir and os.path.exists(cacheDir)
  rootDir = os.path.dirname(filePath)
  externalFileName = generateExternalDataFileName(sha1Sum)
  externalFile = os.path.join(rootDir, externalFileName)
  logger.info("Checking %s" % externalFile)
  if os.path.exists(externalFile):
    if generateSha1Sum(externalFile) == sha1Sum:
      return (True, externalFile)
    else:
      os.remove(externalFile)
  """ try to find the file in the cache dir """
  externalFile = os.path.join(cacheDir, externalFileName.replace('_','/'))
  logger.info("Checking %s" % externalFile)
  if os.path.exists(externalFile):
    if generateSha1Sum(externalFile) == sha1Sum:
      return (True, externalFile)
    else:
      os.remove(externalFile)
  """ make sure cacheDir has the right layout """
  rootDir = os.path.dirname(externalFile)
  if not os.path.exists(rootDir):
    os.makedirs(rootDir)
  """ download from remote """
  extDownloader = ExternalDataDownloader()
  logger.info("Downloading from remote link")
  result = extDownloader.downloadExternalDataByHash(sha1Sum, externalFile)
  if not result:
    logger.error("Downloading from remote failed")
    if os.path.exists(externalFile):
      os.remove(externalFile)
    externalFile = None
  logger.info("%s, %s" % (result, externalFile))
  return (result, externalFile)
Exemple #2
0
def obtainKIDSBuildFileBySha1(filePath, sha1Sum, cacheDir):
  assert cacheDir and os.path.exists(cacheDir)
  rootDir = os.path.dirname(filePath)
  externalFileName = generateExternalDataFileName(sha1Sum)
  externalFile = os.path.join(rootDir, externalFileName)
  logger.info("Checking %s" % externalFile)
  if os.path.exists(externalFile):
    if generateSha1Sum(externalFile) == sha1Sum:
      return (True, externalFile)
    else:
      os.remove(externalFile)
  """ try to find the file in the cache dir """
  externalFile = os.path.join(cacheDir, externalFileName.replace('_','/'))
  logger.info("Checking %s" % externalFile)
  if os.path.exists(externalFile):
    if generateSha1Sum(externalFile) == sha1Sum:
      return (True, externalFile)
    else:
      os.remove(externalFile)
  """ make sure cacheDir has the right layout """
  rootDir = os.path.dirname(externalFile)
  if not os.path.exists(rootDir):
    os.makedirs(rootDir)
  """ download from remote """
  extDownloader = ExternalDataDownloader()
  logger.info("Downloading from remote link")
  result = extDownloader.downloadExternalDataByHash(sha1Sum, externalFile)
  if not result:
    logger.error("Downloading from remote failed")
    if os.path.exists(externalFile):
      os.remove(externalFile)
    externalFile = None
  logger.info("%s, %s" % (result, externalFile))
  return (result, externalFile)
Exemple #3
0
 def generateSha1SumForPatchInfo(patchInfo):
   if patchInfo.kidsSha1 is None:
     patchInfo.kidsSha1 = generateSha1Sum(patchInfo.kidsFilePath)
   if patchInfo.kidsInfoPath:
     patchInfo.kidsInfoSha1 = generateSha1Sum(patchInfo.kidsInfoPath)
   if patchInfo.otherKidsInfoList:
     for item in patchInfo.otherKidsInfoList:
       if item[0]:
         item[1] = generateSha1Sum(item[0])
Exemple #4
0
 def generateSha1SumForPatchInfo(patchInfo):
   if patchInfo.kidsSha1 is None:
     patchInfo.kidsSha1 = generateSha1Sum(patchInfo.kidsFilePath)
   if patchInfo.kidsInfoPath:
     patchInfo.kidsInfoSha1 = generateSha1Sum(patchInfo.kidsInfoPath)
   if patchInfo.otherKidsInfoList:
     for item in patchInfo.otherKidsInfoList:
       if item[0]:
         item[1] = generateSha1Sum(item[0])
 def downloadExternalDataByHash(self, sha1Sum, fileToSave):
   dwnUrl = "%s/%s" % (self._siteUrl, sha1Sum)
   if not self.downloadExternalDataDirectly(dwnUrl, fileToSave):
     return False
   """ verify the sha1sum of downloaded file """
   sha1SumDwn = generateSha1Sum(fileToSave)
   if sha1Sum == sha1SumDwn:
     return True
   logger.error("sha1Sum mismatch %s:%s" % (sha1Sum, sha1SumDwn))
   os.remove(fileToSave)
Exemple #6
0
 def downloadExternalDataByHash(self, sha1Sum, fileToSave):
   dwnUrl = "%s/%s" % (self._siteUrl, sha1Sum)
   if not self.downloadExternalDataDirectly(dwnUrl, fileToSave):
     return False
   """ verify the sha1sum of downloaded file """
   sha1SumDwn = generateSha1Sum(fileToSave)
   if sha1Sum == sha1SumDwn:
     return True
   logger.error("sha1Sum mismatch %s:%s" % (sha1Sum, sha1SumDwn))
   os.remove(fileToSave)
Exemple #7
0
def downloadAllKIDSSha1File(topDir, cacheDir):
  from ConvertToExternalData import isValidKIDSBuildSha1Suffix
  from ConvertToExternalData import readSha1SumFromSha1File
  import shutil
  initConsoleLogging()
  absCurDir = os.path.abspath(topDir)
  for (root, dirs, files) in os.walk(absCurDir):
    for f in files:
      if not isValidKIDSBuildSha1Suffix(f):
        continue
      filePath = os.path.join(root, f)
      sha1Sum = readSha1SumFromSha1File(filePath)
      result, extFilePath = obtainKIDSBuildFileBySha1(filePath, sha1Sum, cacheDir)
      if result:
        destFile = filePath[:filePath.rfind('.')]
        if os.path.exists(destFile) and generateSha1Sum(destFile) == sha1Sum:
          logger.info("%s is already current" % destFile)
          continue
        logger.info("%s => %s" % (extFilePath, destFile))
        shutil.copyfile(extFilePath, destFile)