def checkSumCache(self): self.cacheDir.indexUnknownClear() uuid2del = set() for uuid in self.cacheDir.index.keys(): filename = self.cacheDir.index[uuid]['filename'] filepath = os.path.join(self.cacheDir.directory, filename) file_metadata = file_extract_metadata(filepath) if file_metadata == None: uuid2del.add(uuid) continue MatchingUuid = None if int(file_metadata[u'hv:size']) != int( self.cacheDir.index[uuid]['size']): self.log.error("Image '%s' size incorrect." % (uuid)) uuid2del.add(uuid) continue if file_metadata[u'sl:checksum:sha512'] != self.cacheDir.index[ uuid]['sha512']: self.log.error("Image '%s' sha512 incorrect." % (uuid)) uuid2del.add(uuid) continue # Now we remove files that have been corrupted for uuid in uuid2del: if 'filename' in self.cacheDir.index[uuid]: filepath = os.path.join(self.cacheDir.directory, self.cacheDir.index[uuid]['filename']) if os.path.isfile(filepath): os.remove(filepath) self.log.error("Image '%s' was corrupted deleting." % (uuid)) del self.cacheDir.index[uuid] return True
def checkSumCache(self): self.cacheDir.indexUnknownClear() uuid2del = set() for uuid in self.cacheDir.index.keys(): filename = self.cacheDir.index[uuid]['filename'] filepath = os.path.join(self.cacheDir.directory,filename) file_metadata = file_extract_metadata(filepath) if file_metadata == None: uuid2del.add(uuid) continue MatchingUuid = None if int(file_metadata[u'hv:size']) != int(self.cacheDir.index[uuid]['size']): self.log.error("Image '%s' size incorrect." % (uuid)) uuid2del.add(uuid) continue if file_metadata[u'sl:checksum:sha512'] != self.cacheDir.index[uuid]['sha512']: self.log.error("Image '%s' sha512 incorrect." % (uuid)) uuid2del.add(uuid) continue # Now we remove files that have been corrupted for uuid in uuid2del: if 'filename' in self.cacheDir.index[uuid]: filepath = os.path.join(self.cacheDir.directory,self.cacheDir.index[uuid]['filename']) if os.path.isfile(filepath): os.remove(filepath) self.log.error("Image '%s' was corrupted deleting." % (uuid)) del self.cacheDir.index[uuid] return True
def download(self,uuid): if not uuid in self.index.keys(): self.log.error("Image '%s' is not known" % (uuid)) return False DownLoadTrys = 0 if u'DownLoadTrys' in self.index[uuid].keys(): if 'msgHash' in self.index[uuid]['DownLoadTrys'].keys(): if self.index[uuid]['DownLoadTrys']['msgHash'] == self.index[uuid]['msgHash']: DownLoadTrys = int(self.index[uuid]['DownLoadTrys']['count']) if DownLoadTrys >= 5: self.log.warning("Image '%s' has failed to download '%s' times, giving up until new image list." % (uuid,DownLoadTrys)) return False self.index[uuid]['DownLoadTrys'] = { 'count' : DownLoadTrys + 1 , 'msgHash' : self.index[uuid]['msgHash'] } self.index[uuid]['filename'] = uuid self.indexSave() if self.index[uuid]['size'] != None: stats = os.statvfs(self.directory) diskspace = (stats[statvfs.F_BSIZE] * stats[statvfs.F_BAVAIL]) imagesize = self.index[uuid]['size'] if imagesize > diskspace: self.log.error("Image '%s' is too large at '%s' bytes, available disk space is '%s' bytes." % (uuid,imagesize,diskspace)) return False self.log.info("Downloading '%s'." % (uuid)) filepath = os.path.join(self.directory,self.index[uuid]['filename']) uriParsed = uglyUriParser(self.index[uuid]['uri']) unknownTransport = True if uriParsed['scheme'] == u'http': cmd = "wget -q -O %s %s" % (filepath,self.index[uuid]['uri']) rc,output = commands.getstatusoutput(cmd) if rc != 0: self.log.error("Command '%s' failed with return code '%s' for image '%s'" % (cmd,rc,uuid)) if len(output) > 0: self.log.info("Output '%s'" % (output)) os.remove(filepath) return False unknownTransport = False if uriParsed['scheme'] == u'https': cmd = "wget -q -O %s %s --no-check-certificate" % (filepath,self.index[uuid]['uri']) rc,output = commands.getstatusoutput(cmd) if rc != 0: self.log.error("Command '%s' failed with return code '%s' for image '%s'" % (cmd,rc,uuid)) if len(output) > 0: self.log.info("Output '%s'" % (output)) os.remove(filepath) return False unknownTransport = False if uriParsed['scheme'] == u'file': shutil.copyfile(uriParsed['path'],filepath) unknownTransport = False if unknownTransport: self.log.error("Unknown transport '%s' for URI for '%s'" % (uriParsed['scheme'],uuid)) return False file_metadata = file_extract_metadata(filepath) if file_metadata == None: self.log.error("Failed to get metadata for image '%s' at path '%s'." % (uuid,filepath)) return False if file_metadata[u'sl:checksum:sha512'] != self.index[uuid]['sha512']: os.remove(filepath) self.log.error("Downloaded image '%s' has unexpected sha512." % (uuid)) self.log.info("Expected sha512 '%s'" % (self.index[uuid]['sha512'])) self.log.info("Downloaded sha512 '%s'" % (file_metadata[u'sl:checksum:sha512'])) return False if self.index[uuid]['size'] != None: if int(file_metadata[u'hv:size']) != int(self.index[uuid]['size']): os.remove(filepath) self.log.error("Image '%s' is incorrect size." % (uuid)) return False return True