def check(self): ''' Check repository for unreferenced and missing files ''' # Check if the repo is local if not self.local: raise ISError(u"Repository must be local") local_files = set(listdir(self.config.path)) local_files.remove(self.config.dbname) local_files.remove(self.config.lastname) db_files = set(self.getallmd5()) # check missing files arrow("Checking missing files") missing_files = db_files - local_files if len(missing_files) > 0: out(linesep.join(missing_files)) # check unreferenced files arrow("Checking unreferenced files") unref_files = local_files - db_files if len(unref_files) > 0: out(linesep.join(unref_files)) # check corruption of local files arrow("Checking corrupted files") for f in local_files: fo = PipeFile(join(self.config.path, f)) fo.consume() fo.close() if fo.md5 != f: out(f)
def get(self, name, version=None): ''' Return an image from a name and version ''' # is no version take the last if version is None: version = self.last(name) if version is None: raise ISError(u"Unable to find image %s in %s" % (name, self.config.name)) # get file md5 from db r = self.db.ask("select md5 from image where name = ? and version = ? limit 1", (name, version)).fetchone() if r is None: raise ISError(u"Unable to find image %s v%s in %s" % (name, version, self.config.name)) path = join(self.config.path, r[0]) # getting the file arrow(u"Loading image %s v%s from repository %s" % (name, version, self.config.name)) memfile = StringIO() try: fo = PipeFile(path, "r") fo.consume(memfile) fo.close() except Exception as e: raise ISError(u"Loading image %s v%s failed" % (name, version), e) memfile.seek(0) pkg = PackageImage(path, fileobj=memfile, md5name=True) if pkg.md5 != r[0]: raise ISError(u"Image MD5 verification failure") return pkg
def create_payload_tarball(self, tar_path, data_path, compressor): ''' Create a payload tarball ''' try: # get compressor argv (first to escape file creation if not found) a_comp = get_compressor_path(compressor, compress=True) a_tar = ["tar", "--create", "--numeric-owner", "--directory", data_path, "."] # create destination file f_dst = PipeFile(tar_path, "w", progressbar=True) # run tar process p_tar = Popen(a_tar, shell=False, close_fds=True, stdout=PIPE) # run compressor process p_comp = Popen(a_comp, shell=False, close_fds=True, stdin=p_tar.stdout, stdout=PIPE) # write data from compressor to tar_path f_dst.consume(p_comp.stdout) # close all fd p_tar.stdout.close() p_comp.stdout.close() f_dst.close() # check tar return 0 if p_tar.wait() != 0: raise ISError("Tar return is not zero") # check compressor return 0 if p_comp.wait() != 0: raise ISError(u"Compressor %s return is not zero" % a_comp[0]) except (SystemExit, KeyboardInterrupt): if exists(tar_path): unlink(tar_path) raise
def create_payload_file(self, dest, source, compressor): ''' Create a payload file ''' try: # get compressor argv (first to escape file creation if not found) a_comp = get_compressor_path(compressor, compress=True) # open source file f_src = open(source, "r") # create destination file f_dst = PipeFile(dest, "w", progressbar=True) # run compressor p_comp = Popen(a_comp, shell=False, close_fds=True, stdin=f_src, stdout=PIPE) # close source file fd f_src.close() # write data from compressor to dest file f_dst.consume(p_comp.stdout) # close compressor stdin and destination file p_comp.stdout.close() f_dst.close() # check compressor return 0 if p_comp.wait() != 0: raise ISError(u"Compressor %s return is not zero" % a_comp[0]) except (SystemExit, KeyboardInterrupt): if exists(dest): unlink(dest) raise
def download(self, directory, force=False, image=True, payload=False): ''' Download image in directory Doesn't use in memory image because we cannot access it This is done to don't parasitize self._tarfile access to memfile ''' # check if destination exists directory = abspath(directory) if image: dest = join(directory, self.filename) if not force and exists(dest): raise ISError(u"Image destination already exists: %s" % dest) # some display arrow(u"Downloading image in %s" % directory) debug(u"Downloading %s from %s" % (self.filename, self.path)) # open source fs = PipeFile(self.path, progressbar=True) # check if announced file size is good if fs.size is not None and self.size != fs.size: raise ISError(u"Downloading image %s failed: Invalid announced size" % self.name) # open destination fd = open(self.filename, "wb") fs.consume(fd) fs.close() fd.close() if self.size != fs.consumed_size: raise ISError(u"Download image %s failed: Invalid size" % self.name) if self.md5 != fs.md5: raise ISError(u"Download image %s failed: Invalid MD5" % self.name) if payload: for payname in self.payload: arrow(u"Downloading payload %s in %s" % (payname, directory)) self.payload[payname].info self.payload[payname].download(directory, force=force)
def download(self, dest, force=False): ''' Download payload in directory ''' # if dest is a directory try to create file inside if isdir(dest): dest = join(dest, self.filename) # try to create leading directories elif not exists(dirname(dest)): mkdir(dirname(dest)) # check validity of dest if exists(dest): if isdir(dest): raise ISError(u"Destination %s is a directory" % dest) if not force: raise ISError(u"File %s already exists" % dest) # open remote file debug(u"Downloading payload %s from %s" % (self.filename, self.path)) fs = PipeFile(self.path, progressbar=True) # check if announced file size is good if fs.size is not None and self.size != fs.size: raise ISError(u"Downloading payload %s failed: Invalid announced size" % self.name) fd = open(dest, "wb") fs.consume(fd) # closing fo fs.close() fd.close() # checking download size if self.size != fs.read_size: raise ISError(u"Downloading payload %s failed: Invalid size" % self.name) if self.md5 != fs.md5: raise ISError(u"Downloading payload %s failed: Invalid MD5" % self.name)
def checksummize(self): ''' Fill missing md5/size about payload ''' fileobj = PipeFile(self.path, "r") fileobj.consume() fileobj.close() if self._size is None: self._size = fileobj.read_size if self._md5 is None: self._md5 = fileobj.md5
def extract_file(self, dest, force=False): ''' Copy a payload directly to a file Check md5 on the fly ''' # if dest is a directory try to create file inside if isdir(dest): dest = join(dest, self.name) # try to create leading directories elif not exists(dirname(dest)): mkdir(dirname(dest)) # check validity of dest if exists(dest): if isdir(dest): raise ISError(u"Destination %s is a directory" % dest) if not force: raise ISError(u"File %s already exists" % dest) # get compressor argv (first to escape file creation if not found) a_comp = get_compressor_path(self.compressor, compress=False) # try to open payload file (source) try: f_src = PipeFile(self.path, "r", progressbar=True) except Exception as e: raise ISError(u"Unable to open payload file %s" % self.path, e) # check if announced file size is good if f_src.size is not None and self.size != f_src.size: raise ISError(u"Invalid announced size on %s" % self.path) # opening destination try: f_dst = open(dest, "wb") except Exception as e: raise ISError(u"Unable to open destination file %s" % dest, e) # run compressor process p_comp = Popen(a_comp, shell=False, close_fds=True, stdin=PIPE, stdout=f_dst) # close destination file f_dst.close() # push data into compressor f_src.consume(p_comp.stdin) # closing source fo f_src.close() # checking download size if self.size != f_src.read_size: raise ISError("Invalid size") # checking downloaded md5 if self.md5 != f_src.md5: raise ISError("Invalid MD5") # close compressor pipe p_comp.stdin.close() # check compressor return 0 if p_comp.wait() != 0: raise ISError(u"Compressor %s return is not zero" % a_comp[0]) # settings file orginal rights chrights(dest, self.uid, self.gid, self.mode, self.mtime)
def extract_tar(self, dest, force=False, filelist=None): ''' Extract a payload which is a tarball. This is used mainly to extract payload from a directory ''' # check validity of dest if exists(dest): if not isdir(dest): raise ISError(u"Destination %s is not a directory" % dest) if not force and len(listdir(dest)) > 0: raise ISError(u"Directory %s is not empty (need force)" % dest) else: mkdir(dest) # try to open payload file try: fo = PipeFile(self.path, progressbar=True) except Exception as e: raise ISError(u"Unable to open %s" % self.path) # check if announced file size is good if fo.size is not None and self.size != fo.size: raise ISError(u"Invalid announced size on %s" % self.path) # get compressor argv (first to escape file creation if not found) a_comp = get_compressor_path(self.compressor, compress=False) a_tar = ["tar", "--extract", "--numeric-owner", "--ignore-zeros", "--preserve-permissions", "--directory", dest] # add optionnal selected filename for decompression if filelist is not None: a_tar += filelist p_tar = Popen(a_tar, shell=False, close_fds=True, stdin=PIPE) p_comp = Popen(a_comp, shell=False, close_fds=True, stdin=PIPE, stdout=p_tar.stdin) # close tar fd p_tar.stdin.close() # push data into compressor fo.consume(p_comp.stdin) # close source fd fo.close() # checking downloaded size if self.size != fo.read_size: raise ISError("Invalid size") # checking downloaded md5 if self.md5 != fo.md5: raise ISError("Invalid MD5") # close compressor pipe p_comp.stdin.close() # check compressor return 0 if p_comp.wait() != 0: raise ISError(u"Compressor %s return is not zero" % a_comp[0]) # check tar return 0 if p_tar.wait() != 0: raise ISError("Tar return is not zero")
def check(self): ''' Check that path correspond to current md5 and size ''' if self._size is None or self._md5 is None: debug("Check is called on payload with nothing to check") return True fileobj = PipeFile(self.path, "r") fileobj.consume() fileobj.close() if self._size != fileobj.read_size: raise ISError(u"Invalid size of payload %s" % self.name) if self._md5 != fileobj.md5: raise ISError(u"Invalid MD5 of payload %s" % self._md5)
def generate_json_description(self): ''' Generate a JSON description file ''' arrow("Generating JSON description") arrowlevel(1) # copy description desc = self.description.copy() # only store compressor patterns desc["compressor"] = desc["compressor"]["patterns"] # timestamp image arrow("Timestamping") desc["date"] = int(time()) # watermark desc["is_build_version"] = VERSION # append payload infos arrow("Checksumming payloads") desc["payload"] = {} for payload_name in self.select_payloads(): arrow(payload_name, 1) # getting payload info payload_desc = self.describe_payload(payload_name) # compute md5 and size fileobj = PipeFile(payload_desc["link_path"], "r") fileobj.consume() fileobj.close() # create payload entry desc["payload"][payload_name] = { "md5": fileobj.md5, "size": fileobj.size, "isdir": payload_desc["isdir"], "uid": payload_desc["uid"], "gid": payload_desc["gid"], "mode": payload_desc["mode"], "mtime": payload_desc["mtime"], "compressor": payload_desc["compressor"] } arrowlevel(-1) # check md5 are uniq md5s = [v["md5"] for v in desc["payload"].values()] if len(md5s) != len(set(md5s)): raise ISError("Two payloads cannot have the same md5") # serialize return dumps(desc)
def check(self, message="Check MD5"): ''' Check md5 and size of tarballs are correct Download tarball from path and compare the loaded md5 and remote ''' arrow(message) arrowlevel(1) # check image fo = PipeFile(self.path, "r") fo.consume() fo.close() if self.size != fo.read_size: raise ISError(u"Invalid size of image %s" % self.name) if self.md5 != fo.md5: raise ISError(u"Invalid MD5 of image %s" % self.name) # check payloads for pay_name, pay_obj in self.payload.items(): arrow(pay_name) pay_obj.check() arrowlevel(-1)
def add(self, image, delete=False): ''' Add a packaged image to repository if delete is true, remove original files ''' # check local repository if not self.local: raise ISError(u"Repository addition must be local") # cannot add already existant image if self.has(image.name, image.version): raise ISError(u"Image already in database, delete first!") # adding file to repository arrow("Copying images and payload") for obj in [ image ] + image.payload.values(): dest = join(self.config.path, obj.md5) basesrc = basename(obj.path) if exists(dest): arrow(u"Skipping %s: already exists" % basesrc, 1) else: arrow(u"Adding %s (%s)" % (basesrc, obj.md5), 1) dfo = open(dest, "wb") sfo = PipeFile(obj.path, "r", progressbar=True) sfo.consume(dfo) sfo.close() dfo.close() chrights(dest, self.config.uid, self.config.gid, self.config.fmod) # copy is done. create a image inside repo r_image = PackageImage(join(self.config.path, image.md5), md5name=True) # checking must be done with original md5 r_image.md5 = image.md5 # checking image and payload after copy r_image.check("Check image and payload") self._add(image) # removing orginal files if delete: arrow("Removing original files") for obj in [ image ] + image.payload.values(): arrow(basename(obj.path), 1) unlink(obj.path)
def __init__(self, path, fileobj=None, md5name=False): ''' Initialize a package image fileobj must be a seekable fileobj ''' Image.__init__(self) self.path = abspath(path) self.base_path = dirname(self.path) # tarball are named by md5 and not by real name self.md5name = md5name try: if fileobj is None: fileobj = PipeFile(self.path, "r") else: fileobj = PipeFile(mode="r", fileobj=fileobj) memfile = StringIO() fileobj.consume(memfile) # close source fileobj.close() # get downloaded size and md5 self.size = fileobj.read_size self.md5 = fileobj.md5 memfile.seek(0) self._tarball = Tarball.open(fileobj=memfile, mode='r:gz') except Exception as e: raise ISError(u"Unable to open image %s" % path, e) self._metadata = self.read_metadata() # print info arrow(u"Image %s v%s loaded" % (self.name, self.version)) arrow(u"Author: %s" % self.author, 1) arrow(u"Date: %s" % time_rfc2822(self.date), 1) # build payloads info self.payload = {} for pname, pval in self._metadata["payload"].items(): pfilename = u"%s-%s%s" % (self.filename[:-len(Image.extension)], pname, Payload.extension) if self.md5name: ppath = join(self.base_path, self._metadata["payload"][pname]["md5"]) else: ppath = join(self.base_path, pfilename) self.payload[pname] = Payload(pname, pfilename, ppath, **pval)
def _cachify(self, config, temp=False, nosync=False): ''' Return a config of a cached repository from an orignal config file :param config: repository configuration :param temp: repository db should be stored in a temporary location :param nosync: if a cache exists, don't try to update it ''' # if cache is disable => temp =True if self.cache_path is None: temp = True try: original_dbpath = config.dbpath if temp and nosync: raise ISError("sync is disabled") elif temp: # this is a temporary cached repository tempfd, config.dbpath = tempfile.mkstemp() os.close(tempfd) self.tempfiles.append(config.dbpath) else: config.dbpath = os.path.join(self.cache_path, config.name) if not nosync: # Open remote database rdb = PipeFile(original_dbpath, timeout=self.timeout) # get remote last modification if rdb.mtime is None: # We doesn't have modification time, we use the last file try: rlast = int(PipeFile(config.lastpath, mode='r', timeout=self.timeout).read().strip()) except ISError: rlast = -1 else: rlast = rdb.mtime # get local last value if os.path.exists(config.dbpath): llast = int(os.stat(config.dbpath).st_mtime) else: llast = -2 # if repo is out of date, download it if rlast != llast: try: arrow(u"Downloading %s" % original_dbpath) rdb.progressbar = True ldb = open(config.dbpath, "wb") rdb.consume(ldb) ldb.close() rdb.close() istools.chrights(config.dbpath, uid=config.uid, gid=config.gid, mode=config.fmod, mtime=rlast) except: if os.path.exists(config.dbpath): os.unlink(config.dbpath) raise except ISError as e : # if something append bad during caching, we mark repo as offline debug(u"Unable to cache repository %s: %s" % (config.name, e)) config.offline = True return self.factory.create(config)