def init(self): ''' Initialize an empty base repository ''' config = self.config # check local repository if not self.local: raise ISError(u"Repository creation must be local") # create base directories arrow("Creating base directories") arrowlevel(1) # creating local directory try: if os.path.exists(config.path): arrow(u"%s already exists" % config.path) else: istools.mkdir(config.path, config.uid, config.gid, config.dmod) arrow(u"%s directory created" % config.path) except Exception as e: raise ISError(u"Unable to create directory %s" % config.path, e) arrowlevel(-1) # create database d = Database.create(config.dbpath) istools.chrights(config.dbpath, uid=config.uid, gid=config.gid, mode=config.fmod) # load database self.db = Database(config.dbpath) # mark repo as not offline self.config.offline = False # create/update last file self.update_last()
class Repository(object): ''' Repository class ''' @staticmethod def is_repository_name(name): return re.match("^[-_\w]+$", name) is not None @staticmethod def check_repository_name(name): ''' Raise exception is repository name is invalid ''' if not Repository.is_repository_name(name): raise ISError(u"Invalid repository name %s" % name) return name @staticmethod def split_image_path(path): ''' Split an image path (repo/image:version) in a tuple (repo, image, version) ''' x = re.match(u"^(?:([^/:]+)/)?([^/:]+)?(?::v?([^/:]+)?)?$", path) if x is None: raise ISError(u"invalid image path: %s" % path) return x.group(1, 2, 3) @staticmethod def split_repository_list(repolist, filter=None): ''' Return a list of repository from a comma/spaces separated names of repo ''' if filter is None: filter = Repository.is_repository_name return [r for r in re.split("[ ,\n\t\v]+", repolist) if filter(r)] @classmethod def diff(cls, repo1, repo2): ''' Compute a diff between two repositories ''' arrow(u"Diff between repositories #y#%s#R# and #g#%s#R#" % (repo1.config.name, repo2.config.name)) # Get info from databases i_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask( "SELECT md5, name, version FROM image").fetchall()) i_set1 = set(i_dict1.keys()) i_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask( "SELECT md5, name, version FROM image").fetchall()) i_set2 = set(i_dict2.keys()) p_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask( "SELECT md5, name FROM payload").fetchall()) p_set1 = set(p_dict1.keys()) p_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask( "SELECT md5, name FROM payload").fetchall()) p_set2 = set(p_dict2.keys()) # computing diff i_only1 = i_set1 - i_set2 i_only2 = i_set2 - i_set1 p_only1 = p_set1 - p_set2 p_only2 = p_set2 - p_set1 # printing functions pimg = lambda r,c,m,d,: out("#%s#Image only in repository %s: %s v%s (%s)#R#" % (c, r.config.name, d[m][0], d[m][1], m)) ppay = lambda r,c,m,d,: out("#%s#Payload only in repository %s: %s (%s)#R#" % (c, r.config.name, d[m][0], m)) # printing image diff for md5 in i_only1: pimg(repo1, "y", md5, i_dict1) for md5 in p_only1: ppay(repo1, "y", md5, p_dict1) for md5 in i_only2: pimg(repo2, "g", md5, i_dict2) for md5 in p_only2: ppay(repo2, "g", md5, p_dict2) def __init__(self, config, db=None): self.config = config self.local = istools.isfile(self.config.path) self.db = db def __getattribute__(self, name): ''' Raise an error if repository is unavailable Unavailable can be caused because db is not accessible or because repository is not initialized ''' config = object.__getattribute__(self, "config") # config, init, local and upgrade are always accessible if name in ("init", "config", "local", "upgrade"): return object.__getattribute__(self, name) # if no db (not init or not accessible) raise error if config.offline: raise ISError(u"Repository %s is offline" % config.name) return object.__getattribute__(self, name) @property def version(self): ''' Return repository version ''' return self.db.version @property def uuid(self): ''' Return repository UUID ''' return self.db.ask("SELECT uuid from repository").fetchone()[0] def init(self): ''' Initialize an empty base repository ''' config = self.config # check local repository if not self.local: raise ISError(u"Repository creation must be local") # create base directories arrow("Creating base directories") arrowlevel(1) # creating local directory try: if os.path.exists(config.path): arrow(u"%s already exists" % config.path) else: istools.mkdir(config.path, config.uid, config.gid, config.dmod) arrow(u"%s directory created" % config.path) except Exception as e: raise ISError(u"Unable to create directory %s" % config.path, e) arrowlevel(-1) # create database d = Database.create(config.dbpath) istools.chrights(config.dbpath, uid=config.uid, gid=config.gid, mode=config.fmod) # load database self.db = Database(config.dbpath) # mark repo as not offline self.config.offline = False # create/update last file self.update_last() def update_last(self): ''' Update last file to current time ''' # check local repository if not self.local: raise ISError(u"Repository must be local") try: arrow("Updating last file") last_path = os.path.join(self.config.path, self.config.lastname) open(last_path, "w").write("%s\n" % int(time.time())) istools.chrights(last_path, self.config.uid, self.config.gid, self.config.fmod) except Exception as e: raise ISError(u"Update last file failed", e) def last(self, name): ''' Return last version of name in repo or None if not found ''' r = self.db.ask("SELECT version FROM image WHERE name = ?", (name,)).fetchall() # no row => no way if r is None: return None f = lambda x,y: x[0] if istools.compare_versions(x[0], y[0]) > 0 else y[0] # return last return reduce(f, r) def _add(self, image): ''' Add description to db ''' arrow("Adding metadata") self.db.begin() # insert image information arrow("Image", 1) self.db.ask("INSERT INTO image values (?,?,?,?,?,?,?,?,?)", (image.md5, image.name, image.version, image.date, image.author, image.description, image.size, image.is_min_version, image.format, )) # insert data information arrow("Payloads", 1) for name, obj in image.payload.items(): self.db.ask("INSERT INTO payload values (?,?,?,?,?)", (obj.md5, image.md5, name, obj.isdir, obj.size, )) # on commit self.db.commit() # update last file self.update_last() def add(self, image, delete=False): ''' Add a packaged image to repository if delete is true, remove original files ''' # check local repository if not self.local: raise ISError(u"Repository addition must be local") # cannot add already existant image if self.has(image.name, image.version): raise ISError(u"Image already in database, delete first!") # adding file to repository arrow("Copying images and payload") for obj in [ image ] + image.payload.values(): dest = os.path.join(self.config.path, obj.md5) basesrc = os.path.basename(obj.path) if os.path.exists(dest): arrow(u"Skipping %s: already exists" % basesrc, 1) else: arrow(u"Adding %s (%s)" % (basesrc, obj.md5), 1) dfo = open(dest, "wb") sfo = PipeFile(obj.path, "r", progressbar=True) sfo.consume(dfo) sfo.close() dfo.close() istools.chrights(dest, self.config.uid, self.config.gid, self.config.fmod) # copy is done. create a image inside repo r_image = PackageImage(os.path.join(self.config.path, image.md5), md5name=True) # checking must be done with original md5 r_image.md5 = image.md5 # checking image and payload after copy r_image.check("Check image and payload") self._add(image) # removing orginal files if delete: arrow("Removing original files") for obj in [ image ] + image.payload.values(): arrow(os.path.basename(obj.path), 1) os.unlink(obj.path) def getallmd5(self): ''' Get list of all md5 in DB ''' res = self.db.ask("SELECT md5 FROM image UNION SELECT md5 FROM payload").fetchall() return [ md5[0] for md5 in res ] def check(self): ''' Check repository for unreferenced and missing files ''' # Check if the repo is local if not self.local: raise ISError(u"Repository must be local") local_files = set(os.listdir(self.config.path)) local_files.remove(self.config.dbname) local_files.remove(self.config.lastname) db_files = set(self.getallmd5()) # check missing files arrow("Checking missing files") missing_files = db_files - local_files if len(missing_files) > 0: out(os.linesep.join(missing_files)) # check unreferenced files arrow("Checking unreferenced files") unref_files = local_files - db_files if len(unref_files) > 0: out(os.linesep.join(unref_files)) # check corruption of local files arrow("Checking corrupted files") for f in local_files: fo = PipeFile(os.path.join(self.config.path, f)) fo.consume() fo.close() if fo.md5 != f: out(f) def clean(self, force=False): ''' Clean the repository's content ''' # Check if the repo is local if not self.local: raise ISError(u"Repository must be local") allmd5 = set(self.getallmd5()) repofiles = set(os.listdir(self.config.path)) - set([self.config.dbname, self.config.lastname]) dirtyfiles = repofiles - allmd5 if len(dirtyfiles) > 0: # print dirty files arrow("Dirty files:") for f in dirtyfiles: arrow(f, 1) # ask confirmation if not force and not confirm("Remove dirty files? (yes) "): raise ISError(u"Aborted!") # start cleaning arrow("Cleaning") for f in dirtyfiles: p = os.path.join(self.config.path, f) arrow(u"Removing %s" % p, 1) try: if os.path.isdir(p): os.rmdir(p) else: os.unlink(p) except: warn(u"Removing %s failed" % p) else: arrow("Nothing to clean") def delete(self, name, version, payloads=True): ''' Delete an image from repository ''' # check local repository if not self.local: raise ISError(u"Repository deletion must be local") # get md5 of files related to images (exception is raised if not exists md5s = self.getmd5(name, version) # cleaning db (must be done before cleaning) arrow("Cleaning database") arrow("Remove payloads from database", 1) self.db.begin() for md5 in md5s[1:]: self.db.ask("DELETE FROM payload WHERE md5 = ? AND image_md5 = ?", (md5, md5s[0])).fetchone() arrow("Remove image from database", 1) self.db.ask("DELETE FROM image WHERE md5 = ?", (md5s[0],)).fetchone() self.db.commit() # Removing files arrow("Removing files from pool") # if asked don't remove payloads if not payloads: md5s = [ md5s[0] ] arrowlevel(1) for md5 in md5s: self._remove_file(md5) arrowlevel(-1) # update last file self.update_last() def images(self): ''' Return a dict of information on images ''' db_images = self.db.ask("SELECT md5, name, version, date, author, \ description, size, is_min_version, format \ FROM image ORDER BY name, version").fetchall() images = [] field = ("md5", "name", "version", "date", "author", "description", "size", "is_min_version", "format") for info in db_images: d = dict(zip(field, info)) d["repo"] = self.config.name d["url"] = os.path.join(self.config.path, d["md5"]) images.append(d) return images def payloads(self): ''' Return a dict of information on payloads ''' db_payloads = self.db.ask("SELECT payload.md5,payload.size,payload.isdir,image.name,image.version,payload.name FROM payload inner join image on payload.image_md5 = image.md5").fetchall() res = {} for payload in db_payloads: md5 = payload[0] # create entry if not exists if md5 not in res: res[md5] = {"size": payload[1], "isdir": payload[2], "images": {}} # add image to list imgpath = u"%s/%s:%s" % (self.config.name, payload[3], payload[4]) res[md5]["images"][imgpath] = {"repo": self.config.name, "imgname": payload[3], "imgver": payload[4], "payname": payload[5]} return res def search(self, pattern): ''' Search pattern in a repository ''' images = self.db.ask("SELECT name, version, author, description\ FROM image\ WHERE name LIKE ? OR\ description LIKE ? OR\ author LIKE ?", tuple( [u"%%%s%%" % pattern ] * 3) ).fetchall() for name, version, author, description in images: arrow(u"%s v%s" % (name, version), 1) out(u" #yellow#Author:#reset# %s" % author) out(u" #yellow#Description:#reset# %s" % description) def _remove_file(self, filename): ''' Remove a filename from pool. Check if it's not needed by db before ''' # check existance in table image have = False for table in ("image", "payload"): have = have or self.db.ask(u"SELECT md5 FROM %s WHERE md5 = ? LIMIT 1" % table, (filename,)).fetchone() is not None # if no reference, delete! if not have: arrow(u"%s, deleted" % filename) os.unlink(os.path.join(self.config.path, filename)) else: arrow(u"%s, skipped" % filename) def has(self, name, version): ''' Return the existance of a package ''' return self.db.ask("SELECT name,version FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() is not None def get(self, name, version=None): ''' Return an image from a name and version ''' # is no version take the last if version is None: version = self.last(name) if version is None: raise ISError(u"Unable to find image %s in %s" % (name, self.config.name)) # get file md5 from db r = self.db.ask("select md5 from image where name = ? and version = ? limit 1", (name, version)).fetchone() if r is None: raise ISError(u"Unable to find image %s v%s in %s" % (name, version, self.config.name)) path = os.path.join(self.config.path, r[0]) # getting the file arrow(u"Loading image %s v%s from repository %s" % (name, version, self.config.name)) memfile = cStringIO.StringIO() try: fo = PipeFile(path, "r") fo.consume(memfile) fo.close() except Exception as e: raise ISError(u"Loading image %s v%s failed" % (name, version), e) memfile.seek(0) pkg = PackageImage(path, fileobj=memfile, md5name=True) if pkg.md5 != r[0]: raise ISError(u"Image MD5 verification failure") return pkg def getmd5(self, name, version): ''' Return an image md5 and payload md5 from name and version. Order matter ! Image md5 will still be the first ''' # get file md5 from db a = self.db.ask("SELECT md5 FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() if a is None: raise ISError(u"No such image %s version %s" % (name, version)) b = self.db.ask("SELECT md5 FROM payload WHERE image_md5 = ?", (a[0],)).fetchall() return [ a[0] ] + [ x[0] for x in b ] def upgrade(self): if self.version == Database.version: info("Repository already up-to-date (%s)" % self.version) return else: arrow("Start repository upgrade") arrowlevel(1) # Create dummy repository tmpdir = tempfile.mkdtemp() try: repoconf = RepositoryConfig("tmp_migrate_repo", path=tmpdir) dstrepo = Repository(repoconf) # Symlink content from repository into dummy repo for file in os.listdir(self.config.path): os.symlink(os.path.join(self.config.path, file), os.path.join(tmpdir, file)) os.unlink(repoconf.dbpath) os.unlink(repoconf.lastpath) old_verbosity = installsystems.verbosity arrow("Initialize new database") # Disable unwanted message during upgrade installsystems.verbosity = 0 dstrepo.init() # Restore verbosity installsystems.verbosity = old_verbosity md5s = self.db.ask("SELECT md5 FROM image").fetchall() # Copy images to dummy repository (fill new database) arrow("Fill database with images") arrowlevel(1) installsystems.verbosity = 0 for img in [PackageImage(os.path.join(self.config.path, md5[0]), md5name=True) for md5 in md5s]: installsystems.verbosity = old_verbosity arrow("%s v%s" % (img.name, img.version)) installsystems.verbosity = 0 dstrepo.add(img) installsystems.verbosity = old_verbosity arrowlevel(-1) arrow("Backup old database") shutil.move(self.config.dbpath, os.path.join("%s.bak" % self.config.dbpath)) # Replace old db with the new from dummy repository shutil.move(repoconf.dbpath, self.config.dbpath) self.update_last() arrowlevel(-1) arrow("Repository upgrade complete") finally: # Remove dummy repository shutil.rmtree(tmpdir) @property def motd(self): ''' Return repository message of the day ''' motd = self.db.ask("SELECT motd FROM repository").fetchone()[0] return None if len(motd) == 0 else motd def setmotd(self, value=""): ''' Set repository message of the day ''' # check local repository if not self.local: raise ISError(u"Repository must be local") arrow("Updating motd") self.db.ask("UPDATE repository SET motd = ?", (value,)) self.update_last()