def main(argv=None): if argv is None: argv = sys.argv arguments = [('h', 'help', 'Export::Options::Help'), ('c', 'copy', 'Export::Options::Copy'), ('d', 'directory', 'Export::Options::Directory', 'HasArg'), ('r', 'relative', 'Export::Options::Relative'), ('s', 'suite', 'Export::Options::Suite', 'HasArg')] cnf = Config() apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) options = cnf.subtree('Export::Options') if 'Help' in options or 'Suite' not in options: usage() sys.exit(0) session = DBConn().session() suite = session.query(Suite).filter_by(suite_name=options['Suite']).first() if suite is None: print("Unknown suite '{0}'".format(options['Suite'])) sys.exit(1) directory = options.get('Directory') if not directory: print("No target directory.") sys.exit(1) symlink = 'Copy' not in options relative = 'Relative' in options if relative and not symlink: print("E: --relative and --copy cannot be used together.") sys.exit(1) binaries = suite.binaries sources = suite.sources files = [] files.extend([b.poolfile for b in binaries]) for s in sources: files.extend([ds.poolfile for ds in s.srcfiles]) with FilesystemTransaction() as fs: for f in files: af = session.query(ArchiveFile) \ .join(ArchiveFile.component).join(ArchiveFile.file) \ .filter(ArchiveFile.archive == suite.archive) \ .filter(ArchiveFile.file == f).first() src = af.path if relative: src = os.path.relpath(src, directory) dst = os.path.join(directory, f.basename) if not os.path.exists(dst): fs.copy(src, dst, symlink=symlink) fs.commit()
def test_unlink_and_commit(self): with TemporaryDirectory() as t: self._write_to_a(t) a = t.filename('a') with FilesystemTransaction() as fs: self.assert_(os.path.exists(a)) fs.unlink(a) self.assert_(not os.path.exists(a)) self.assert_(not os.path.exists(a))
def test_copy_existing_and_commit(self): with TemporaryDirectory() as t: self._write_to_a(t) with FilesystemTransaction() as fs: self._copy_a_b(t, fs) self.assert_(os.path.exists(t.filename('a'))) self.assert_(os.path.exists(t.filename('b'))) self.assert_(os.path.exists(t.filename('a'))) self.assert_(os.path.exists(t.filename('b')))
def test_unlink_and_rollback(self): with TemporaryDirectory() as t: self._write_to_a(t) a = t.filename('a') class TestException(Exception): pass try: with FilesystemTransaction() as fs: self.assert_(os.path.exists(a)) fs.unlink(a) self.assert_(not os.path.exists(a)) raise TestException() except TestException: pass self.assert_(os.path.exists(a))
def test_copy_existing_and_rollback(self): with TemporaryDirectory() as t: self._write_to_a(t) class TestException(Exception): pass try: with FilesystemTransaction() as fs: self._copy_a_b(t, fs) self.assert_(os.path.exists(t.filename('a'))) self.assert_(os.path.exists(t.filename('b'))) raise TestException() except TestException: pass self.assert_(os.path.exists(t.filename('a'))) self.assert_(not os.path.exists(t.filename('b')))
def main(argv=None): if argv is None: argv = sys.argv arguments = [('h', 'help', 'Process-Commands::Options::Help'), ('d', 'directory', 'Process-Commands::Options::Directory', 'HasArg')] cnf = Config() cnf['Process-Commands::Options::Dummy'] = '' filenames = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) options = cnf.subtree('Process-Commands::Options') if 'Help' in options or (len(filenames) == 0 and 'Directory' not in options): usage() sys.exit(0) log = Logger('command') now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) rejectdir = cnf['Dir::Reject'] if len(filenames) == 0: cdir = options['Directory'] filenames = [ os.path.join(cdir, fn) for fn in os.listdir(cdir) if fn.endswith('.dak-commands') ] for fn in filenames: basename = os.path.basename(fn) if not fn.endswith('.dak-commands'): log.log(['unexpected filename', basename]) continue with open(fn, 'r') as fh: data = fh.read() try: command = CommandFile(basename, data, log) command.evaluate() except CommandError as e: created = os.stat(fn).st_mtime now = time.time() too_new = (now - created < int(cnf.get('Dinstall::SkipTime', '60'))) if too_new: log.log(['skipped (too new)']) continue log.log(['reject', basename, e]) except Exception as e: log.log_traceback('Exception while processing %s:' % (basename), e) dst = find_next_free(os.path.join(rejectdir, basename)) else: log.log(['done', basename]) dst = find_next_free(os.path.join(donedir, basename)) with FilesystemTransaction() as fs: fs.unlink(fn) fs.create(dst, mode=0o644).write(data) fs.commit() log.close()
def copy(): with TemporaryDirectory() as t: with FilesystemTransaction() as fs: self._copy_a_b(t, fs)
def __init__(self): self.fs = FilesystemTransaction() self.session = DBConn().session()
class ArchiveTransaction(object): """manipulate the archive in a transaction """ def __init__(self): self.fs = FilesystemTransaction() self.session = DBConn().session() def get_file(self, hashed_file, source_name, check_hashes=True): """Look for file C{hashed_file} in database @type hashed_file: L{daklib.upload.HashedFile} @param hashed_file: file to look for in the database @type source_name: str @param source_name: source package name @type check_hashes: bool @param check_hashes: check size and hashes match @raise KeyError: file was not found in the database @raise HashMismatchException: hash mismatch @rtype: L{daklib.dbconn.PoolFile} @return: database entry for the file """ poolname = os.path.join(utils.poolify(source_name), hashed_file.filename) try: poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one() if check_hashes and (poolfile.filesize != hashed_file.size or poolfile.md5sum != hashed_file.md5sum or poolfile.sha1sum != hashed_file.sha1sum or poolfile.sha256sum != hashed_file.sha256sum): raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename)) return poolfile except NoResultFound: raise KeyError('{0} not found in database.'.format(poolname)) def _install_file(self, directory, hashed_file, archive, component, source_name): """Install a file Will not give an error when the file is already present. @rtype: L{daklib.dbconn.PoolFile} @return: database object for the new file """ session = self.session poolname = os.path.join(utils.poolify(source_name), hashed_file.filename) try: poolfile = self.get_file(hashed_file, source_name) except KeyError: poolfile = PoolFile(filename=poolname, filesize=hashed_file.size) poolfile.md5sum = hashed_file.md5sum poolfile.sha1sum = hashed_file.sha1sum poolfile.sha256sum = hashed_file.sha256sum session.add(poolfile) session.flush() try: session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one() except NoResultFound: archive_file = ArchiveFile(archive, component, poolfile) session.add(archive_file) session.flush() path = os.path.join(archive.path, 'pool', component.component_name, poolname) hashed_file_path = os.path.join(directory, hashed_file.filename) self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode) return poolfile def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None): """Install a binary package @type directory: str @param directory: directory the binary package is located in @type binary: L{daklib.upload.Binary} @param binary: binary package to install @type suite: L{daklib.dbconn.Suite} @param suite: target suite @type component: L{daklib.dbconn.Component} @param component: target component @type allow_tainted: bool @param allow_tainted: allow to copy additional files from tainted archives @type fingerprint: L{daklib.dbconn.Fingerprint} @param fingerprint: optional fingerprint @type source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True} @param source_suites: suites to copy the source from if they are not in C{suite} or C{True} to allow copying from any suite. @type extra_source_archives: list of L{daklib.dbconn.Archive} @param extra_source_archives: extra archives to copy Built-Using sources from @rtype: L{daklib.dbconn.DBBinary} @return: databse object for the new package """ session = self.session control = binary.control maintainer = get_or_set_maintainer(control['Maintainer'], session) architecture = get_architecture(control['Architecture'], session) (source_name, source_version) = binary.source source_query = session.query(DBSource).filter_by(source=source_name, version=source_version) source = source_query.filter(DBSource.suites.contains(suite)).first() if source is None: if source_suites != True: source_query = source_query.join(DBSource.suites) \ .filter(Suite.suite_id == source_suites.c.id) source = source_query.first() if source is None: raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name)) self.copy_source(source, suite, component) db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name) unique = dict( package=control['Package'], version=control['Version'], architecture=architecture, ) rest = dict( source=source, maintainer=maintainer, poolfile=db_file, binarytype=binary.type, fingerprint=fingerprint, ) try: db_binary = session.query(DBBinary).filter_by(**unique).one() for key, value in rest.iteritems(): if getattr(db_binary, key) != value: raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename)) except NoResultFound: db_binary = DBBinary(**unique) for key, value in rest.iteritems(): setattr(db_binary, key, value) session.add(db_binary) session.flush() import_metadata_into_db(db_binary, session) self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives) if suite not in db_binary.suites: db_binary.suites.append(suite) session.flush() return db_binary def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None): """ensure source exists in the given archive This is intended to be used to check that Built-Using sources exist. @type filename: str @param filename: filename to use in error messages @type source: L{daklib.dbconn.DBSource} @param source: source to look for @type archive: L{daklib.dbconn.Archive} @param archive: archive to look in @type extra_archives: list of L{daklib.dbconn.Archive} @param extra_archives: list of archives to copy the source package from if it is not yet present in C{archive} """ session = self.session db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first() if db_file is not None: return True # Try to copy file from one extra archive if extra_archives is None: extra_archives = [] db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first() if db_file is None: raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name)) source_archive = db_file.archive for dsc_file in source.srcfiles: af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one() # We were given an explicit list of archives so it is okay to copy from tainted archives. self._copy_file(af.file, archive, db_file.component, allow_tainted=True) def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None): """Add Built-Using sources to C{db_binary.extra_sources} """ session = self.session built_using = control.get('Built-Using', None) if built_using is not None: for dep in apt_pkg.parse_depends(built_using): assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field' bu_source_name, bu_source_version, comp = dep[0] assert comp == '=', 'Built-Using must contain strict dependencies' bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first() if bu_source is None: raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version)) self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives) db_binary.extra_sources.append(bu_source) def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None): """Install a source package @type directory: str @param directory: directory the source package is located in @type source: L{daklib.upload.Source} @param source: source package to install @type suite: L{daklib.dbconn.Suite} @param suite: target suite @type component: L{daklib.dbconn.Component} @param component: target component @type changed_by: L{daklib.dbconn.Maintainer} @param changed_by: person who prepared this version of the package @type allow_tainted: bool @param allow_tainted: allow to copy additional files from tainted archives @type fingerprint: L{daklib.dbconn.Fingerprint} @param fingerprint: optional fingerprint @rtype: L{daklib.dbconn.DBSource} @return: database object for the new source """ session = self.session archive = suite.archive control = source.dsc maintainer = get_or_set_maintainer(control['Maintainer'], session) source_name = control['Source'] ### Add source package to database # We need to install the .dsc first as the DBSource object refers to it. db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name) unique = dict( source=source_name, version=control['Version'], ) rest = dict( maintainer=maintainer, changedby=changed_by, #install_date=datetime.now().date(), poolfile=db_file_dsc, fingerprint=fingerprint, dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'), ) created = False try: db_source = session.query(DBSource).filter_by(**unique).one() for key, value in rest.iteritems(): if getattr(db_source, key) != value: raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename)) except NoResultFound: created = True db_source = DBSource(**unique) for key, value in rest.iteritems(): setattr(db_source, key, value) # XXX: set as default in postgres? db_source.install_date = datetime.now().date() session.add(db_source) session.flush() # Add .dsc file. Other files will be added later. db_dsc_file = DSCFile() db_dsc_file.source = db_source db_dsc_file.poolfile = db_file_dsc session.add(db_dsc_file) session.flush() if suite in db_source.suites: return db_source db_source.suites.append(suite) if not created: for f in db_source.srcfiles: self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted) return db_source ### Now add remaining files and copy them to the archive. for hashed_file in source.files.itervalues(): hashed_file_path = os.path.join(directory, hashed_file.filename) if os.path.exists(hashed_file_path): db_file = self._install_file(directory, hashed_file, archive, component, source_name) session.add(db_file) else: db_file = self.get_file(hashed_file, source_name) self._copy_file(db_file, archive, component, allow_tainted=allow_tainted) db_dsc_file = DSCFile() db_dsc_file.source = db_source db_dsc_file.poolfile = db_file session.add(db_dsc_file) session.flush() # Importing is safe as we only arrive here when we did not find the source already installed earlier. import_metadata_into_db(db_source, session) # Uploaders are the maintainer and co-maintainers from the Uploaders field db_source.uploaders.append(maintainer) if 'Uploaders' in control: from daklib.textutils import split_uploaders for u in split_uploaders(control['Uploaders']): db_source.uploaders.append(get_or_set_maintainer(u, session)) session.flush() return db_source def _copy_file(self, db_file, archive, component, allow_tainted=False): """Copy a file to the given archive and component @type db_file: L{daklib.dbconn.PoolFile} @param db_file: file to copy @type archive: L{daklib.dbconn.Archive} @param archive: target archive @type component: L{daklib.dbconn.Archive} @param component: target component @type allow_tainted: bool @param allow_tainted: allow to copy from tainted archives (such as NEW) """ session = self.session if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None: query = session.query(ArchiveFile).filter_by(file=db_file) if not allow_tainted: query = query.join(Archive).filter(Archive.tainted == False) source_af = query.first() if source_af is None: raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename)) target_af = ArchiveFile(archive, component, db_file) session.add(target_af) session.flush() self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode) def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None): """Copy a binary package to the given suite and component @type db_binary: L{daklib.dbconn.DBBinary} @param db_binary: binary to copy @type suite: L{daklib.dbconn.Suite} @param suite: target suite @type component: L{daklib.dbconn.Component} @param component: target component @type allow_tainted: bool @param allow_tainted: allow to copy from tainted archives (such as NEW) @type extra_archives: list of L{daklib.dbconn.Archive} @param extra_archives: extra archives to copy Built-Using sources from """ session = self.session archive = suite.archive if archive.tainted: allow_tainted = True filename = db_binary.poolfile.filename # make sure source is present in target archive db_source = db_binary.source if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None: raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name)) # make sure built-using packages are present in target archive for db_source in db_binary.extra_sources: self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives) # copy binary db_file = db_binary.poolfile self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted) if suite not in db_binary.suites: db_binary.suites.append(suite) self.session.flush() def copy_source(self, db_source, suite, component, allow_tainted=False): """Copy a source package to the given suite and component @type db_source: L{daklib.dbconn.DBSource} @param db_source: source to copy @type suite: L{daklib.dbconn.Suite} @param suite: target suite @type component: L{daklib.dbconn.Component} @param component: target component @type allow_tainted: bool @param allow_tainted: allow to copy from tainted archives (such as NEW) """ archive = suite.archive if archive.tainted: allow_tainted = True for db_dsc_file in db_source.srcfiles: self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted) if suite not in db_source.suites: db_source.suites.append(suite) self.session.flush() def remove_file(self, db_file, archive, component): """Remove a file from a given archive and component @type db_file: L{daklib.dbconn.PoolFile} @param db_file: file to remove @type archive: L{daklib.dbconn.Archive} @param archive: archive to remove the file from @type component: L{daklib.dbconn.Component} @param component: component to remove the file from """ af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component) self.fs.unlink(af.path) self.session.delete(af) def remove_binary(self, binary, suite): """Remove a binary from a given suite and component @type binary: L{daklib.dbconn.DBBinary} @param binary: binary to remove @type suite: L{daklib.dbconn.Suite} @param suite: suite to remove the package from """ binary.suites.remove(suite) self.session.flush() def remove_source(self, source, suite): """Remove a source from a given suite and component @type source: L{daklib.dbconn.DBSource} @param source: source to remove @type suite: L{daklib.dbconn.Suite} @param suite: suite to remove the package from @raise ArchiveException: source package is still referenced by other binaries in the suite """ session = self.session query = session.query(DBBinary).filter_by(source=source) \ .filter(DBBinary.suites.contains(suite)) if query.first() is not None: raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name)) source.suites.remove(suite) session.flush() def commit(self): """commit changes""" try: self.session.commit() self.fs.commit() finally: self.session.rollback() self.fs.rollback() def rollback(self): """rollback changes""" self.session.rollback() self.fs.rollback() def __enter__(self): return self def __exit__(self, type, value, traceback): if type is None: self.commit() else: self.rollback() return None