def main(argv=None):
if argv is None:
argv = sys.argv
arguments = [('h', 'help', 'Export::Options::Help'),
('c', 'copy', 'Export::Options::Copy'),
('d', 'directory', 'Export::Options::Directory', 'HasArg'),
('r', 'relative', 'Export::Options::Relative'),
('s', 'suite', 'Export::Options::Suite', 'HasArg')]
cnf = Config()
apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
options = cnf.subtree('Export::Options')
if 'Help' in options or 'Suite' not in options:
usage()
sys.exit(0)
session = DBConn().session()
suite = session.query(Suite).filter_by(suite_name=options['Suite']).first()
if suite is None:
print("Unknown suite '{0}'".format(options['Suite']))
sys.exit(1)
directory = options.get('Directory')
if not directory:
print("No target directory.")
sys.exit(1)
symlink = 'Copy' not in options
relative = 'Relative' in options
if relative and not symlink:
print("E: --relative and --copy cannot be used together.")
sys.exit(1)
binaries = suite.binaries
sources = suite.sources
files = []
files.extend([b.poolfile for b in binaries])
for s in sources:
files.extend([ds.poolfile for ds in s.srcfiles])
with FilesystemTransaction() as fs:
for f in files:
af = session.query(ArchiveFile) \
.join(ArchiveFile.component).join(ArchiveFile.file) \
.filter(ArchiveFile.archive == suite.archive) \
.filter(ArchiveFile.file == f).first()
src = af.path
if relative:
src = os.path.relpath(src, directory)
dst = os.path.join(directory, f.basename)
if not os.path.exists(dst):
fs.copy(src, dst, symlink=symlink)
fs.commit()
def test_unlink_and_commit(self):
with TemporaryDirectory() as t:
self._write_to_a(t)
a = t.filename('a')
with FilesystemTransaction() as fs:
self.assert_(os.path.exists(a))
fs.unlink(a)
self.assert_(not os.path.exists(a))
self.assert_(not os.path.exists(a))
def test_copy_existing_and_commit(self):
with TemporaryDirectory() as t:
self._write_to_a(t)
with FilesystemTransaction() as fs:
self._copy_a_b(t, fs)
self.assert_(os.path.exists(t.filename('a')))
self.assert_(os.path.exists(t.filename('b')))
self.assert_(os.path.exists(t.filename('a')))
self.assert_(os.path.exists(t.filename('b')))
def test_unlink_and_rollback(self):
with TemporaryDirectory() as t:
self._write_to_a(t)
a = t.filename('a')
class TestException(Exception):
pass
try:
with FilesystemTransaction() as fs:
self.assert_(os.path.exists(a))
fs.unlink(a)
self.assert_(not os.path.exists(a))
raise TestException()
except TestException:
pass
self.assert_(os.path.exists(a))
def test_copy_existing_and_rollback(self):
with TemporaryDirectory() as t:
self._write_to_a(t)
class TestException(Exception):
pass
try:
with FilesystemTransaction() as fs:
self._copy_a_b(t, fs)
self.assert_(os.path.exists(t.filename('a')))
self.assert_(os.path.exists(t.filename('b')))
raise TestException()
except TestException:
pass
self.assert_(os.path.exists(t.filename('a')))
self.assert_(not os.path.exists(t.filename('b')))
def main(argv=None):
if argv is None:
argv = sys.argv
arguments = [('h', 'help', 'Process-Commands::Options::Help'),
('d', 'directory', 'Process-Commands::Options::Directory',
'HasArg')]
cnf = Config()
cnf['Process-Commands::Options::Dummy'] = ''
filenames = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
options = cnf.subtree('Process-Commands::Options')
if 'Help' in options or (len(filenames) == 0
and 'Directory' not in options):
usage()
sys.exit(0)
log = Logger('command')
now = datetime.datetime.now()
donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
rejectdir = cnf['Dir::Reject']
if len(filenames) == 0:
cdir = options['Directory']
filenames = [
os.path.join(cdir, fn) for fn in os.listdir(cdir)
if fn.endswith('.dak-commands')
]
for fn in filenames:
basename = os.path.basename(fn)
if not fn.endswith('.dak-commands'):
log.log(['unexpected filename', basename])
continue
with open(fn, 'r') as fh:
data = fh.read()
try:
command = CommandFile(basename, data, log)
command.evaluate()
except CommandError as e:
created = os.stat(fn).st_mtime
now = time.time()
too_new = (now - created < int(cnf.get('Dinstall::SkipTime',
'60')))
if too_new:
log.log(['skipped (too new)'])
continue
log.log(['reject', basename, e])
except Exception as e:
log.log_traceback('Exception while processing %s:' % (basename), e)
dst = find_next_free(os.path.join(rejectdir, basename))
else:
log.log(['done', basename])
dst = find_next_free(os.path.join(donedir, basename))
with FilesystemTransaction() as fs:
fs.unlink(fn)
fs.create(dst, mode=0o644).write(data)
fs.commit()
log.close()
def copy():
with TemporaryDirectory() as t:
with FilesystemTransaction() as fs:
self._copy_a_b(t, fs)
def __init__(self):
self.fs = FilesystemTransaction()
self.session = DBConn().session()
class ArchiveTransaction(object):
"""manipulate the archive in a transaction
"""
def __init__(self):
self.fs = FilesystemTransaction()
self.session = DBConn().session()
def get_file(self, hashed_file, source_name, check_hashes=True):
"""Look for file C{hashed_file} in database
@type hashed_file: L{daklib.upload.HashedFile}
@param hashed_file: file to look for in the database
@type source_name: str
@param source_name: source package name
@type check_hashes: bool
@param check_hashes: check size and hashes match
@raise KeyError: file was not found in the database
@raise HashMismatchException: hash mismatch
@rtype: L{daklib.dbconn.PoolFile}
@return: database entry for the file
"""
poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
try:
poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
if check_hashes and (poolfile.filesize != hashed_file.size
or poolfile.md5sum != hashed_file.md5sum
or poolfile.sha1sum != hashed_file.sha1sum
or poolfile.sha256sum != hashed_file.sha256sum):
raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
return poolfile
except NoResultFound:
raise KeyError('{0} not found in database.'.format(poolname))
def _install_file(self, directory, hashed_file, archive, component, source_name):
"""Install a file
Will not give an error when the file is already present.
@rtype: L{daklib.dbconn.PoolFile}
@return: database object for the new file
"""
session = self.session
poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
try:
poolfile = self.get_file(hashed_file, source_name)
except KeyError:
poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
poolfile.md5sum = hashed_file.md5sum
poolfile.sha1sum = hashed_file.sha1sum
poolfile.sha256sum = hashed_file.sha256sum
session.add(poolfile)
session.flush()
try:
session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
except NoResultFound:
archive_file = ArchiveFile(archive, component, poolfile)
session.add(archive_file)
session.flush()
path = os.path.join(archive.path, 'pool', component.component_name, poolname)
hashed_file_path = os.path.join(directory, hashed_file.filename)
self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
return poolfile
def install_binary(self, directory, binary, suite, component, allow_tainted=False, fingerprint=None, source_suites=None, extra_source_archives=None):
"""Install a binary package
@type directory: str
@param directory: directory the binary package is located in
@type binary: L{daklib.upload.Binary}
@param binary: binary package to install
@type suite: L{daklib.dbconn.Suite}
@param suite: target suite
@type component: L{daklib.dbconn.Component}
@param component: target component
@type allow_tainted: bool
@param allow_tainted: allow to copy additional files from tainted archives
@type fingerprint: L{daklib.dbconn.Fingerprint}
@param fingerprint: optional fingerprint
@type source_suites: SQLAlchemy subquery for C{daklib.dbconn.Suite} or C{True}
@param source_suites: suites to copy the source from if they are not
in C{suite} or C{True} to allow copying from any
suite.
@type extra_source_archives: list of L{daklib.dbconn.Archive}
@param extra_source_archives: extra archives to copy Built-Using sources from
@rtype: L{daklib.dbconn.DBBinary}
@return: databse object for the new package
"""
session = self.session
control = binary.control
maintainer = get_or_set_maintainer(control['Maintainer'], session)
architecture = get_architecture(control['Architecture'], session)
(source_name, source_version) = binary.source
source_query = session.query(DBSource).filter_by(source=source_name, version=source_version)
source = source_query.filter(DBSource.suites.contains(suite)).first()
if source is None:
if source_suites != True:
source_query = source_query.join(DBSource.suites) \
.filter(Suite.suite_id == source_suites.c.id)
source = source_query.first()
if source is None:
raise ArchiveException('{0}: trying to install to {1}, but could not find source'.format(binary.hashed_file.filename, suite.suite_name))
self.copy_source(source, suite, component)
db_file = self._install_file(directory, binary.hashed_file, suite.archive, component, source_name)
unique = dict(
package=control['Package'],
version=control['Version'],
architecture=architecture,
)
rest = dict(
source=source,
maintainer=maintainer,
poolfile=db_file,
binarytype=binary.type,
fingerprint=fingerprint,
)
try:
db_binary = session.query(DBBinary).filter_by(**unique).one()
for key, value in rest.iteritems():
if getattr(db_binary, key) != value:
raise ArchiveException('{0}: Does not match binary in database.'.format(binary.hashed_file.filename))
except NoResultFound:
db_binary = DBBinary(**unique)
for key, value in rest.iteritems():
setattr(db_binary, key, value)
session.add(db_binary)
session.flush()
import_metadata_into_db(db_binary, session)
self._add_built_using(db_binary, binary.hashed_file.filename, control, suite, extra_archives=extra_source_archives)
if suite not in db_binary.suites:
db_binary.suites.append(suite)
session.flush()
return db_binary
def _ensure_extra_source_exists(self, filename, source, archive, extra_archives=None):
"""ensure source exists in the given archive
This is intended to be used to check that Built-Using sources exist.
@type filename: str
@param filename: filename to use in error messages
@type source: L{daklib.dbconn.DBSource}
@param source: source to look for
@type archive: L{daklib.dbconn.Archive}
@param archive: archive to look in
@type extra_archives: list of L{daklib.dbconn.Archive}
@param extra_archives: list of archives to copy the source package from
if it is not yet present in C{archive}
"""
session = self.session
db_file = session.query(ArchiveFile).filter_by(file=source.poolfile, archive=archive).first()
if db_file is not None:
return True
# Try to copy file from one extra archive
if extra_archives is None:
extra_archives = []
db_file = session.query(ArchiveFile).filter_by(file=source.poolfile).filter(ArchiveFile.archive_id.in_([ a.archive_id for a in extra_archives])).first()
if db_file is None:
raise ArchiveException('{0}: Built-Using refers to package {1} (= {2}) not in target archive {3}.'.format(filename, source.source, source.version, archive.archive_name))
source_archive = db_file.archive
for dsc_file in source.srcfiles:
af = session.query(ArchiveFile).filter_by(file=dsc_file.poolfile, archive=source_archive, component=db_file.component).one()
# We were given an explicit list of archives so it is okay to copy from tainted archives.
self._copy_file(af.file, archive, db_file.component, allow_tainted=True)
def _add_built_using(self, db_binary, filename, control, suite, extra_archives=None):
"""Add Built-Using sources to C{db_binary.extra_sources}
"""
session = self.session
built_using = control.get('Built-Using', None)
if built_using is not None:
for dep in apt_pkg.parse_depends(built_using):
assert len(dep) == 1, 'Alternatives are not allowed in Built-Using field'
bu_source_name, bu_source_version, comp = dep[0]
assert comp == '=', 'Built-Using must contain strict dependencies'
bu_source = session.query(DBSource).filter_by(source=bu_source_name, version=bu_source_version).first()
if bu_source is None:
raise ArchiveException('{0}: Built-Using refers to non-existing source package {1} (= {2})'.format(filename, bu_source_name, bu_source_version))
self._ensure_extra_source_exists(filename, bu_source, suite.archive, extra_archives=extra_archives)
db_binary.extra_sources.append(bu_source)
def install_source(self, directory, source, suite, component, changed_by, allow_tainted=False, fingerprint=None):
"""Install a source package
@type directory: str
@param directory: directory the source package is located in
@type source: L{daklib.upload.Source}
@param source: source package to install
@type suite: L{daklib.dbconn.Suite}
@param suite: target suite
@type component: L{daklib.dbconn.Component}
@param component: target component
@type changed_by: L{daklib.dbconn.Maintainer}
@param changed_by: person who prepared this version of the package
@type allow_tainted: bool
@param allow_tainted: allow to copy additional files from tainted archives
@type fingerprint: L{daklib.dbconn.Fingerprint}
@param fingerprint: optional fingerprint
@rtype: L{daklib.dbconn.DBSource}
@return: database object for the new source
"""
session = self.session
archive = suite.archive
control = source.dsc
maintainer = get_or_set_maintainer(control['Maintainer'], session)
source_name = control['Source']
### Add source package to database
# We need to install the .dsc first as the DBSource object refers to it.
db_file_dsc = self._install_file(directory, source._dsc_file, archive, component, source_name)
unique = dict(
source=source_name,
version=control['Version'],
)
rest = dict(
maintainer=maintainer,
changedby=changed_by,
#install_date=datetime.now().date(),
poolfile=db_file_dsc,
fingerprint=fingerprint,
dm_upload_allowed=(control.get('DM-Upload-Allowed', 'no') == 'yes'),
)
created = False
try:
db_source = session.query(DBSource).filter_by(**unique).one()
for key, value in rest.iteritems():
if getattr(db_source, key) != value:
raise ArchiveException('{0}: Does not match source in database.'.format(source._dsc_file.filename))
except NoResultFound:
created = True
db_source = DBSource(**unique)
for key, value in rest.iteritems():
setattr(db_source, key, value)
# XXX: set as default in postgres?
db_source.install_date = datetime.now().date()
session.add(db_source)
session.flush()
# Add .dsc file. Other files will be added later.
db_dsc_file = DSCFile()
db_dsc_file.source = db_source
db_dsc_file.poolfile = db_file_dsc
session.add(db_dsc_file)
session.flush()
if suite in db_source.suites:
return db_source
db_source.suites.append(suite)
if not created:
for f in db_source.srcfiles:
self._copy_file(f.poolfile, archive, component, allow_tainted=allow_tainted)
return db_source
### Now add remaining files and copy them to the archive.
for hashed_file in source.files.itervalues():
hashed_file_path = os.path.join(directory, hashed_file.filename)
if os.path.exists(hashed_file_path):
db_file = self._install_file(directory, hashed_file, archive, component, source_name)
session.add(db_file)
else:
db_file = self.get_file(hashed_file, source_name)
self._copy_file(db_file, archive, component, allow_tainted=allow_tainted)
db_dsc_file = DSCFile()
db_dsc_file.source = db_source
db_dsc_file.poolfile = db_file
session.add(db_dsc_file)
session.flush()
# Importing is safe as we only arrive here when we did not find the source already installed earlier.
import_metadata_into_db(db_source, session)
# Uploaders are the maintainer and co-maintainers from the Uploaders field
db_source.uploaders.append(maintainer)
if 'Uploaders' in control:
from daklib.textutils import split_uploaders
for u in split_uploaders(control['Uploaders']):
db_source.uploaders.append(get_or_set_maintainer(u, session))
session.flush()
return db_source
def _copy_file(self, db_file, archive, component, allow_tainted=False):
"""Copy a file to the given archive and component
@type db_file: L{daklib.dbconn.PoolFile}
@param db_file: file to copy
@type archive: L{daklib.dbconn.Archive}
@param archive: target archive
@type component: L{daklib.dbconn.Archive}
@param component: target component
@type allow_tainted: bool
@param allow_tainted: allow to copy from tainted archives (such as NEW)
"""
session = self.session
if session.query(ArchiveFile).filter_by(archive=archive, component=component, file=db_file).first() is None:
query = session.query(ArchiveFile).filter_by(file=db_file)
if not allow_tainted:
query = query.join(Archive).filter(Archive.tainted == False)
source_af = query.first()
if source_af is None:
raise ArchiveException('cp: Could not find {0} in any archive.'.format(db_file.filename))
target_af = ArchiveFile(archive, component, db_file)
session.add(target_af)
session.flush()
self.fs.copy(source_af.path, target_af.path, link=False, mode=archive.mode)
def copy_binary(self, db_binary, suite, component, allow_tainted=False, extra_archives=None):
"""Copy a binary package to the given suite and component
@type db_binary: L{daklib.dbconn.DBBinary}
@param db_binary: binary to copy
@type suite: L{daklib.dbconn.Suite}
@param suite: target suite
@type component: L{daklib.dbconn.Component}
@param component: target component
@type allow_tainted: bool
@param allow_tainted: allow to copy from tainted archives (such as NEW)
@type extra_archives: list of L{daklib.dbconn.Archive}
@param extra_archives: extra archives to copy Built-Using sources from
"""
session = self.session
archive = suite.archive
if archive.tainted:
allow_tainted = True
filename = db_binary.poolfile.filename
# make sure source is present in target archive
db_source = db_binary.source
if session.query(ArchiveFile).filter_by(archive=archive, file=db_source.poolfile).first() is None:
raise ArchiveException('{0}: cannot copy to {1}: source is not present in target archive'.format(filename, suite.suite_name))
# make sure built-using packages are present in target archive
for db_source in db_binary.extra_sources:
self._ensure_extra_source_exists(filename, db_source, archive, extra_archives=extra_archives)
# copy binary
db_file = db_binary.poolfile
self._copy_file(db_file, suite.archive, component, allow_tainted=allow_tainted)
if suite not in db_binary.suites:
db_binary.suites.append(suite)
self.session.flush()
def copy_source(self, db_source, suite, component, allow_tainted=False):
"""Copy a source package to the given suite and component
@type db_source: L{daklib.dbconn.DBSource}
@param db_source: source to copy
@type suite: L{daklib.dbconn.Suite}
@param suite: target suite
@type component: L{daklib.dbconn.Component}
@param component: target component
@type allow_tainted: bool
@param allow_tainted: allow to copy from tainted archives (such as NEW)
"""
archive = suite.archive
if archive.tainted:
allow_tainted = True
for db_dsc_file in db_source.srcfiles:
self._copy_file(db_dsc_file.poolfile, archive, component, allow_tainted=allow_tainted)
if suite not in db_source.suites:
db_source.suites.append(suite)
self.session.flush()
def remove_file(self, db_file, archive, component):
"""Remove a file from a given archive and component
@type db_file: L{daklib.dbconn.PoolFile}
@param db_file: file to remove
@type archive: L{daklib.dbconn.Archive}
@param archive: archive to remove the file from
@type component: L{daklib.dbconn.Component}
@param component: component to remove the file from
"""
af = self.session.query(ArchiveFile).filter_by(file=db_file, archive=archive, component=component)
self.fs.unlink(af.path)
self.session.delete(af)
def remove_binary(self, binary, suite):
"""Remove a binary from a given suite and component
@type binary: L{daklib.dbconn.DBBinary}
@param binary: binary to remove
@type suite: L{daklib.dbconn.Suite}
@param suite: suite to remove the package from
"""
binary.suites.remove(suite)
self.session.flush()
def remove_source(self, source, suite):
"""Remove a source from a given suite and component
@type source: L{daklib.dbconn.DBSource}
@param source: source to remove
@type suite: L{daklib.dbconn.Suite}
@param suite: suite to remove the package from
@raise ArchiveException: source package is still referenced by other
binaries in the suite
"""
session = self.session
query = session.query(DBBinary).filter_by(source=source) \
.filter(DBBinary.suites.contains(suite))
if query.first() is not None:
raise ArchiveException('src:{0} is still used by binaries in suite {1}'.format(source.source, suite.suite_name))
source.suites.remove(suite)
session.flush()
def commit(self):
"""commit changes"""
try:
self.session.commit()
self.fs.commit()
finally:
self.session.rollback()
self.fs.rollback()
def rollback(self):
"""rollback changes"""
self.session.rollback()
self.fs.rollback()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if type is None:
self.commit()
else:
self.rollback()
return None