def connection(app, request):
"""Session-wide test database."""
# Connect and create the temporary database
print "=" * 80
print "CREATING TEMPORARY DATABASE FOR TESTS"
print "=" * 80
subprocess.call(['dropdb', DBNAME])
subprocess.call(['createdb', DBNAME])
# Bind and create the database tables
_db.clear()
engine_url = '%s/%s' % (BIND, DBNAME)
db_engine = create_engine(engine_url, encoding='utf-8', poolclass=NullPool)
# AKA models.start()
_db.Session.bind = db_engine
_db.metadata.bind = _db.Session.bind
_db.Base.metadata.create_all(db_engine)
_db.commit()
_db.clear()
#connection = db_engine.connect()
def teardown():
_db.Base.metadata.drop_all(db_engine)
request.addfinalizer(teardown)
# Slap our test app on it
_db.app = app
return _db
def test_filter_binaries_by_sha1(self, session):
models.Binary(
'ceph-1.0.deb',
self.p,
ref='firefly',
distro='ubuntu',
distro_version='precise',
arch='all',
sha1="sha1",
)
models.Binary(
'ceph-1.0.deb',
self.p,
ref='firefly',
distro='ubuntu',
distro_version='trusty',
arch='all',
sha1="head",
)
models.commit()
result = util.get_extra_binaries(
'ceph',
'ubuntu',
'trusty',
distro_versions=['precise', 'trusty'],
ref='firefly',
sha1="sha1")
assert len(result) == 1
def delete_repositories(repo_objects, lifespan, keep_minimum):
logger.info('processing deletion for repos %s days and older', lifespan)
if keep_minimum:
logger.info('will keep at most %s repositories after purging', keep_minimum)
else:
logger.info('will not keep any repositories after purging is completed')
for r in repo_objects:
logger.info('repo %s is being processed for removal', r)
for b in r.binaries:
try:
os.remove(b.path)
except OSError as err:
# no such file, ignore
if err.errno == errno.ENOENT:
pass
b.delete()
models.flush()
try:
if r.path:
shutil.rmtree(r.path)
except OSError as err:
# no such file, ignore
if err.errno == errno.ENOENT:
pass
else:
raise
post_deleted(r)
r.delete()
models.commit()
def delete_repositories(repo_objects, lifespan, keep_minimum):
logger.info('processing deletion for repos %s days and older', lifespan)
if keep_minimum:
logger.info('will keep at most %s repositories after purging',
keep_minimum)
else:
logger.info(
'will not keep any repositories after purging is completed')
for r in repo_objects:
logger.info('repo %s is being processed for removal', r)
for b in r.binaries:
try:
os.remove(b.path)
except OSError as err:
# no such file, ignore
if err.errno == errno.ENOENT:
pass
b.delete()
models.flush()
try:
if r.path:
shutil.rmtree(r.path)
except OSError as err:
# no such file, ignore
if err.errno == errno.ENOENT:
pass
else:
raise
post_deleted(r)
r.delete()
models.commit()
def connection(app, request):
"""Session-wide test database."""
# Connect and create the temporary database
print "=" * 80
print "CREATING TEMPORARY DATABASE FOR TESTS"
print "=" * 80
subprocess.call(['dropdb', DBNAME])
subprocess.call(['createdb', DBNAME])
# Bind and create the database tables
_db.clear()
engine_url = '%s/%s' % (BIND, DBNAME)
db_engine = create_engine(
engine_url,
encoding='utf-8',
poolclass=NullPool)
# AKA models.start()
_db.Session.bind = db_engine
_db.metadata.bind = _db.Session.bind
_db.Base.metadata.create_all(db_engine)
_db.commit()
_db.clear()
def teardown():
_db.Base.metadata.drop_all(db_engine)
request.addfinalizer(teardown)
# Slap our test app on it
_db.app = app
return _db
def test_ref_matches_binaries_from_distro_versions(self, session):
models.Binary(
'ceph-1.0.deb',
self.p,
ref='firefly',
distro='ubuntu',
distro_version='precise',
arch='all',
)
models.Binary(
'ceph-1.0.deb',
self.p,
ref='firefly',
distro='ubuntu',
distro_version='trusty',
arch='all',
)
models.commit()
result = util.get_extra_binaries(
'ceph',
'ubuntu',
'trusty',
distro_versions=['precise', 'trusty'],
ref='firefly')
assert len(result) == 2
def test_no_matching_ref_with_specific_ref(self, session):
models.commit()
result = util.get_extra_binaries('ceph',
'ubuntu',
'precise',
ref='master')
assert result == []
def poll_repos():
"""
Poll the repository objects that need to be updated and call the tasks
that can create (or update) repositories with that information
"""
logger.info('polling repos....')
for r in models.Repo.query.filter_by(needs_update=True).all():
# this repo is being processed, do not pile up and try to get it
# processed again until it is done doing work
if r.is_updating:
continue
if r.needs_update:
logger.info("repo %s needs to be updated/created", r)
if r.type == 'rpm':
create_rpm_repo.apply_async(
(r.id,),
countdown=pecan.conf.quiet_time)
elif r.type == 'deb':
create_deb_repo.apply_async(
(r.id,),
countdown=pecan.conf.quiet_time)
else:
_type = r.infer_type()
if _type is None:
logger.warning('failed to infer repository type')
logger.warning('got a repository with an unknown type: %s', r)
else:
logger.warning('inferred repo type as: %s', _type)
r.type = _type
models.commit()
logger.info('completed repo polling')
def create_rpm_repo(repo_id):
"""
Go create or update repositories with specific IDs.
"""
directories = ['SRPMS', 'noarch', 'x86_64']
# get the root path for storing repos
# TODO: Is it possible we can get an ID that doesn't exist anymore?
repo = models.Repo.get(repo_id)
logger.info("processing repository: %s", repo)
# Determine paths for this repository
paths = util.repo_paths(repo)
repo_dirs = [os.path.join(paths['absolute'], d) for d in directories]
# Before doing work that might take very long to complete, set the repo
# path in the object and mark needs_update as False
repo.path = paths['absolute']
repo.needs_update = False
models.commit()
# this is safe to do, behind the scenes it is just trying to create them if
# they don't exist and it will include the 'absolute' path
for d in repo_dirs:
util.makedirs(d)
# now that structure is done, we need to symlink the RPMs that belong
# to this repo so that we can create the metadata.
conf_extra_repos = util.get_extra_repos(repo.project.name, repo.ref)
extra_binaries = []
for project_name, project_refs in conf_extra_repos.items():
for ref in project_refs:
extra_binaries += util.get_extra_binaries(
project_name,
repo.distro,
repo.distro_version,
ref=ref if ref != 'all' else None
)
all_binaries = extra_binaries + [b for b in repo.binaries]
for binary in all_binaries:
source = binary.path
arch_directory = util.infer_arch_directory(binary.name)
destination_dir = os.path.join(paths['absolute'], arch_directory)
destination = os.path.join(destination_dir, binary.name)
try:
if not os.path.exists(destination):
os.symlink(source, destination)
except OSError:
logger.exception('could not symlink')
for d in repo_dirs:
subprocess.check_call(['createrepo', d])
def run(self, args):
super(PopulateCommand, self).run(args)
out("LOADING ENVIRONMENT")
self.load_app()
out("BUILDING SCHEMA")
try:
out("STARTING A TRANSACTION...")
models.start()
models.Base.metadata.create_all(conf.sqlalchemy.engine)
except:
models.rollback()
out("ROLLING BACK... ")
raise
else:
out("COMMITING... ")
models.commit()
def test_no_ref_matches_binaries(self, session):
models.Binary(
'ceph-1.1.deb',
self.p,
distro='ubuntu',
distro_version='trusty',
arch='all',
)
models.Binary(
'ceph-1.0.deb',
self.p,
distro='ubuntu',
distro_version='trusty',
arch='all',
)
models.commit()
result = util.get_extra_binaries('ceph', 'ubuntu', 'trusty')
assert len(result) == 2
def run(self, args):
super(PopulateCommand, self).run(args)
out("LOADING ENVIRONMENT")
self.load_app()
out("BUILDING SCHEMA")
try:
out("STARTING A TRANSACTION...")
models.start()
models.Base.metadata.create_all(conf.sqlalchemy.engine)
except:
models.rollback()
out("ROLLING BACK... ")
raise
else:
out("COMMITING... ")
models.commit()
out("STAMPING INITIAL STATE WITH ALEMBIC... ")
alembic_cfg = Config(get_alembic_config())
command.stamp(alembic_cfg, "head")
def run(self, args):
super(PopulateCommand, self).run(args)
out("LOADING ENVIRONMENT")
self.load_app()
out("BUILDING SCHEMA")
try:
out("STARTING A TRANSACTION...")
models.start()
models.Base.metadata.create_all(conf.sqlalchemy.engine)
except:
models.rollback()
out("ROLLING BACK... ")
raise
else:
out("COMMITING... ")
models.commit()
out("STAMPING INITIAL STATE WITH ALEMBIC... ")
alembic_cfg = Config(get_alembic_config())
command.stamp(alembic_cfg, "head")
def poll_repos():
"""
Poll the repository objects that need to be updated and call the tasks
that can create (or update) repositories with that information
"""
logger.info('polling repos....')
for r in models.Repo.query.filter_by(needs_update=True,
is_queued=False).all():
# this repo is being processed, do not pile up and try to get it
# processed again until it is done doing work
if r.is_updating:
continue
if r.needs_update:
logger.info("repo %s needs to be updated/created", r)
r.is_queued = True
post_queued(r)
if r.type == 'rpm':
rpm.create_rpm_repo.apply_async(
(r.id, ),
countdown=pecan.conf.quiet_time,
queue='build_repos',
)
elif r.type == 'deb':
debian.create_deb_repo.apply_async(
(r.id, ),
countdown=pecan.conf.quiet_time,
queue='build_repos',
)
else:
_type = r.infer_type()
if _type is None:
logger.warning('failed to infer repository type')
logger.warning('got a repository with an unknown type: %s',
r)
else:
logger.warning('inferred repo type as: %s', _type)
r.type = _type
models.commit()
logger.info('completed repo polling')
def create_deb_repo(repo_id):
"""
Go create or update repositories with specific IDs.
"""
# get the root path for storing repos
# TODO: Is it possible we can get an ID that doesn't exist anymore?
repo = models.Repo.get(repo_id)
timer = Timer(__name__, suffix="create.deb.%s" % repo.metric_name)
counter = Counter(__name__, suffix="create.deb.%s" % repo.metric_name)
timer.start()
post_building(repo)
logger.info("processing repository: %s", repo)
if util.repository_is_disabled(repo.project.name):
logger.info("will not process repository: %s", repo)
repo.needs_update = False
repo.is_queued = False
models.commit()
return
# Determine paths for this repository
paths = util.repo_paths(repo)
# Before doing work that might take very long to complete, set the repo
# path in the object, mark needs_update as False, and mark it as being
# updated so we prevent piling up if other binaries are being posted
repo.path = paths['absolute']
repo.is_updating = True
repo.is_queued = False
repo.needs_update = False
models.commit()
# determine if other repositories might need to be queried to add extra
# binaries (repos are tied to binaries which are all related with refs,
# archs, distros, and distro versions.
conf_extra_repos = util.get_extra_repos(repo.project.name, repo.ref)
combined_versions = util.get_combined_repos(repo.project.name)
extra_binaries = []
# See if there are any generic/universal binaries so that they can be
# automatically added from the current project
for binary in util.get_extra_binaries(
repo.project.name,
repo.distro,
None,
distro_versions=['generic', 'universal', 'any'],
ref=repo.ref,
sha1=repo.sha1):
extra_binaries.append(binary)
for project_name, project_refs in conf_extra_repos.items():
for ref in project_refs:
logger.info('fetching binaries for project: %s, ref: %s',
project_name, ref)
found_binaries = util.get_extra_binaries(
project_name,
None,
repo.distro_version,
distro_versions=combined_versions,
ref=ref if ref != 'all' else None)
extra_binaries += found_binaries
# See if there are any generic/universal binaries so that they can be
# automatically added from projects coming from extra repos
for binary in util.get_extra_binaries(
project_name,
repo.distro,
None,
distro_versions=['generic', 'universal', 'any'],
ref=ref if ref != 'all' else None):
extra_binaries.append(binary)
# check for the option to 'combine' repositories with different
# debian/ubuntu versions
for distro_version in combined_versions:
logger.info('fetching distro_version %s for project: %s',
distro_version, repo.project.name)
# When combining distro_versions we cannot filter by distribution as
# well, otherwise it will be an impossible query. E.g. "get wheezy,
# precise and trusty but only for the Ubuntu distro"
extra_binaries += util.get_extra_binaries(repo.project.name,
None,
distro_version,
ref=repo.ref,
sha1=repo.sha1)
# try to create the absolute path to the repository if it doesn't exist
util.makedirs(paths['absolute'])
all_binaries = extra_binaries + [b for b in repo.binaries]
timer.intermediate('collection')
for binary in set(all_binaries):
# XXX This is really not a good alternative but we are not going to be
# using .changes for now although we can store it.
if binary.extension == 'changes':
continue
try:
commands = util.reprepro_commands(
paths['absolute'],
binary,
distro_versions=combined_versions,
fallback_version=repo.distro_version)
except KeyError: # probably a tar.gz or similar file that should not be added directly
continue
for command in commands:
logger.info('running command: %s', ' '.join(command))
result = subprocess.Popen(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
if result.returncode > 0:
logger.error('failed to add binary %s', binary.name)
stdout, stderr = result.communicate()
for line in stdout.split('\n'):
logger.info(line)
for line in stderr.split('\n'):
logger.warning(line)
logger.info("finished processing repository: %s", repo)
repo.is_updating = False
models.commit()
timer.stop()
counter += 1
post_ready(repo)
def test_no_matching_ref_with_specific_ref(self, session):
models.commit()
result = util.get_extra_binaries(
'ceph', 'ubuntu', 'precise', ref='master')
assert result == []
def create_rpm_repo(repo_id):
"""
Go create or update repositories with specific IDs.
"""
directories = ['SRPMS', 'noarch', 'x86_64', 'aarch64']
# get the root path for storing repos
# TODO: Is it possible we can get an ID that doesn't exist anymore?
repo = models.Repo.get(repo_id)
post_building(repo)
timer = Timer(__name__, suffix="create.rpm.%s" % repo.metric_name)
counter = Counter(__name__, suffix="create.rpm.%s" % repo.metric_name)
timer.start()
logger.info("processing repository: %s", repo)
if util.repository_is_disabled(repo.project.name):
logger.info("will not process repository: %s", repo)
repo.needs_update = False
repo.is_queued = False
return
# Determine paths for this repository
paths = util.repo_paths(repo)
repo_dirs = [os.path.join(paths['absolute'], d) for d in directories]
# Before doing work that might take very long to complete, set the repo
# path in the object and mark needs_update as False
repo.path = paths['absolute']
repo.is_updating = True
repo.is_queued = False
repo.needs_update = False
models.commit()
# this is safe to do, behind the scenes it is just trying to create them if
# they don't exist and it will include the 'absolute' path
for d in repo_dirs:
util.makedirs(d)
# now that structure is done, we need to symlink the RPMs that belong
# to this repo so that we can create the metadata.
conf_extra_repos = util.get_extra_repos(repo.project.name, repo.ref)
extra_binaries = []
for project_name, project_refs in conf_extra_repos.items():
for ref in project_refs:
extra_binaries += util.get_extra_binaries(
project_name,
repo.distro,
repo.distro_version,
ref=ref if ref != 'all' else None)
all_binaries = extra_binaries + [b for b in repo.binaries]
timer.intermediate('collection')
for binary in all_binaries:
source = binary.path
arch_directory = util.infer_arch_directory(binary.name)
destination_dir = os.path.join(paths['absolute'], arch_directory)
destination = os.path.join(destination_dir, binary.name)
try:
if not os.path.exists(destination):
os.symlink(source, destination)
except OSError:
logger.exception('could not symlink')
for d in repo_dirs:
subprocess.check_call(['createrepo', d])
logger.info("finished processing repository: %s", repo)
repo.is_updating = False
models.commit()
timer.stop()
counter += 1
post_ready(repo)
def create_deb_repo(repo_id):
"""
Go create or update repositories with specific IDs.
"""
# get the root path for storing repos
# TODO: Is it possible we can get an ID that doesn't exist anymore?
repo = models.Repo.get(repo_id)
timer = Timer(__name__, suffix="create.deb.%s" % repo.metric_name)
counter = Counter(__name__, suffix="create.deb.%s" % repo.metric_name)
timer.start()
post_building(repo)
logger.info("processing repository: %s", repo)
if util.repository_is_disabled(repo.project.name):
logger.info("will not process repository: %s", repo)
repo.needs_update = False
repo.is_queued = False
models.commit()
return
# Determine paths for this repository
paths = util.repo_paths(repo)
# Before doing work that might take very long to complete, set the repo
# path in the object, mark needs_update as False, and mark it as being
# updated so we prevent piling up if other binaries are being posted
repo.path = paths['absolute']
repo.is_updating = True
repo.is_queued = False
repo.needs_update = False
models.commit()
# determine if other repositories might need to be queried to add extra
# binaries (repos are tied to binaries which are all related with refs,
# archs, distros, and distro versions.
conf_extra_repos = util.get_extra_repos(repo.project.name, repo.ref)
combined_versions = util.get_combined_repos(repo.project.name)
extra_binaries = []
# See if there are any generic/universal binaries so that they can be
# automatically added from the current project
for binary in util.get_extra_binaries(
repo.project.name,
repo.distro,
None,
distro_versions=['generic', 'universal', 'any'],
ref=repo.ref,
sha1=repo.sha1):
extra_binaries.append(binary)
for project_name, project_refs in conf_extra_repos.items():
for ref in project_refs:
logger.info('fetching binaries for project: %s, ref: %s', project_name, ref)
found_binaries = util.get_extra_binaries(
project_name,
None,
repo.distro_version,
distro_versions=combined_versions,
ref=ref if ref != 'all' else None
)
extra_binaries += found_binaries
# See if there are any generic/universal binaries so that they can be
# automatically added from projects coming from extra repos
for binary in util.get_extra_binaries(
project_name,
repo.distro,
None,
distro_versions=['generic', 'universal', 'any'],
ref=ref if ref != 'all' else None):
extra_binaries.append(binary)
# check for the option to 'combine' repositories with different
# debian/ubuntu versions
for distro_version in combined_versions:
logger.info(
'fetching distro_version %s for project: %s',
distro_version,
repo.project.name
)
# When combining distro_versions we cannot filter by distribution as
# well, otherwise it will be an impossible query. E.g. "get wheezy,
# precise and trusty but only for the Ubuntu distro"
extra_binaries += util.get_extra_binaries(
repo.project.name,
None,
distro_version,
ref=repo.ref,
sha1=repo.sha1
)
# try to create the absolute path to the repository if it doesn't exist
util.makedirs(paths['absolute'])
all_binaries = extra_binaries + [b for b in repo.binaries]
timer.intermediate('collection')
for binary in set(all_binaries):
# XXX This is really not a good alternative but we are not going to be
# using .changes for now although we can store it.
if binary.extension == 'changes':
continue
try:
commands = util.reprepro_commands(
paths['absolute'],
binary,
distro_versions=combined_versions,
fallback_version=repo.distro_version
)
except KeyError: # probably a tar.gz or similar file that should not be added directly
continue
for command in commands:
logger.info('running command: %s', ' '.join(command))
result = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
if result.returncode > 0:
logger.error('failed to add binary %s', binary.name)
stdout, stderr = result.communicate()
for line in stdout.split('\n'):
logger.info(line)
for line in stderr.split('\n'):
logger.warning(line)
logger.info("finished processing repository: %s", repo)
repo.is_updating = False
models.commit()
timer.stop()
counter += 1
post_ready(repo)
def create_deb_repo(repo_id):
"""
Go create or update repositories with specific IDs.
"""
# get the root path for storing repos
# TODO: Is it possible we can get an ID that doesn't exist anymore?
repo = models.Repo.get(repo_id)
logger.info("processing repository: %s", repo)
# Determine paths for this repository
paths = util.repo_paths(repo)
# determine if other repositories might need to be queried to add extra
# binaries (repos are tied to binaries which are all related with refs,
# archs, distros, and distro versions.
conf_extra_repos = util.get_extra_repos(repo.project.name, repo.ref)
combined_versions = util.get_combined_repos(repo.project.name)
extra_binaries = []
for project_name, project_refs in conf_extra_repos.items():
for ref in project_refs:
logger.info('fetching binaries for project: %s, ref: %s', project_name, ref)
found_binaries = util.get_extra_binaries(
project_name,
None,
repo.distro_version,
distro_versions=combined_versions,
ref=ref if ref != 'all' else None
)
extra_binaries += found_binaries
# check for the option to 'combine' repositories with different
# debian/ubuntu versions
for distro_version in combined_versions:
logger.info(
'fetching distro_version %s for project: %s',
distro_version,
repo.project.name
)
# When combining distro_versions we cannot filter by distribution as
# well, otherwise it will be an impossible query. E.g. "get wheezy,
# precise and trusty but only for the Ubuntu distro"
extra_binaries += util.get_extra_binaries(
repo.project.name,
None,
distro_version,
ref=repo.ref
)
# try to create the absolute path to the repository if it doesn't exist
util.makedirs(paths['absolute'])
all_binaries = extra_binaries + [b for b in repo.binaries]
for binary in all_binaries:
# XXX This is really not a good alternative but we are not going to be
# using .changes for now although we can store it.
if binary.extension == 'changes':
continue
try:
command = util.reprepro_command(paths['absolute'], binary)
except KeyError: # probably a tar.gz or similar file that should not be added directly
continue
try:
logger.info('running command: %s', ' '.join(command))
except TypeError:
logger.exception('was not able to add binary: %s', binary)
continue
else:
try:
subprocess.check_call(command)
except subprocess.CalledProcessError:
logger.exception('failed to add binary %s', binary.name)
# Finally, set the repo path in the object and mark needs_update as False
repo.path = paths['absolute']
repo.needs_update = False
models.commit()
