def reset_pulp(cfg):
"""Stop Pulp, reset its database, remove certain files, and start it.
:param pulp_smash.config.PulpSmashConfig cfg: Information about
the Pulp server being targeted.
:returns: Nothing.
"""
svc_mgr = cli.GlobalServiceManager(cfg)
svc_mgr.stop(PULP_SERVICES)
# Reset the database and nuke accumulated files.
#
# Why use `runuser` instead of `sudo`? Because some systems are configured
# to refuse to execute `sudo` unless a tty is present (The author has
# encountered this on at least one RHEL 7.2 host.)
#
# Why not use runuser's `-u` flag? Because RHEL 6 ships an old version of
# runuser that doesn't support the flag, and RHEL 6 is a supported Pulp
# platform.
client = cli.Client(cfg, pulp_host=cfg.get_hosts('mongod')[0])
client.run('mongo pulp_database --eval db.dropDatabase()'.split())
for index, host in enumerate(cfg.get_hosts('api')):
prefix = '' if cli.is_root(cfg, pulp_host=host) else 'sudo '
if index == 0:
client.run((
prefix + 'runuser --shell /bin/sh apache --command '
'pulp-manage-db'
).split())
client.run((prefix + 'rm -rf /var/lib/pulp/content').split())
client.run((prefix + 'rm -rf /var/lib/pulp/published').split())
svc_mgr.start(PULP_SERVICES)
def reset_pulp(server_config):
"""Stop Pulp, reset its database, remove certain files, and start it.
:param pulp_smash.config.ServerConfig server_config: Information about the
Pulp server being targeted.
:returns: Nothing.
"""
services = tuple((
cli.Service(server_config, service) for service in PULP_SERVICES
))
for service in services:
service.stop()
# Reset the database and nuke accumulated files.
#
# Why use `runuser` instead of `sudo`? Because some systems are configured
# to refuse to execute `sudo` unless a tty is present (The author has
# encountered this on at least one RHEL 7.2 system.)
#
# Why not use runuser's `-u` flag? Because RHEL 6 ships an old version of
# runuser that doesn't support the flag, and RHEL 6 is a supported Pulp
# platform.
client = cli.Client(server_config)
prefix = '' if is_root(server_config) else 'sudo '
client.run('mongo pulp_database --eval db.dropDatabase()'.split())
client.run((
prefix + 'runuser --shell /bin/sh apache --command pulp-manage-db'
).split())
client.run((prefix + 'rm -rf /var/lib/pulp/content').split())
client.run((prefix + 'rm -rf /var/lib/pulp/published').split())
for service in services:
service.start()
def do_test(self, distributor_config_update):
"""Implement most of the test logic."""
rpms = tuple(
utils.http_get(url)
for url in (RPM_UNSIGNED_URL, RPM2_UNSIGNED_URL))
# Create a repository.
client = api.Client(self.cfg, api.json_handler)
body = gen_repo()
body['distributors'] = [gen_distributor()]
body['distributors'][0]['distributor_config'].update(
distributor_config_update)
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(client.delete, repo['_href'])
repo = client.get(repo['_href'], params={'details': True})
# Upload an RPM, publish the repo, and count metadata files twice.
cli_client = cli.Client(self.cfg)
sudo = () if cli.is_root(self.cfg) else ('sudo', )
find_repodata_cmd = sudo + (
'find',
os.path.join('/var/lib/pulp/published/yum/master/yum_distributor/',
str(repo['id'])), '-type', 'd', '-name', 'repodata')
found = []
for rpm in rpms:
upload_import_unit(self.cfg, rpm, {'unit_type_id': 'rpm'}, repo)
publish_repo(self.cfg, repo)
repodata_path = cli_client.run(find_repodata_cmd).stdout.strip()
found.append(
cli_client.run(sudo + ('find', repodata_path, '-type',
'f')).stdout.splitlines())
return found
def write_private_key(self, cfg, private_key):
"""Write the given private key to a file on disk.
Ensure that the file is owned by user "apache" and has permissions of
``600``. In addition, schedule the key for deletion with
``self.addCleanup``.
:param cfg: Information about the
host being targeted.
:returns: The path to the private key on disk, as a string.
"""
sudo = '' if cli.is_root(cfg) else 'sudo '
client = cli.Client(cfg)
ssh_identity_file = client.run(['mktemp']).stdout.strip()
self.addCleanup(client.run, (sudo + 'rm ' + ssh_identity_file).split())
client.machine.session().run(
"echo '{}' > {}".format(private_key, ssh_identity_file)
)
client.run(['chmod', '600', ssh_identity_file])
client.run((sudo + 'chown apache ' + ssh_identity_file).split())
# Pulp's SELinux policy requires files handled by Pulp to have the
# httpd_sys_rw_content_t label
enforcing = client.run(['getenforce']).stdout.strip()
if enforcing.lower() != 'disabled':
client.run(
(sudo + 'chcon -t httpd_sys_rw_content_t ' + ssh_identity_file)
.split()
)
return ssh_identity_file
def set_pulp_manage_rsync(cfg, boolean):
"""Set the ``pulp_manage_rsync`` SELinux policy.
If the ``setsebool`` executable is not available, return. (This is the case
if SELinux isn't installed on the system under test.) Otherwise, set the
``pulp_manage_rsync SELinux policy on or off, depending on the truthiness
of ``boolean``.
For more information on the ``pulp_manage_rsync`` SELinux policy, see `ISO
rsync Distributor → Configuration
<http://docs.pulpproject.org/plugins/pulp_rpm/tech-reference/iso-rsync-distributor.html#configuration>`_.
:param cfg: Information about a Pulp
host.
:param boolean: Either ``True`` or ``False``.
:returns: Information about the executed command, or ``None`` if no command
was executed.
"""
sudo = () if cli.is_root(cfg) else ('sudo',)
client = cli.Client(cfg)
try:
# setsebool is installed at /usr/sbin/setsebool on some distros, and
# requires root privileges to discover.
client.run(sudo + ('which', 'setsebool'))
except exceptions.CalledProcessError:
return None
cmd = sudo + ('setsebool', 'pulp_manage_rsync')
cmd += ('on',) if boolean else ('off',)
return client.run(cmd)
def verify_remote_units_path(self, cfg, distributor_cfg, num_units=None):
"""Verify the RPM rsync distributor has placed RPMs as appropriate.
Verify that path ``{root}/{remote_units_path}/rpm/`` exists in the
target system's filesystem, and that the correct number of RPMs are
present in that directory.
:param cfg: Information about the
system onto which files have been published.
:param distributor_cfg: A dict of information about an RPM rsync
distributor.
:param num_units: The number of units that should be on the target
system's filesystem. Defaults to
``pulp_2_tests.constants.RPM_SIGNED_FEED_COUNT`.
:returns: Nothing.
"""
if num_units is None:
num_units = RPM_SIGNED_FEED_COUNT
cli_client = cli.Client(cfg)
sudo = () if cli.is_root(cfg) else ('sudo', )
path = distributor_cfg['config']['remote']['root']
remote_units_path = (distributor_cfg['config'].get(
'remote_units_path', 'content/units'))
for segment in _split_path(remote_units_path):
cmd = sudo + ('ls', '-1', path)
files = set(cli_client.run(cmd).stdout.strip().split('\n'))
self.assertIn(segment, files)
path = os.path.join(path, segment)
cmd = sudo + ('find', path, '-name', '*.rpm')
files = cli_client.run(cmd).stdout.strip().split('\n')
self.assertEqual(len(files), num_units, files)
def reset_squid(cfg):
"""Stop Squid, reset its cache directory, and restart it.
:param pulp_smash.config.PulpSmashConfig cfg: Information about a Pulp
host.
:returns: Nothing.
"""
squid_version = _get_squid_version(cfg)
svc_mgr = cli.GlobalServiceManager(cfg)
svc_mgr.stop(('squid',))
# Remove and re-initialize the cache directory.
sudo = () if cli.is_root(cfg) else ('sudo',)
client = cli.Client(cfg)
client.run(sudo + ('rm', '-rf', '/var/spool/squid'))
client.run(sudo + (
'mkdir', '--context=system_u:object_r:squid_cache_t:s0', '--mode=750',
'/var/spool/squid'))
client.run(sudo + ('chown', 'squid:squid', '/var/spool/squid'))
if squid_version < Version('4'):
client.run(sudo + ('squid', '-z'))
else:
client.run(sudo + ('squid', '-z', '--foreground'))
svc_mgr.start(('squid',))
def test_dry_run(self):
"""Make sure pulp-manage-db runs if --dry-run is passed."""
if not selectors.bug_is_fixed(2776, self.cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/2776')
cmd = () if cli.is_root(self.cfg) else ('sudo', )
cmd += ('runuser', '--shell', '/bin/sh', '--command',
'pulp-manage-db --dry-run', '-', 'apache')
cli.Client(self.cfg).run(cmd)
def sudo(self):
"""Return either ``''`` or ``'sudo '``.
Return the former if root, and the latter if not.
"""
if self.__sudo is None:
self.__sudo = '' if cli.is_root(self.cfg) else 'sudo '
return self.__sudo
def setUpClass(cls):
"""Get all of the processes running on the target Pulp system."""
cfg = config.get_config()
cmd = [] if cli.is_root(config.get_config()) else ['sudo']
cmd.extend(('ps', '-A', '-w', '-w', '-o', ','.join(PS_FIELDS)))
cls.procs = [
Process(*line.split(maxsplit=1))
for line in cli.Client(cfg).run(cmd).stdout.splitlines()
]
def test_all(self):
"""Test whether copied files retain their original mtime.
This test targets the following issues:
* `Pulp #2783 <https://pulp.plan.io/issues/2783>`_
* `Pulp Smash #720 <https://github.com/PulpQE/pulp-smash/issues/720>`_
Do the following:
1. Create, sync and publish a repository, with ``generate_sqlite`` set
to true.
2. Get the ``mtime`` of the sqlite files.
3. Upload an RPM package into the repository, and sync the repository.
4. Get the ``mtime`` of the sqlite files again. Verify that the mtimes
are the same.
"""
cfg = config.get_config()
if not selectors.bug_is_fixed(2783, cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/2783')
# Create, sync and publish a repository.
client = api.Client(cfg, api.json_handler)
body = gen_repo()
body['importer_config']['feed'] = RPM_UNSIGNED_FEED_URL
body['distributors'] = [gen_distributor()]
body['distributors'][0]['distributor_config']['generate_sqlite'] = True
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(client.delete, repo['_href'])
repo = client.get(repo['_href'], params={'details': True})
sync_repo(cfg, repo)
publish_repo(cfg, repo)
# Get the mtime of the sqlite files.
cli_client = cli.Client(cfg, cli.echo_handler)
cmd = '' if cli.is_root(cfg) else 'sudo '
cmd += "bash -c \"stat --format %Y '{}'/*\"".format(
os.path.join(
_PATH,
repo['distributors'][0]['config']['relative_url'],
'repodata',
))
# machine.session is used here to keep SSH session open
mtimes_pre = (
cli_client.machine.session().run(cmd)[1].strip().split().sort())
# Upload to the repo, and sync it.
rpm = utils.http_get(RPM_SIGNED_URL)
upload_import_unit(cfg, rpm, {'unit_type_id': 'rpm'}, repo)
sync_repo(cfg, repo)
# Get the mtime of the sqlite files again.
time.sleep(1)
# machine.session is used here to keep SSH session open
mtimes_post = (
cli_client.machine.session().run(cmd)[1].strip().split().sort())
self.assertEqual(mtimes_pre, mtimes_post)
def setUpClass(cls):
"""Maybe skip this test case."""
if inspect.getmro(cls)[0] == BaseTestCase:
raise unittest.SkipTest('Abstract base class.')
cls.cfg = config.get_config()
if not selectors.bug_is_fixed(2186, cls.cfg.pulp_version):
raise unittest.SkipTest('https://pulp.plan.io/issues/2186')
cls.cmd = () if cli.is_root(cls.cfg) else ('sudo', )
cls.cmd += ('runuser', '--shell', '/bin/sh', '--command',
'pulp-manage-db', '-', 'apache')
def setUp(self):
"""Ensure there is only one Pulp worker."""
self.cfg = config.get_config()
if not selectors.bug_is_fixed(2835, self.cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/2835')
sudo = '' if cli.is_root(self.cfg) else 'sudo'
# machine.session is used here to keep SSH session open
cli.Client(self.cfg).machine.session().run(
"{} bash -c 'echo PULP_CONCURRENCY=1 >> {}'".format(
sudo, _PULP_WORKERS_CFG))
cli.GlobalServiceManager(self.cfg).restart(PULP_SERVICES)
def test_all(self):
"""Test Pulp's handling of its ``PULP_MAX_TASKS_PER_CHILD`` setting."""
cfg = config.get_config()
if not selectors.bug_is_fixed(2172, cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/2172')
pulp_3540_testable = selectors.bug_is_fixed(3540, cfg.pulp_version)
if os_is_f27(cfg) and not pulp_3540_testable:
self.skipTest('https://pulp.plan.io/issues/3540')
svc_mgr = cli.GlobalServiceManager(cfg)
sudo = () if cli.is_root(cfg) else ('sudo', )
set_cmd = sudo + (
'sed', '-i', '-e',
's/.*PULP_MAX_TASKS_PER_CHILD=[0-9]*$/PULP_MAX_TASKS_PER_CHILD=2/',
'/etc/default/pulp_workers')
unset_cmd = sudo + (
'sed', '-i', '-e',
's/^PULP_MAX_TASKS_PER_CHILD=2$/# PULP_MAX_TASKS_PER_CHILD=2/',
'/etc/default/pulp_workers')
procs_over_time = []
# Step 1
procs_over_time.append(get_pulp_worker_procs(cfg))
for proc in procs_over_time[-1]:
self.assertNotIn('--maxtasksperchild=2', proc, procs_over_time)
# Step 2
client = cli.Client(cfg)
client.run(set_cmd)
self.addCleanup(svc_mgr.restart, PULP_SERVICES)
if not pulp_3540_testable:
self.addCleanup(time.sleep, 30)
self.addCleanup(client.run, unset_cmd)
svc_mgr.restart(PULP_SERVICES)
procs_over_time.append(get_pulp_worker_procs(cfg))
for proc in procs_over_time[-1]:
self.assertIn('--maxtasksperchild=2', proc, procs_over_time)
# Step 3
repo_id = utils.uuid4()
proc = client.run(('pulp-admin', 'rpm', 'repo', 'create', '--repo-id',
repo_id, '--feed', RPM_UNSIGNED_FEED_URL))
self.addCleanup(
client.run,
('pulp-admin', 'rpm', 'repo', 'delete', '--repo-id', repo_id))
self.assertNotIn('Task Failed', proc.stdout)
proc = client.run(
('pulp-admin', 'rpm', 'repo', 'sync', 'run', '--repo-id', repo_id))
self.assertNotIn('Task Failed', proc.stdout)
# Step 4
self.doCleanups()
procs_over_time.append(get_pulp_worker_procs(cfg))
for proc in procs_over_time[-1]:
self.assertNotIn('--maxtasksperchild=2', proc, procs_over_time)
def _verify_files_not_in_dir(self, cfg, *, yum_distributor,
rpm_rsync_distributor):
"""Verify no RPMs are in the distributor's ``relative_url`` dir."""
path = os.path.join(
rpm_rsync_distributor['config']['remote']['root'],
yum_distributor['config']['relative_url'],
)
cmd = ['find', path, '-name', '*.rpm']
if not cli.is_root(cfg):
cmd.insert(0, 'sudo')
files = cli.Client(cfg).run(cmd).stdout.strip().split('\n')
self.assertEqual(files, ['']) # strange, but correct
def get_pulp_worker_procs(cfg):
"""Use ``ps aux`` to get information about each Pulp worker process.
:param cfg: Information about the Pulp
deployment being targeted.
:return: An iterable of strings, one per line of matching output.
"""
sudo = () if cli.is_root(cfg) else ('sudo', )
cmd = sudo + ('ps', 'aux')
return tuple(
(proc for proc in cli.Client(cfg).run(cmd).stdout.splitlines()
if 'celery worker' in proc and 'resource_manager' not in proc))
def gen_yum_config_file(cfg, repositoryid, baseurl, name, **kwargs):
"""Generate a yum configuration file and write it to ``/etc/yum.repos.d/``.
Generate a yum configuration file containing a single repository section,
and write it to ``/etc/yum.repos.d/{repositoryid}.repo``.
:param cfg: The system on which to create
a yum configuration file.
:param repositoryid: The section's ``repositoryid``. Used when naming the
configuration file and populating the brackets at the head of the file.
For details, see yum.conf(5).
:param baseurl: The required option ``baseurl`` specifying the url of repo.
For details, see yum.conf(5)
:param name: The required option ``name`` specifying the name of repo.
For details, see yum.conf(5).
:param kwargs: Section options. Each kwarg corresponds to one option. For
details, see yum.conf(5).
:returns: The path to the yum configuration file.
"""
# required repo options
kwargs.setdefault('name', name)
kwargs.setdefault('baseurl', baseurl)
# assume some common used defaults
kwargs.setdefault('enabled', 1)
kwargs.setdefault('gpgcheck', 0)
kwargs.setdefault('metadata_expire', 0) # force metadata load every time
# Check if the settings specifies a content host role else assume ``api``
try:
content_host = cfg.get_hosts('content')[0].roles['content']
except IndexError:
content_host = cfg.get_hosts('api')[0].roles['api']
# if sslverify is not provided in kwargs it is inferred from cfg
kwargs.setdefault(
'sslverify', content_host.get('verify') and 'yes' or 'no'
)
path = os.path.join('/etc/yum.repos.d/', repositoryid + '.repo')
with StringIO() as section:
section.write('[{}]\n'.format(repositoryid))
for key, value in kwargs.items():
section.write('{} = {}\n'.format(key, value))
# machine.session is used here to keep SSH session open
cli.Client(cfg).machine.session().run(
'echo "{}" | {}tee {} > /dev/null'.format(
section.getvalue(),
'' if cli.is_root(cfg) else 'sudo ',
path
)
)
return path
def gen_yum_config_file(cfg, repositoryid, baseurl, name, **kwargs):
"""Generate a yum configuration file and write it to ``/etc/yum.repos.d/``.
Generate a yum configuration file containing a single repository section,
and write it to ``/etc/yum.repos.d/{repositoryid}.repo``.
:param cfg: The system on which to create
a yum configuration file.
:param repositoryid: The section's ``repositoryid``. Used when naming the
configuration file and populating the brackets at the head of the file.
For details, see yum.conf(5).
:param baseurl: The required option ``baseurl`` specifying the url of repo.
For details, see yum.conf(5)
:param name: The required option ``name`` specifying the name of repo.
For details, see yum.conf(5).
:param kwargs: Section options. Each kwarg corresponds to one option. For
details, see yum.conf(5).
:returns: The path to the yum configuration file.
"""
# required repo options
kwargs.setdefault('name', name)
kwargs.setdefault('baseurl', baseurl)
# assume some common used defaults
kwargs.setdefault('enabled', 1)
kwargs.setdefault('gpgcheck', 0)
kwargs.setdefault('metadata_expire', 0) # force metadata load every time
# Check if the settings specifies a content host role else assume ``api``
try:
content_host = cfg.get_hosts('content')[0].roles['content']
except IndexError:
content_host = cfg.get_hosts('api')[0].roles['api']
# if sslverify is not provided in kwargs it is inferred from cfg
kwargs.setdefault(
'sslverify', content_host.get('verify') and 'yes' or 'no'
)
path = os.path.join('/etc/yum.repos.d/', repositoryid + '.repo')
with StringIO() as section:
section.write('[{}]\n'.format(repositoryid))
for key, value in kwargs.items():
section.write('{} = {}\n'.format(key, value))
# machine.session is used here to keep SSH session open
cli.Client(cfg).machine.session().run(
'echo "{}" | {}tee {} > /dev/null'.format(
section.getvalue(),
'' if cli.is_root(cfg) else 'sudo ',
path
)
)
return path
def test_all(self):
"""Use the ``force_full`` RPM rsync distributor option."""
cfg = config.get_config()
cli_client = cli.Client(cfg)
sudo = '' if cli.is_root(cfg) else 'sudo '
# Create a user and repo with an importer and distribs. Sync the repo.
ssh_user, priv_key = self.make_user(cfg)
ssh_identity_file = self.write_private_key(cfg, priv_key)
repo = self.make_repo(
cfg, {
'remote': {
'host': urlparse(cfg.get_base_url()).hostname,
'root': '/home/' + ssh_user,
'ssh_identity_file': ssh_identity_file,
'ssh_user': ssh_user,
}
})
sync_repo(cfg, repo)
# Publish the repo with the yum and rsync distributors, respectively.
# Verify that the RPM rsync distributor has placed files.
distribs = get_dists_by_type_id(cfg, repo)
self.maybe_disable_selinux(cfg, 2199)
for type_id in ('yum_distributor', 'rpm_rsync_distributor'):
publish_repo(cfg, repo, {'id': distribs[type_id]['id']})
self.verify_remote_units_path(cfg, distribs['rpm_rsync_distributor'])
# Remove all files from the target directory, and publish again. Verify
# that the RPM rsync distributor didn't place any files.
cmd = sudo + 'rm -rf /home/{}/content'.format(ssh_user)
cli_client.run(cmd.split())
self.verify_publish_is_skip(
cfg,
publish_repo(cfg, repo, {
'id': distribs['rpm_rsync_distributor']['id']
}).json())
dirs = self.remote_root_files(cfg, distribs['rpm_rsync_distributor'])
self.assertNotIn('content', dirs)
# Publish the repo with ``force_full`` set to true. Verify that the RPM
# rsync distributor placed files.
if not selectors.bug_is_fixed(2202, cfg.pulp_version):
return
publish_repo(
cfg, repo, {
'id': distribs['rpm_rsync_distributor']['id'],
'override_config': {
'force_full': True
},
})
self.verify_remote_units_path(cfg, distribs['rpm_rsync_distributor'])
def _gen_content_source(cfg, content_source_body):
"""Write out a content source to a system and return its path.
:param content_source_body: A string that can be used as the body of a
content source file. Consider using :func:`_gen_content_source_body`.
:return: The path to the content source file.
"""
sudo = '' if cli.is_root(cfg) else 'sudo'
path = os.path.join(CONTENT_SOURCES_PATH, utils.uuid4() + '.conf')
client = cli.Client(cfg)
client.machine.session().run("{} bash -c \"echo >'{}' '{}'\"".format(
sudo, path, content_source_body))
return path
def test_all(self):
"""Check for content synced from a feed with PULP_DISTRIBUTION.xml."""
if self.cfg.pulp_version < version.Version('2.11.2'):
self.skipTest(
'PULP_DISTRIBUTION.xml improved parsing is available on Pulp '
'2.11.2+')
client = api.Client(self.cfg, api.json_handler)
distributor = gen_distributor()
distributor['auto_publish'] = True
body = gen_repo()
body['distributors'] = [distributor]
body['importer_config'] = {
'feed': RPM_WITH_PULP_DISTRIBUTION_FEED_URL,
}
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(client.delete, repo['_href'])
sync_repo(self.cfg, repo)
repo = client.get(repo['_href'], params={'details': True})
self.assertEqual(repo['content_unit_counts']['distribution'], 1)
cli_client = cli.Client(self.cfg, cli.code_handler)
relative_url = repo['distributors'][0]['config']['relative_url']
sudo = () if cli.is_root(self.cfg) else ('sudo', )
pulp_distribution = cli_client.run(sudo + (
'cat',
os.path.join(
'/var/lib/pulp/published/yum/http/repos/',
relative_url,
'PULP_DISTRIBUTION.xml',
),
)).stdout
# make sure published repository PULP_DISTRIBUTION.xml does not include
# any extra file from the original repo's PULP_DISTRIBUTION.xml under
# metadata directory
self.assertNotIn('metadata/productid', pulp_distribution)
release_info = cli_client.run(sudo + (
'cat',
os.path.join(
'/var/lib/pulp/published/yum/http/repos/',
relative_url,
'release-notes/release-info',
),
)).stdout
response = requests.get(
urljoin(
urljoin(RPM_WITH_PULP_DISTRIBUTION_FEED_URL, 'release-notes/'),
'release-info',
))
# make sure published repository has extra files outside the metadata
# directory from the origiginal repo's PULP_DISTRIBUTION.xml
self.assertEqual(release_info, response.text)
def setUpClass(cls):
"""Create an RPM repository and issue a task to download the repo.
Do the following:
1. Reset Pulp.
2. Create a repository with the "on demand" download policy.
3. Sync and publish the repository.
4. Trigger a repository download.
5. Corrupt a file in the repository.
6. Trigger a repository download, without unit verification.
7. Trigger a repository download, with unit verification.
"""
super().setUpClass()
if (not selectors.bug_is_fixed(1905, cls.cfg.pulp_version)
and os_is_rhel6(cls.cfg)):
raise unittest.SkipTest('https://pulp.plan.io/issues/1905')
# Ensure Pulp is empty of units otherwise we might just associate pre-
# existing units.
reset_pulp(cls.cfg)
# Create, sync and publish a repository.
repo = _create_repo(cls.cfg, 'on_demand')
cls.resources.add(repo['_href'])
sync_repo(cls.cfg, repo)
# Trigger a repository download. Read the repo before and after.
api_client = api.Client(cls.cfg, api.json_handler)
download_path = urljoin(repo['_href'], 'actions/download/')
params = {'details': True}
cls.repo_pre_download = api_client.get(repo['_href'], params=params)
api_client.post(download_path, {'verify_all_units': False})
cls.repo_post_download = api_client.get(repo['_href'], params=params)
# Corrupt an RPM. The file is there, but the checksum isn't right.
rpm_abs_path = cls.get_rpm_abs_path()
cli_client = cli.Client(cls.cfg)
sudo = '' if cli.is_root(cls.cfg) else 'sudo '
checksum_cmd = (sudo + 'sha256sum ' + rpm_abs_path).split()
cls.sha_pre_corruption = cli_client.run(checksum_cmd).stdout.strip()
cli_client.run((sudo + 'rm ' + rpm_abs_path).split())
cli_client.run((sudo + 'touch ' + rpm_abs_path).split())
cli_client.run((sudo + 'chown apache:apache ' + rpm_abs_path).split())
cls.sha_post_corruption = cli_client.run(checksum_cmd).stdout.strip()
# Trigger repository downloads that don't and do checksum files, resp.
api_client.post(download_path, {'verify_all_units': False})
cls.unverified_file_sha = cli_client.run(checksum_cmd).stdout.strip()
api_client.post(download_path, {'verify_all_units': True})
cls.verified_file_sha = cli_client.run(checksum_cmd).stdout.strip()
def test_all(self):
"""Test whether one can force Pulp to perform a full sync."""
cfg = config.get_config()
if not selectors.bug_is_fixed(1982, cfg.pulp_version):
self.skipTest('https://pulp.plan.io/issues/1982')
# Create and sync a repository.
client = cli.Client(cfg)
repo_id = utils.uuid4()
client.run((
'pulp-admin',
'rpm',
'repo',
'create',
'--repo-id',
repo_id,
'--feed',
RPM_UNSIGNED_FEED_URL,
))
self.addCleanup(client.run, (
'pulp-admin',
'rpm',
'repo',
'delete',
'--repo-id',
repo_id,
))
sync_repo(cfg, repo_id)
# Delete a random RPM from the filesystem.
rpms = self._list_rpms(cfg)
rpm = random.choice(rpms)
cmd = []
if not cli.is_root(cfg):
cmd.append('sudo')
cmd.extend(('rm', '-rf', rpm))
client.run(cmd)
with self.subTest(comment='verify the rpm has been removed'):
self.assertEqual(len(self._list_rpms(cfg)), len(rpms) - 1, rpm)
# Sync the repository without --force-full.
sync_repo(cfg, repo_id)
with self.subTest(comment='verify the rpm has not yet been restored'):
self.assertEqual(len(self._list_rpms(cfg)), len(rpms) - 1, rpm)
# Sync the repository with --force-full.
sync_repo(cfg, repo_id, force_sync=True)
with self.subTest(comment='verify the rpm has been restored'):
self.assertEqual(len(self._list_rpms(cfg)), len(rpms), rpm)
def test_all(self):
"""Update an RPM in a repository and on a host."""
cfg = config.get_config()
if check_issue_3876(cfg):
raise unittest.SkipTest('https://pulp.plan.io/issues/3876')
if check_issue_3104(cfg):
raise unittest.SkipTest('https://pulp.plan.io/issues/3104')
if check_issue_2277(cfg):
raise unittest.SkipTest('https://pulp.plan.io/issues/2277')
if check_issue_2620(cfg):
raise unittest.SkipTest('https://pulp.plan.io/issues/2620')
client = cli.Client(cfg)
pkg_mgr = cli.PackageManager(cfg)
sudo = () if cli.is_root(cfg) else ('sudo', )
verify = cfg.get_hosts('api')[0].roles['api'].get('verify')
# Create the second repository.
repo_id = self.create_repo(cfg)
# Pick an RPM with two versions.
rpm_name = 'walrus'
rpm_versions = _get_rpm_names_versions(cfg, _REPO_ID)[rpm_name]
# Copy the older RPM to the second repository, and publish it.
self._copy_and_publish(cfg, rpm_name, rpm_versions[0], repo_id)
# Install the RPM on a host.
repo_path = gen_yum_config_file(
cfg,
baseurl=urljoin(cfg.get_base_url(), 'pulp/repos/' + repo_id),
enabled=1,
gpgcheck=0,
metadata_expire=0, # force metadata to load every time
name=repo_id,
repositoryid=repo_id,
sslverify='yes' if verify else 'no',
)
self.addCleanup(client.run, sudo + ('rm', repo_path))
pkg_mgr.install(rpm_name)
self.addCleanup(pkg_mgr.uninstall, rpm_name)
client.run(('rpm', '-q', rpm_name))
# Copy the newer RPM to the second repository, and publish it.
self._copy_and_publish(cfg, rpm_name, rpm_versions[1], repo_id)
# Update the installed RPM on the host.
proc = pkg_mgr.upgrade(rpm_name)
self.assertNotIn('Nothing to do.', proc.stdout)
def remote_root_files(cfg, distributor_cfg):
"""Get the directory content of the remote root directory.
Return the directory content of the remote root path configured in the
distributor. Only non-hidden files/directories are returned.
:param cfg: Information about the
system onto which files have been published.
:param distributor_cfg: A dict of information about an RPM rsync
distributor.
:returns: set of file/directory names
"""
path = distributor_cfg['config']['remote']['root']
sudo = () if cli.is_root(cfg) else ('sudo', )
cmd = sudo + ('ls', '-1', path)
return set(cli.Client(cfg).run(cmd).stdout.splitlines())
def test_all(self):
"""Test puppet_install_distributor.
Do the following:
1. Create a puppet repository with a puppet_install_distributor
2. Upload a puppet module
3. Publish the repository
4. Check if the puppet_install_distributor config was properly used
"""
if (not selectors.bug_is_fixed(3314, self.cfg.pulp_version) and
os_is_f27(self.cfg)):
self.skipTest('https://pulp.plan.io/issues/3314')
cli_client = cli.Client(self.cfg)
sudo = () if cli.is_root(self.cfg) else ('sudo',)
# Create a directory and make sure Pulp can write to it.
install_path = cli_client.run(('mktemp', '--directory')).stdout.strip()
self.addCleanup(cli_client.run, sudo + ('rm', '-rf', install_path))
cli_client.run(sudo + ('chown', 'apache:apache', install_path))
cli_client.run(sudo + ('chcon', '-t', 'puppet_etc_t', install_path))
# Make sure the pulp_manage_puppet boolean is enabled
cli_client.run(sudo + ('setsebool', 'pulp_manage_puppet', 'on'))
self.addCleanup(cli_client.run, sudo + (
'setsebool', 'pulp_manage_puppet', 'off'))
# Create and populate a Puppet repository.
distributor = gen_install_distributor()
distributor['distributor_config']['install_path'] = install_path
body = gen_repo()
body['distributors'] = [distributor]
client = api.Client(self.cfg, api.json_handler)
repo = client.post(REPOSITORY_PATH, body)
self.addCleanup(client.delete, repo['_href'])
repo = client.get(repo['_href'], params={'details': True})
unit = utils.http_get(PUPPET_MODULE_URL_1)
upload_import_unit(
self.cfg, unit, {'unit_type_id': 'puppet_module'}, repo)
# Publish, and verify the module is present. (Dir has 700 permissions.)
publish_repo(self.cfg, repo)
proc = cli_client.run(sudo + (
'runuser', '--shell', '/bin/sh', '--command',
'ls -1 {}'.format(install_path), '-', 'apache'
))
self.assertIn(PUPPET_MODULE_1['name'], proc.stdout.split('\n'), proc)
def maybe_disable_selinux(self, cfg, pulp_issue_id):
"""Disable SELinux if appropriate.
If the given Pulp issue is unresolved, and if SELinux is installed and
enforcing on the target Pulp system, then disable SELinux and schedule
it to be re-enabled. (Method ``addCleanup`` is used for the schedule.)
:param cfg: Information about the
Pulp deployment being targeted.
:param pulp_issue_id: The (integer) ID of a `Pulp issue`_. If the
referenced issue is fixed in the Pulp system under test, this
method immediately returns.
:returns: Nothing.
.. _Pulp issue: https://pulp.plan.io/issues/
"""
# Abort if the Pulp issue is resolved, if SELinux is not installed or
# if SELinux is not enforcing.
#
# NOTE: Hard-coding the absolute path to a command is a Bad Idea™.
# However, non-login non-root shells may have short PATH environment
# variables. For example:
#
# /usr/lib64/qt-3.3/bin:/usr/local/bin:/usr/bin
#
# We cannot execute `PATH=${PATH}:/usr/sbin which getenforce` because
# Plumbum does a good job of preventing shell expansions. See:
# https://github.com/PulpQE/pulp-smash/issues/89
if selectors.bug_is_fixed(pulp_issue_id, cfg.pulp_version):
return
client = cli.Client(cfg, cli.echo_handler)
cmd = 'test -e /usr/sbin/getenforce'.split()
if client.run(cmd).returncode != 0:
return
client.response_handler = cli.code_handler
cmd = ['/usr/sbin/getenforce']
if client.run(cmd).stdout.strip().lower() != 'enforcing':
return
# Temporarily disable SELinux.
sudo = '' if cli.is_root(cfg) else 'sudo '
cmd = (sudo + 'setenforce 0').split()
client.run(cmd)
cmd = (sudo + 'setenforce 1').split()
self.addCleanup(client.run, cmd)
def write_manifest_list(cfg, manifest_list):
"""Write out a content source to JSON file.
:param cfg: The Pulp deployment on
which to create a repository.
:param manifest_list: A detailed dict of information about the manifest
list.
:return: The path to created file, and the path to dir that stores the
file.
"""
sudo = '' if cli.is_root(cfg) else 'sudo'
client = cli.Client(cfg)
dir_path = client.run('mktemp --directory'.split()).stdout.strip()
file_path = os.path.join(dir_path, utils.uuid4() + '.json')
manifest_list_json = json.dumps(manifest_list)
# machine.session is used here to keep SSH session open
client.machine.session().run("{} echo '{}' > {}".format(
sudo, manifest_list_json, file_path))
return file_path, dir_path
def reset_squid(server_config):
"""Stop Squid, reset its cache directory, and restart it.
:param pulp_smash.config.ServerConfig server_config: Information about the
Pulp server being targeted.
:returns: Nothing.
"""
squid_service = cli.Service(server_config, 'squid')
squid_service.stop()
# Clean out the cache directory and reinitialize it.
client = cli.Client(server_config)
prefix = '' if is_root(server_config) else 'sudo '
client.run((prefix + 'rm -rf /var/spool/squid').split())
client.run((prefix + 'mkdir --context=system_u:object_r:squid_cache_t:s0' +
' --mode=750 /var/spool/squid').split())
client.run((prefix + 'chown squid:squid /var/spool/squid').split())
client.run((prefix + 'squid -z').split())
squid_service.start()
def _create_repo_file(self, cfg, repo):
"""Create a file in ``/etc/yum.repos.d/`` referencing the repository.
Also, schedule it for deletion. Return nothing.
"""
verify = cfg.get_hosts('api')[0].roles['api'].get('verify')
sudo = () if cli.is_root(cfg) else ('sudo', )
repo_path = gen_yum_config_file(
cfg,
baseurl=urljoin(
cfg.get_base_url(),
urljoin('pulp/repos/',
repo['distributors'][0]['config']['relative_url'])),
name=repo['_href'],
enabled=1,
gpgcheck=0,
metadata_expire=0, # force metadata to load every time
repositoryid=repo['id'],
sslverify='yes' if verify else 'no',
)
self.addCleanup(cli.Client(cfg).run, sudo + ('rm', repo_path))
def _make_user(cfg):
"""Create a user account on a target system.
This method is implemented as a generator. When executed, it will yield
a username and private key. The corresponding public key is made the
one and only key in the user's ``authorized_keys`` file.
The username and private key are yielded one-by-one rather than as a
pair because the user creation and key creation steps are executed
serially. Should the latter fail, the calling function will still be
able to delete the created user.
The user is given a home directory. When deleting this user, make sure
to pass ``--remove`` to ``userdel``. Otherwise, the home directory will
be left in place.
"""
client = cli.Client(cfg)
sudo = '' if cli.is_root(cfg) else 'sudo '
# According to useradd(8), usernames may be up to 32 characters long.
# But long names break the rsync publish process: (SNIP == username)
#
# unix_listener:
# "/tmp/rsync_distributor-[SNIP]@example.com:22.64tcAiD8em417CiN"
# too long for Unix domain socket
#
username = utils.uuid4()[:12]
cmd = 'useradd --create-home {0}'
client.run((sudo + cmd.format(username)).split())
yield username
cmd = 'runuser --shell /bin/sh {} --command'.format(username)
cmd = (sudo + cmd).split()
cmd.append('ssh-keygen -N "" -f /home/{}/.ssh/mykey'.format(username))
client.run(cmd)
cmd = 'cp /home/{0}/.ssh/mykey.pub /home/{0}/.ssh/authorized_keys'
client.run((sudo + cmd.format(username)).split())
cmd = 'cat /home/{0}/.ssh/mykey'
private_key = client.run((sudo + cmd.format(username)).split()).stdout
yield private_key
def _do_test(self, file_, label, recursive=False):
"""Assert that certain files have a label of ``label``.
Get the SELinux label of the given ``file_``, or all files rooted at
``file_` if ``recursive`` is true. For each SELinux label, strip off
the leading "user" portion of the label, and assert that the result — a
string in the form :role:type:level — has the given ``label``.
"""
# Typical output:
#
# # getfattr --name=security.selinux /etc/passwd
# getfattr: Removing leading '/' from absolute path names
# # file: etc/passwd
# security.selinux="system_u:object_r:passwd_file_t:s0"
#
cmd = [] if cli.is_root(config.get_config()) else ['sudo']
cmd.extend(('getfattr', '--name=security.selinux'))
if recursive:
cmd.append('--recursive')
cmd.append(file_)
lines = self.client.run(cmd).stdout.splitlines()
matches = 0
getfattr_file = None # tracks file currently under consideration
for line in lines:
match = self.file_matcher.match(line)
if match is not None:
getfattr_file = match.groups(1)
continue
match = self.label_matcher.match(line)
if match is not None:
matches += 1
# Strip "user" prefix from label. For example:
# user:role:type:level → :role:type:level
file_label = match.group(1)
file_label = file_label[file_label.find(':'):]
self.assertEqual(file_label, label, getfattr_file)
self.assertGreater(matches, 0, lines)
def delete_user(cfg, username):
"""Delete a user.
The Pulp rsync distributor has a habit of leaving (idle?) SSH sessions
open even after publishing a repository. When executed, this function
will:
1. Poll the process list until all processes belonging to ``username``
have died, or raise a ``unittest.SkipTest`` exception if the time
limit is exceeded.
2. Delete ``username``.
"""
sudo = () if cli.is_root(cfg) else ('sudo',)
client = cli.Client(cfg)
# values are arbitrary
iter_time = 2 # seconds
iter_limit = 15 # unitless
# Wait for user's processes to die.
cmd = sudo + ('ps', '-wwo', 'args', '--user', username, '--no-headers')
i = 0
while i <= iter_limit:
try:
user_processes = client.run(cmd).stdout.splitlines()
except exceptions.CalledProcessError:
break
i += 1
time.sleep(iter_time)
else:
raise unittest.SkipTest(
'User still has processes running after {}+ seconds. Aborting '
'test. User processes: {}'
.format(iter_time * iter_limit, user_processes)
)
# Delete user.
cmd = sudo + ('userdel', '--remove', username)
client.run(cmd)
def test_all(self):
"""Delete an artifact, it is removed from the filesystem.
Do the following:
1. Create an artifact, and verify it is present on the filesystem.
2. Delete the artifact, and verify it is absent on the filesystem.
"""
cfg = config.get_config()
api_client = api.Client(cfg, api.json_handler)
cli_client = cli.Client(cfg)
# create
files = {'file': utils.http_get(FILE_URL)}
artifact = api_client.post(ARTIFACTS_PATH, files=files)
self.addCleanup(api_client.delete, artifact['_href'])
sudo = () if cli.is_root(cfg) else ('sudo',)
cmd = sudo + ('ls', artifact['file'])
cli_client.run(cmd)
# delete
self.doCleanups()
with self.assertRaises(CalledProcessError):
cli_client.run(cmd)
def setUpClass(cls):
"""Create class-wide variables."""
cls.cfg = config.get_config()
cls.api_client = api.Client(cls.cfg, api.json_handler)
cls.cli_client = cli.Client(cls.cfg)
cls.sudo = () if cli.is_root(cls.cfg) else ('sudo',)
def test_negative(self):
"""Test what happens when we aren't root on the target host."""
with mock.patch.object(cli, 'Client') as client:
client.return_value.run.return_value.stdout = ' 1 '
self.assertFalse(cli.is_root(mock.MagicMock()))
def test_true(self):
"""Assert the method returns ``True`` when root."""
with mock.patch.object(cli, 'Client') as clien:
clien.return_value.run.return_value.stdout.strip.return_value = '0'
self.assertTrue(cli.is_root(None))
def test_false(self):
"""Assert the method returns ``False`` when non-root."""
with mock.patch.object(cli, 'Client') as clien:
clien.return_value.run.return_value.stdout.strip.return_value = '1'
self.assertFalse(cli.is_root(None))
