def test_init(self):
config = PluginCallConfiguration(None, {
constants.PUBLISH_HTTPS_KEYWORD: True,
constants.PUBLISH_HTTP_KEYWORD: True})
step = publish.Publisher(self.publisher.get_repo(),
self.publisher.get_conduit(),
config, YUM_DISTRIBUTOR_ID)
self.assertTrue(isinstance(step.children[-1], publish.GenerateListingFileStep))
self.assertTrue(isinstance(step.children[-2], publish.GenerateListingFileStep))
self.assertTrue(isinstance(step.children[-3], publish.AtomicDirectoryPublishStep))
atomic_publish = step.children[-3]
repo = self.publisher.get_repo()
target_publish_locations = [
['/', os.path.join(configuration.get_https_publish_dir(config),
configuration.get_repo_relative_path(repo, config))],
['/', os.path.join(configuration.get_http_publish_dir(config),
configuration.get_repo_relative_path(repo, config))]
]
self.assertEquals(atomic_publish.publish_locations, target_publish_locations)
# verify that listing file steps got the correct arguments
http_step = step.children[-1]
https_step = step.children[-2]
self.assertEqual(http_step.target_dir, target_publish_locations[1][1])
self.assertEqual(http_step.root_dir, configuration.get_http_publish_dir(config))
self.assertEqual(https_step.target_dir, target_publish_locations[0][1])
self.assertEqual(https_step.root_dir, configuration.get_https_publish_dir(config))
def test_init(self):
config = PluginCallConfiguration(None, {
constants.PUBLISH_HTTPS_KEYWORD: True,
constants.PUBLISH_HTTP_KEYWORD: True})
step = publish.Publisher(self.publisher.get_repo(),
self.publisher.get_conduit(),
config, YUM_DISTRIBUTOR_ID, working_dir=self.working_dir)
self.assertTrue(isinstance(step.children[-1], publish.GenerateListingFileStep))
self.assertTrue(isinstance(step.children[-2], publish.GenerateListingFileStep))
self.assertTrue(isinstance(step.children[-3], publish.AtomicDirectoryPublishStep))
atomic_publish = step.children[-3]
repo = self.publisher.get_repo()
target_publish_locations = [
['/', os.path.join(configuration.get_https_publish_dir(config),
configuration.get_repo_relative_path(repo, config))],
['/', os.path.join(configuration.get_http_publish_dir(config),
configuration.get_repo_relative_path(repo, config))]
]
self.assertEquals(atomic_publish.publish_locations, target_publish_locations)
# verify that listing file steps got the correct arguments
http_step = step.children[-1]
https_step = step.children[-2]
self.assertEqual(http_step.target_dir, target_publish_locations[1][1])
self.assertEqual(http_step.root_dir, configuration.get_http_publish_dir(config))
self.assertEqual(https_step.target_dir, target_publish_locations[0][1])
self.assertEqual(https_step.root_dir, configuration.get_https_publish_dir(config))
def _add_necesary_steps(self, date_filter=None, config=None):
"""
This method adds all the steps that are needed to accomplish an ISO rsync publish. This
includes:
Unit Query Step - selects units associated with the repo based on the date_filter and
creates relative symlinks
Rsync Step (content units) - rsyncs units discovered in previous step to the remote server
Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server
Rsync Step (PULP_MANIFEST) - rsyncs PULP_MANIFEST to remote server
:param date_filter: Q object with start and/or end dates, or None if start and end dates
are not provided
:type date_filter: mongoengine.Q or types.NoneType
:param config: Pulp configuration for the distributor
:type config: pulp.plugins.config.PluginCallConfiguration
"""
remote_repo_path = get_repo_relative_path(self.repo.repo_obj, self.predistributor.config)
# Find all the units associated with repo before last publish with predistributor
gen_step = RSyncFastForwardUnitPublishStep("Unit query step (ISO)",
ISORsyncPublisher.REPO_CONTENT_MODELS,
repo=self.repo,
repo_content_unit_q=date_filter,
remote_repo_path=remote_repo_path,
published_unit_path=[],
unit_fields=ISORsyncPublisher.UNIT_FIELDS)
self.add_child(gen_step)
dest_content_units_dir = self.get_units_directory_dest_path()
src_content_units_dir = self.get_units_src_path()
# Rsync content units
self.add_child(RSyncPublishStep("Rsync step (content units)", self.content_unit_file_list,
src_content_units_dir, dest_content_units_dir,
config=config, exclude=[]))
# Stop here if distributor is only supposed to publish actual content
if self.get_config().flatten().get("content_units_only"):
return
# Rsync symlinks to the remote server
self.add_child(RSyncPublishStep("Rsync step (symlinks)",
self.symlink_list, self.symlink_src,
remote_repo_path,
config=config, links=True, exclude=["PULP_MANIFEST"],
delete=self.config.get("delete")))
predistributor_master_dir = self.get_master_directory()
# Rsync PULP_MANIFEST
self.add_child(RSyncPublishStep("Rsync step (PULP_MANIFEST)",
['PULP_MANIFEST'], predistributor_master_dir,
remote_repo_path,
config=config))
if self.predistributor:
self.add_child(UpdateLastPredistDateStep(self.distributor,
self.predistributor["last_publish"]))
def get_hosting_locations(self, repo, config):
"""
Get the paths on the filesystem where the build directory should be copied
:param repo: The repository that is going to be hosted
:type repo: pulp.plugins.model.Repository
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return : list of paths on the filesystem where the build directory should be copied
:rtype list of str
"""
hosting_locations = []
repo_relative_path = get_repo_relative_path(repo.repo_obj, config)
# Publish the HTTP portion, if applicable
http_dest_dir = os.path.join(constants.ISO_HTTP_DIR, repo_relative_path)
serve_http = config.get_boolean(constants.CONFIG_SERVE_HTTP)
serve_http = serve_http if serve_http is not None else constants.CONFIG_SERVE_HTTP_DEFAULT
if serve_http:
hosting_locations.append(http_dest_dir)
# Publish the HTTPs portion, if applicable
if self._is_https_supported(config):
https_dest_dir = os.path.join(constants.ISO_HTTPS_DIR, repo_relative_path)
hosting_locations.append(https_dest_dir)
return hosting_locations
def get_hosting_locations(self, repo, config):
"""
Get the paths on the filesystem where the build directory should be copied
:param repo: The repository that is going to be hosted
:type repo: pulp.plugins.model.Repository
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return : list of paths on the filesystem where the build directory should be copied
:rtype list of str
"""
hosting_locations = []
repo_relative_path = get_repo_relative_path(repo.repo_obj, config)
# Publish the HTTP portion, if applicable
http_dest_dir = os.path.join(constants.ISO_HTTP_DIR,
repo_relative_path)
serve_http = config.get_boolean(constants.CONFIG_SERVE_HTTP)
serve_http = serve_http if serve_http is not None else constants.CONFIG_SERVE_HTTP_DEFAULT
if serve_http:
hosting_locations.append(http_dest_dir)
# Publish the HTTPs portion, if applicable
if self._is_https_supported(config):
https_dest_dir = os.path.join(constants.ISO_HTTPS_DIR,
repo_relative_path)
hosting_locations.append(https_dest_dir)
return hosting_locations
def test_get_repo_relative_path_repo_id(self):
repo_id = 'Highlander'
repo = Repository(repo_id)
config = self._generate_call_config()
relative_dir = configuration.get_repo_relative_path(repo, config)
self.assertEqual(relative_dir, repo_id)
def test_get_repo_relative_path_repo_id(self):
repo_id = 'Highlander'
repo = Repository(repo_id)
config = self._generate_call_config()
relative_dir = configuration.get_repo_relative_path(repo, config)
self.assertEqual(relative_dir, repo_id)
def _add_necesary_steps(self, date_filter=None, config=None):
"""
This method adds all the steps that are needed to accomplish an ISO rsync publish. This
includes:
Unit Query Step - selects units associated with the repo based on the date_filter and
creates relative symlinks
Rsync Step (content units) - rsyncs units discovered in previous step to the remote server
Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server
Rsync Step (PULP_MANIFEST) - rsyncs PULP_MANIFEST to remote server
:param date_filter: Q object with start and/or end dates, or None if start and end dates
are not provided
:type date_filter: mongoengine.Q or types.NoneType
:param config: Pulp configuration for the distributor
:type config: pulp.plugins.config.PluginCallConfiguration
"""
predistributor = self._get_predistributor()
remote_repo_path = get_repo_relative_path(self.repo.repo_obj, predistributor.config)
# Find all the units associated with repo before last publish with predistributor
gen_step = RSyncFastForwardUnitPublishStep("Unit query step (ISO)",
ISORsyncPublisher.REPO_CONTENT_MODELS,
repo=self.repo,
repo_content_unit_q=date_filter,
remote_repo_path=remote_repo_path,
published_unit_path=[],
unit_fields=ISORsyncPublisher.UNIT_FIELDS)
self.add_child(gen_step)
dest_content_units_dir = self.get_units_directory_dest_path()
src_content_units_dir = self.get_units_src_path()
# Rsync content units
self.add_child(RSyncPublishStep("Rsync step (content units)", self.content_unit_file_list,
src_content_units_dir, dest_content_units_dir,
config=config, exclude=[]))
# Stop here if distributor is only supposed to publish actual content
if self.get_config().flatten().get("content_units_only"):
return
# Rsync symlinks to the remote server
self.add_child(RSyncPublishStep("Rsync step (symlinks)",
self.symlink_list, self.symlink_src,
remote_repo_path,
config=config, links=True, exclude=["PULP_MANIFEST"],
delete=self.config.get("delete")))
predistributor_master_dir = self.get_master_directory()
# Rsync PULP_MANIFEST
self.add_child(RSyncPublishStep("Rsync step (PULP_MANIFEST)",
['PULP_MANIFEST'], predistributor_master_dir,
remote_repo_path,
config=config))
def test_get_repo_relative_path_configured(self):
repo_id = 'Spaniard'
repo = Repository(repo_id)
configured_relative_url = '/there/can/be/only/one/'
config = self._generate_call_config(relative_url=configured_relative_url)
relative_url = configuration.get_repo_relative_path(repo, config)
# get_relative_path should strip off the leading '/'
self.assertEqual(relative_url, configured_relative_url[1:])
def test_get_repo_relative_path_configured(self):
repo_id = 'Spaniard'
repo = Repository(repo_id)
configured_relative_url = '/there/can/be/only/one/'
config = self._generate_call_config(relative_url=configured_relative_url)
relative_url = configuration.get_repo_relative_path(repo, config)
# get_relative_path should strip off the leading '/'
self.assertEqual(relative_url, configured_relative_url[1:])
def test_init_incremental_publish_from_http_dir(self, mock_get_http_dir):
config = PluginCallConfiguration(None, {
constants.PUBLISH_HTTPS_KEYWORD: False,
constants.PUBLISH_HTTP_KEYWORD: True})
# Set the last publish time
self.publisher.get_conduit().last_publish = \
mock.Mock(return_value=datetime.datetime.now(tz=isodate.UTC))
# set up the previous publish directory
repo = self.publisher.get_repo()
mock_get_http_dir.return_value = self.working_dir
specific_master = os.path.join(self.working_dir,
configuration.get_repo_relative_path(repo, config))
os.makedirs(specific_master)
step = publish.Publisher(self.publisher.get_repo(),
self.publisher.get_conduit(),
config, YUM_DISTRIBUTOR_ID)
self.assertTrue(isinstance(step.children[0], publish.CopyDirectoryStep))
def test_init_incremental_publish_from_http_dir(self, mock_get_http_dir):
config = PluginCallConfiguration(None, {
constants.PUBLISH_HTTPS_KEYWORD: False,
constants.PUBLISH_HTTP_KEYWORD: True})
# Set the last publish time
self.publisher.get_conduit().last_publish = \
mock.Mock(return_value=datetime.datetime.now(tz=isodate.UTC))
# set up the previous publish directory
repo = self.publisher.get_repo()
mock_get_http_dir.return_value = self.working_dir
specific_master = os.path.join(self.working_dir,
configuration.get_repo_relative_path(repo, config))
os.makedirs(specific_master)
step = publish.Publisher(self.publisher.get_repo(),
self.publisher.get_conduit(),
config, YUM_DISTRIBUTOR_ID, working_dir=self.working_dir)
self.assertTrue(isinstance(step.children[0], publish.CopyDirectoryStep))
def _add_necesary_steps(self, date_filter=None, config=None):
"""
This method adds all the steps that are needed to accomplish an RPM rsync publish. This
includes:
Unit Query Step - selects units associated with the repo based on the date_filter and
creates relative symlinks
Rsync Step (origin) - rsyncs units discovered in previous step to the remote server
Rsync Step (repodata) - rsyncs repodata from master publish directory to remote server
Rsync Step (content) - rsyncs symlinks from working directory to remote server
:param date_filter: Q object with start and/or end dates, or None if start and end dates
are not provided
:type date_filter: mongoengine.Q or types.NoneType
:param config: distributor configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:return: None
"""
remote_repo_path = yum_config.get_repo_relative_path(self.repo.repo_obj,
self.predistributor.config)
# Find all the units associated with repo before last publish with predistributor
unit_types = ', '.join(RPMRsyncPublisher.REPO_CONTENT_TYPES)
gen_step = RSyncFastForwardUnitPublishStep("Unit query step (%s)" % unit_types,
RPMRsyncPublisher.REPO_CONTENT_MODELS,
repo=self.repo,
repo_content_unit_q=date_filter,
remote_repo_path=remote_repo_path,
published_unit_path=[],
unit_fields=RPMRsyncPublisher.UNIT_FIELDS)
self.add_child(gen_step)
dest_content_units_dir = self.get_units_directory_dest_path()
src_content_units_dir = self.get_units_src_path()
# Rsync content units
self.add_child(RSyncPublishStep("Rsync step (origin)", self.content_unit_file_list,
src_content_units_dir, dest_content_units_dir,
config=config, exclude=[".*", "repodata.old"]))
# Stop here if distributor is only supposed to publish actual content
if self.get_config().flatten().get("content_units_only"):
return
master_dir = self.get_master_directory()
# Rsync symlinks to the remote server
self.add_child(RSyncPublishStep("Rsync step (content)",
self.symlink_list, self.symlink_src,
remote_repo_path,
config=config, links=True, exclude=["repodata"],
delete=self.config.get("delete")))
repodata_file_list = os.listdir(os.path.join(master_dir, 'repodata'))
# Only rsync repodata if distributor is configured to do so
if not self.get_config().get('skip_repodata'):
self.add_child(RSyncPublishStep("Rsync step (repodata)",
repodata_file_list,
"%s/" % os.path.join(master_dir, 'repodata'),
"%s/" % os.path.join(remote_repo_path, 'repodata'),
exclude=[".*", "repodata.old"],
config=config, links=True,
delete=self.config.get("delete")))
if self.predistributor:
self.add_child(UpdateLastPredistDateStep(self.distributor,
self.predistributor["last_publish"]))