def test_init(self): config = PluginCallConfiguration(None, { constants.PUBLISH_HTTPS_KEYWORD: True, constants.PUBLISH_HTTP_KEYWORD: True}) step = publish.Publisher(self.publisher.get_repo(), self.publisher.get_conduit(), config, YUM_DISTRIBUTOR_ID) self.assertTrue(isinstance(step.children[-1], publish.GenerateListingFileStep)) self.assertTrue(isinstance(step.children[-2], publish.GenerateListingFileStep)) self.assertTrue(isinstance(step.children[-3], publish.AtomicDirectoryPublishStep)) atomic_publish = step.children[-3] repo = self.publisher.get_repo() target_publish_locations = [ ['/', os.path.join(configuration.get_https_publish_dir(config), configuration.get_repo_relative_path(repo, config))], ['/', os.path.join(configuration.get_http_publish_dir(config), configuration.get_repo_relative_path(repo, config))] ] self.assertEquals(atomic_publish.publish_locations, target_publish_locations) # verify that listing file steps got the correct arguments http_step = step.children[-1] https_step = step.children[-2] self.assertEqual(http_step.target_dir, target_publish_locations[1][1]) self.assertEqual(http_step.root_dir, configuration.get_http_publish_dir(config)) self.assertEqual(https_step.target_dir, target_publish_locations[0][1]) self.assertEqual(https_step.root_dir, configuration.get_https_publish_dir(config))
def test_init(self): config = PluginCallConfiguration(None, { constants.PUBLISH_HTTPS_KEYWORD: True, constants.PUBLISH_HTTP_KEYWORD: True}) step = publish.Publisher(self.publisher.get_repo(), self.publisher.get_conduit(), config, YUM_DISTRIBUTOR_ID, working_dir=self.working_dir) self.assertTrue(isinstance(step.children[-1], publish.GenerateListingFileStep)) self.assertTrue(isinstance(step.children[-2], publish.GenerateListingFileStep)) self.assertTrue(isinstance(step.children[-3], publish.AtomicDirectoryPublishStep)) atomic_publish = step.children[-3] repo = self.publisher.get_repo() target_publish_locations = [ ['/', os.path.join(configuration.get_https_publish_dir(config), configuration.get_repo_relative_path(repo, config))], ['/', os.path.join(configuration.get_http_publish_dir(config), configuration.get_repo_relative_path(repo, config))] ] self.assertEquals(atomic_publish.publish_locations, target_publish_locations) # verify that listing file steps got the correct arguments http_step = step.children[-1] https_step = step.children[-2] self.assertEqual(http_step.target_dir, target_publish_locations[1][1]) self.assertEqual(http_step.root_dir, configuration.get_http_publish_dir(config)) self.assertEqual(https_step.target_dir, target_publish_locations[0][1]) self.assertEqual(https_step.root_dir, configuration.get_https_publish_dir(config))
def _add_necesary_steps(self, date_filter=None, config=None): """ This method adds all the steps that are needed to accomplish an ISO rsync publish. This includes: Unit Query Step - selects units associated with the repo based on the date_filter and creates relative symlinks Rsync Step (content units) - rsyncs units discovered in previous step to the remote server Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server Rsync Step (PULP_MANIFEST) - rsyncs PULP_MANIFEST to remote server :param date_filter: Q object with start and/or end dates, or None if start and end dates are not provided :type date_filter: mongoengine.Q or types.NoneType :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration """ remote_repo_path = get_repo_relative_path(self.repo.repo_obj, self.predistributor.config) # Find all the units associated with repo before last publish with predistributor gen_step = RSyncFastForwardUnitPublishStep("Unit query step (ISO)", ISORsyncPublisher.REPO_CONTENT_MODELS, repo=self.repo, repo_content_unit_q=date_filter, remote_repo_path=remote_repo_path, published_unit_path=[], unit_fields=ISORsyncPublisher.UNIT_FIELDS) self.add_child(gen_step) dest_content_units_dir = self.get_units_directory_dest_path() src_content_units_dir = self.get_units_src_path() # Rsync content units self.add_child(RSyncPublishStep("Rsync step (content units)", self.content_unit_file_list, src_content_units_dir, dest_content_units_dir, config=config, exclude=[])) # Stop here if distributor is only supposed to publish actual content if self.get_config().flatten().get("content_units_only"): return # Rsync symlinks to the remote server self.add_child(RSyncPublishStep("Rsync step (symlinks)", self.symlink_list, self.symlink_src, remote_repo_path, config=config, links=True, exclude=["PULP_MANIFEST"], delete=self.config.get("delete"))) predistributor_master_dir = self.get_master_directory() # Rsync PULP_MANIFEST self.add_child(RSyncPublishStep("Rsync step (PULP_MANIFEST)", ['PULP_MANIFEST'], predistributor_master_dir, remote_repo_path, config=config)) if self.predistributor: self.add_child(UpdateLastPredistDateStep(self.distributor, self.predistributor["last_publish"]))
def get_hosting_locations(self, repo, config): """ Get the paths on the filesystem where the build directory should be copied :param repo: The repository that is going to be hosted :type repo: pulp.plugins.model.Repository :param config: plugin configuration :type config: pulp.plugins.config.PluginConfiguration :return : list of paths on the filesystem where the build directory should be copied :rtype list of str """ hosting_locations = [] repo_relative_path = get_repo_relative_path(repo.repo_obj, config) # Publish the HTTP portion, if applicable http_dest_dir = os.path.join(constants.ISO_HTTP_DIR, repo_relative_path) serve_http = config.get_boolean(constants.CONFIG_SERVE_HTTP) serve_http = serve_http if serve_http is not None else constants.CONFIG_SERVE_HTTP_DEFAULT if serve_http: hosting_locations.append(http_dest_dir) # Publish the HTTPs portion, if applicable if self._is_https_supported(config): https_dest_dir = os.path.join(constants.ISO_HTTPS_DIR, repo_relative_path) hosting_locations.append(https_dest_dir) return hosting_locations
def test_get_repo_relative_path_repo_id(self): repo_id = 'Highlander' repo = Repository(repo_id) config = self._generate_call_config() relative_dir = configuration.get_repo_relative_path(repo, config) self.assertEqual(relative_dir, repo_id)
def _add_necesary_steps(self, date_filter=None, config=None): """ This method adds all the steps that are needed to accomplish an ISO rsync publish. This includes: Unit Query Step - selects units associated with the repo based on the date_filter and creates relative symlinks Rsync Step (content units) - rsyncs units discovered in previous step to the remote server Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server Rsync Step (PULP_MANIFEST) - rsyncs PULP_MANIFEST to remote server :param date_filter: Q object with start and/or end dates, or None if start and end dates are not provided :type date_filter: mongoengine.Q or types.NoneType :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration """ predistributor = self._get_predistributor() remote_repo_path = get_repo_relative_path(self.repo.repo_obj, predistributor.config) # Find all the units associated with repo before last publish with predistributor gen_step = RSyncFastForwardUnitPublishStep("Unit query step (ISO)", ISORsyncPublisher.REPO_CONTENT_MODELS, repo=self.repo, repo_content_unit_q=date_filter, remote_repo_path=remote_repo_path, published_unit_path=[], unit_fields=ISORsyncPublisher.UNIT_FIELDS) self.add_child(gen_step) dest_content_units_dir = self.get_units_directory_dest_path() src_content_units_dir = self.get_units_src_path() # Rsync content units self.add_child(RSyncPublishStep("Rsync step (content units)", self.content_unit_file_list, src_content_units_dir, dest_content_units_dir, config=config, exclude=[])) # Stop here if distributor is only supposed to publish actual content if self.get_config().flatten().get("content_units_only"): return # Rsync symlinks to the remote server self.add_child(RSyncPublishStep("Rsync step (symlinks)", self.symlink_list, self.symlink_src, remote_repo_path, config=config, links=True, exclude=["PULP_MANIFEST"], delete=self.config.get("delete"))) predistributor_master_dir = self.get_master_directory() # Rsync PULP_MANIFEST self.add_child(RSyncPublishStep("Rsync step (PULP_MANIFEST)", ['PULP_MANIFEST'], predistributor_master_dir, remote_repo_path, config=config))
def test_get_repo_relative_path_configured(self): repo_id = 'Spaniard' repo = Repository(repo_id) configured_relative_url = '/there/can/be/only/one/' config = self._generate_call_config(relative_url=configured_relative_url) relative_url = configuration.get_repo_relative_path(repo, config) # get_relative_path should strip off the leading '/' self.assertEqual(relative_url, configured_relative_url[1:])
def test_init_incremental_publish_from_http_dir(self, mock_get_http_dir): config = PluginCallConfiguration(None, { constants.PUBLISH_HTTPS_KEYWORD: False, constants.PUBLISH_HTTP_KEYWORD: True}) # Set the last publish time self.publisher.get_conduit().last_publish = \ mock.Mock(return_value=datetime.datetime.now(tz=isodate.UTC)) # set up the previous publish directory repo = self.publisher.get_repo() mock_get_http_dir.return_value = self.working_dir specific_master = os.path.join(self.working_dir, configuration.get_repo_relative_path(repo, config)) os.makedirs(specific_master) step = publish.Publisher(self.publisher.get_repo(), self.publisher.get_conduit(), config, YUM_DISTRIBUTOR_ID) self.assertTrue(isinstance(step.children[0], publish.CopyDirectoryStep))
def test_init_incremental_publish_from_http_dir(self, mock_get_http_dir): config = PluginCallConfiguration(None, { constants.PUBLISH_HTTPS_KEYWORD: False, constants.PUBLISH_HTTP_KEYWORD: True}) # Set the last publish time self.publisher.get_conduit().last_publish = \ mock.Mock(return_value=datetime.datetime.now(tz=isodate.UTC)) # set up the previous publish directory repo = self.publisher.get_repo() mock_get_http_dir.return_value = self.working_dir specific_master = os.path.join(self.working_dir, configuration.get_repo_relative_path(repo, config)) os.makedirs(specific_master) step = publish.Publisher(self.publisher.get_repo(), self.publisher.get_conduit(), config, YUM_DISTRIBUTOR_ID, working_dir=self.working_dir) self.assertTrue(isinstance(step.children[0], publish.CopyDirectoryStep))
def _add_necesary_steps(self, date_filter=None, config=None): """ This method adds all the steps that are needed to accomplish an RPM rsync publish. This includes: Unit Query Step - selects units associated with the repo based on the date_filter and creates relative symlinks Rsync Step (origin) - rsyncs units discovered in previous step to the remote server Rsync Step (repodata) - rsyncs repodata from master publish directory to remote server Rsync Step (content) - rsyncs symlinks from working directory to remote server :param date_filter: Q object with start and/or end dates, or None if start and end dates are not provided :type date_filter: mongoengine.Q or types.NoneType :param config: distributor configuration :type config: pulp.plugins.config.PluginCallConfiguration :return: None """ remote_repo_path = yum_config.get_repo_relative_path(self.repo.repo_obj, self.predistributor.config) # Find all the units associated with repo before last publish with predistributor unit_types = ', '.join(RPMRsyncPublisher.REPO_CONTENT_TYPES) gen_step = RSyncFastForwardUnitPublishStep("Unit query step (%s)" % unit_types, RPMRsyncPublisher.REPO_CONTENT_MODELS, repo=self.repo, repo_content_unit_q=date_filter, remote_repo_path=remote_repo_path, published_unit_path=[], unit_fields=RPMRsyncPublisher.UNIT_FIELDS) self.add_child(gen_step) dest_content_units_dir = self.get_units_directory_dest_path() src_content_units_dir = self.get_units_src_path() # Rsync content units self.add_child(RSyncPublishStep("Rsync step (origin)", self.content_unit_file_list, src_content_units_dir, dest_content_units_dir, config=config, exclude=[".*", "repodata.old"])) # Stop here if distributor is only supposed to publish actual content if self.get_config().flatten().get("content_units_only"): return master_dir = self.get_master_directory() # Rsync symlinks to the remote server self.add_child(RSyncPublishStep("Rsync step (content)", self.symlink_list, self.symlink_src, remote_repo_path, config=config, links=True, exclude=["repodata"], delete=self.config.get("delete"))) repodata_file_list = os.listdir(os.path.join(master_dir, 'repodata')) # Only rsync repodata if distributor is configured to do so if not self.get_config().get('skip_repodata'): self.add_child(RSyncPublishStep("Rsync step (repodata)", repodata_file_list, "%s/" % os.path.join(master_dir, 'repodata'), "%s/" % os.path.join(remote_repo_path, 'repodata'), exclude=[".*", "repodata.old"], config=config, links=True, delete=self.config.get("delete"))) if self.predistributor: self.add_child(UpdateLastPredistDateStep(self.distributor, self.predistributor["last_publish"]))