def __init__(self, repo, publish_conduit, config): """ :param repo: Pulp managed Yum repository :type repo: pulp.plugins.model.Repository :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration """ super(WebPublisher, self).__init__( step_type=constants.PUBLISH_STEP_WEB_PUBLISHER, repo=repo, publish_conduit=publish_conduit, config=config) docker_api_version = 'v1' publish_dir = configuration.get_web_publish_dir(repo, config, docker_api_version) app_file = configuration.get_redirect_file_name(repo) app_publish_location = os.path.join( configuration.get_app_publish_dir(config, docker_api_version), app_file) self.working_dir = os.path.join(self.get_working_dir(), docker_api_version) misc.mkdir(self.working_dir) self.web_working_dir = os.path.join(self.get_working_dir(), 'web') master_publish_dir = configuration.get_master_publish_dir(repo, config, docker_api_version) atomic_publish_step = AtomicDirectoryPublishStep(self.get_working_dir(), [('web', publish_dir), (app_file, app_publish_location)], master_publish_dir, step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish_step.description = _('Making v1 files available via web.') self.add_child(PublishImagesStep()) self.add_child(atomic_publish_step)
def __init__(self, repo, conduit, config, working_dir=None, **kwargs): """ :param repo: The repository being published. :type repo: pulp.plugins.model.Repository :param conduit: Conduit providing access to relative Pulp functionality :type conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :param working_dir: The temp directory this step should use for processing. :type working_dir: str """ super(WebPublisher, self).__init__(step_type=constants.PUBLISH_STEP_WEB_PUBLISHER, repo=repo, conduit=conduit, config=config, working_dir=working_dir, plugin_type=constants.WEB_DISTRIBUTOR_TYPE_ID, **kwargs) self.publish_dir = os.path.join(self.get_working_dir(), repo.id) atomic_publish = AtomicDirectoryPublishStep( self.get_working_dir(), [(repo.id, configuration.get_web_publish_dir( repo.repo_obj, config))], configuration.get_master_publish_dir(repo.repo_obj, config), step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish.description = _('Making files available via web.') main = MainStep(config=config) self.add_child(main) self.add_child(atomic_publish) mkdir(self.publish_dir)
def __init__(self, repo, conduit, config): """ :param repo: Pulp managed Yum repository :type repo: pulp.plugins.model.Repository :param conduit: Conduit providing access to relative Pulp functionality :type conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration """ super(WebPublisher, self).__init__( constants.PUBLISH_STEP_WEB_PUBLISHER, repo, conduit, config) publish_dir = configuration.get_web_publish_dir(repo, config) self.web_working_dir = os.path.join(self.get_working_dir(), repo.id) master_publish_dir = configuration.get_master_publish_dir(repo, config) atomic_publish = AtomicDirectoryPublishStep( self.get_working_dir(), [(repo.id, publish_dir)], master_publish_dir, step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish.description = _('Making files available via web.') main = MainStep() self.add_child(main) self.add_child(atomic_publish) mkdir(self.web_working_dir)
def __init__(self, repo, publish_conduit, config): """ :param repo: Pulp managed Yum repository :type repo: pulp.plugins.model.Repository :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration """ super(WebPublisher, self).__init__(constants.PUBLISH_STEP_WEB_PUBLISHER, repo, publish_conduit, config) publish_dir = configuration.get_web_publish_dir(repo, config) self.web_working_dir = os.path.join(self.get_working_dir(), repo.id) master_publish_dir = configuration.get_master_publish_dir(repo, config) atomic_publish_step = AtomicDirectoryPublishStep( self.get_working_dir(), [(repo.id, publish_dir)], master_publish_dir, step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish_step.description = _('Making files available via web.') self.add_child(PublishContentStep(working_dir=self.web_working_dir)) self.add_child(PublishMetadataStep(working_dir=self.web_working_dir)) self.add_child(atomic_publish_step)
def __init__(self, repo, publish_conduit, config): """ :param repo: Pulp managed Yum repository :type repo: pulp.plugins.model.Repository :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration """ super(WebPublisher, self).__init__(constants.PUBLISH_STEP_WEB_PUBLISHER, repo, publish_conduit, config) publish_dir = configuration.get_web_publish_dir(repo, config) self.web_working_dir = os.path.join(self.get_working_dir(), repo.id) master_publish_dir = configuration.get_master_publish_dir(repo, config) atomic_publish_step = AtomicDirectoryPublishStep(self.get_working_dir(), [(repo.id, publish_dir)], master_publish_dir, step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish_step.description = _('Making files available via web.') repo = self.get_repo() if not repo.content_unit_counts: self.add_child(CreateEmptyOSTreeStep()) else: os.makedirs(self.web_working_dir) content_step = PublishContentStep(working_dir=self.web_working_dir) self.add_child(content_step) self.add_child(PublishRefsStep(content_step, working_dir=self.web_working_dir)) self.add_child(atomic_publish_step)
def test_process_main_only_publish_directory_contents(self): source_dir = os.path.join(self.working_directory, 'source') master_dir = os.path.join(self.working_directory, 'master') publish_dir = os.path.join(self.working_directory, 'publish', 'bar') publish_dir += '/' step = AtomicDirectoryPublishStep(source_dir, [('/', publish_dir)], master_dir, only_publish_directory_contents=True) step.parent = Mock(timestamp=str(time.time())) # create some files to test sub_file = os.path.join(source_dir, 'bar.html') touch(sub_file) # create an existing file that will be maintained existing_file = os.path.join(source_dir, 'bar.html') touch(existing_file) # Create an old directory to test old_dir = os.path.join(master_dir, 'foo') os.makedirs(old_dir) step.process_main() target_file = os.path.join(publish_dir, 'bar.html') self.assertEquals(True, os.path.exists(target_file)) self.assertTrue(os.path.exists(existing_file)) self.assertEquals(1, len(os.listdir(master_dir)))
def __init__(self, repo, conduit, config, plugin_type, **kwargs): super(Publisher, self).__init__(step_type=constants.PUBLISH_REPO_STEP, repo=repo, conduit=conduit, config=config, plugin_type=plugin_type) self.description = self.__class__.description self.add_child( ModulePublisher(conduit=conduit, config=config, repo=repo)) repo_relative_path = configuration.get_repo_relative_path(repo, config) master_publish_dir = configuration.get_master_publish_dir( repo, plugin_type) target_directories = [] listing_steps = [] if config.get(constants.PUBLISH_HTTP_KEYWORD): root_publish_dir = configuration.get_http_publish_dir(config) repo_publish_dir = os.path.join(root_publish_dir, repo_relative_path) target_directories.append(('/', repo_publish_dir)) listing_steps.append( GenerateListingFileStep(root_publish_dir, repo_publish_dir)) if config.get(constants.PUBLISH_HTTPS_KEYWORD): root_publish_dir = configuration.get_https_publish_dir(config) repo_publish_dir = os.path.join(root_publish_dir, repo_relative_path) target_directories.append(('/', repo_publish_dir)) listing_steps.append( GenerateListingFileStep(root_publish_dir, repo_publish_dir)) atomic_publish_step = AtomicDirectoryPublishStep( self.get_working_dir(), target_directories, master_publish_dir) atomic_publish_step.description = _("Publishing files to web") self.add_child(atomic_publish_step) for step in listing_steps: self.add_child(step)
def __init__(self, repo, publish_conduit, config): """ :param repo: Pulp managed Yum repository. :type repo: pulp.server.db.model.Repository :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration """ super(WebPublisher, self).__init__(constants.PUBLISH_STEP_WEB_PUBLISHER, repo, publish_conduit, config) publish_dir = configuration.get_web_publish_dir(repo, config) app_file = configuration.get_redirect_file_name(repo) app_publish_location = os.path.join(configuration.get_app_publish_dir(config), app_file) self.web_working_dir = os.path.join(self.get_working_dir(), 'web') master_publish_dir = configuration.get_master_publish_dir(repo, config) atomic_publish_step = AtomicDirectoryPublishStep(self.get_working_dir(), [('web', publish_dir), (app_file, app_publish_location)], master_publish_dir, step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish_step.description = _('Making files available via web.') self.add_child(PublishImagesStep()) self.add_child(atomic_publish_step)
def __init__(self, repo, conduit, config, working_dir=None, **kwargs): """ :param repo: The repository being published. :type repo: pulp.plugins.model.Repository :param conduit: Conduit providing access to relative Pulp functionality :type conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :param working_dir: The temp directory this step should use for processing. :type working_dir: str """ super(WebPublisher, self).__init__( step_type=constants.PUBLISH_STEP_WEB_PUBLISHER, repo=repo, conduit=conduit, config=config, working_dir=working_dir, plugin_type=constants.WEB_DISTRIBUTOR_TYPE_ID, **kwargs) self.publish_dir = os.path.join(self.get_working_dir(), repo.id) atomic_publish = AtomicDirectoryPublishStep( self.get_working_dir(), [(repo.id, configuration.get_web_publish_dir(repo.repo_obj, config))], configuration.get_master_publish_dir(repo.repo_obj, config), step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish.description = _('Making files available via web.') main = MainStep() self.add_child(main) self.add_child(atomic_publish) mkdir(self.publish_dir)
def __init__(self, repo, publish_conduit, config, repo_content_unit_q=None): """ :param repo: Pulp managed Yum repository :type repo: pulp.plugins.model.Repository :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :param repo_content_unit_q: optional Q object that will be applied to the queries performed against RepoContentUnit model :type repo_content_unit_q: mongoengine.Q """ super(WebPublisher, self).__init__( step_type=constants.PUBLISH_STEP_WEB_PUBLISHER, repo=repo, publish_conduit=publish_conduit, config=config) docker_api_version = 'v1' publish_dir = configuration.get_web_publish_dir(repo, config, docker_api_version) app_file = configuration.get_redirect_file_name(repo) app_publish_location = os.path.join( configuration.get_app_publish_dir(config, docker_api_version), app_file) self.working_dir = os.path.join(self.get_working_dir(), docker_api_version) misc.mkdir(self.working_dir) self.web_working_dir = os.path.join(self.get_working_dir(), 'web') master_publish_dir = configuration.get_master_publish_dir(repo, config, docker_api_version) atomic_publish_step = AtomicDirectoryPublishStep(self.get_working_dir(), [('web', publish_dir), (app_file, app_publish_location)], master_publish_dir, step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish_step.description = _('Making v1 files available via web.') self.add_child(PublishImagesStep(repo_content_unit_q=repo_content_unit_q)) self.add_child(atomic_publish_step)
def __init__(self, repo, publish_conduit, config): """ :param repo: Pulp managed Python repository :type repo: pulp.plugins.model.Repository :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration """ super(PythonPublisher, self).__init__(constants.PUBLISH_STEP_PUBLISHER, repo, publish_conduit, config) publish_dir = configuration.get_web_publish_dir(repo, config) if not os.path.exists(self.get_working_dir()): os.makedirs(self.get_working_dir()) self.web_working_dir = os.path.join(self.get_working_dir(), repo.id) master_publish_dir = configuration.get_master_publish_dir(repo, config) atomic_publish_step = AtomicDirectoryPublishStep(self.get_working_dir(), [(repo.id, publish_dir)], master_publish_dir, step_type=constants.PUBLISH_STEP_OVER_HTTP) atomic_publish_step.description = _('Making files available via web.') self.add_child(PublishMetadataStep()) self.add_child(PublishContentStep()) self.add_child(atomic_publish_step)
def __init__(self, repo, publish_conduit, config, distributor_type): """ :param repo: Pulp managed Yum repository :type repo: pulp.plugins.model.Repository :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :param distributor_type: The type of the distributor that is being published :type distributor_type: str """ super(ExportRepoPublisher, self).__init__(repo, publish_conduit, config, distributor_type) date_filter = export_utils.create_date_range_filter(config) if date_filter: # Since this is a partial export we don't generate metadata # we have to clear out the previously added steps # we only need special version s of the rpm, drpm, and errata steps self.clear_children() self.add_child(PublishRpmAndDrpmStepIncremental(association_filters=date_filter)) self.add_child(PublishErrataStepIncremental(association_filters=date_filter)) working_directory = self.get_working_dir() export_dir = config.get(constants.EXPORT_DIRECTORY_KEYWORD) if export_dir: target_dir = os.path.join(export_dir, configuration.get_repo_relative_path(repo, config)) self.add_child(CopyDirectoryStep(working_directory, target_dir)) self.add_child(GenerateListingFileStep(export_dir, target_dir)) else: # Reset the steps to use an internal scratch directory other than the base working dir content_dir = os.path.join(working_directory, 'scratch') for step in self.children: step.working_dir = content_dir # Set up step to copy all the files to a realized directory with no symlinks # This could be optimized with a pathspec so that we don't create all the files # separately realized_dir = os.path.join(working_directory, 'realized') copy_target = os.path.join(realized_dir, configuration.get_repo_relative_path(repo, config)) self.add_child(CopyDirectoryStep(content_dir, copy_target)) self.add_child(GenerateListingFileStep(realized_dir, copy_target)) # Create the steps to generate the ISO and publish them to their final location output_dir = os.path.join(working_directory, 'output') self.add_child(CreateIsoStep(realized_dir, output_dir)) publish_location = [('/', location) for location in configuration.get_export_repo_publish_dirs(repo, config)] master_dir = configuration.get_master_publish_dir(repo, self.get_distributor_type()) atomic_publish = AtomicDirectoryPublishStep(output_dir, publish_location, master_dir) atomic_publish.description = _('Moving ISO to final location') self.add_child(atomic_publish)
def __init__(self, repo, conduit, config, plugin_type, **kwargs): super(Publisher, self).__init__(step_type=constants.PUBLISH_REPO_STEP, repo=repo, conduit=conduit, config=config, plugin_type=plugin_type) self.add_child( ModulePublisher(conduit=conduit, config=config, repo=repo)) master_publish_dir = configuration.get_master_publish_dir( repo, plugin_type) target_directories = [] if config.get(constants.PUBLISH_HTTP_KEYWORD): target_directories.append( configuration.get_http_publish_dir(config)) if config.get(constants.PUBLISH_HTTPS_KEYWORD): target_directories.append( configuration.get_https_publish_dir(config)) repo_path = configuration.get_repo_relative_path(repo, config) target_directories = [('/', os.path.join(x, repo_path)) for x in target_directories] atomic_publish_step = AtomicDirectoryPublishStep( self.get_working_dir(), target_directories, master_publish_dir) self.add_child(atomic_publish_step) self.description = self.__class__.description
def __init__(self, repo_group, publish_conduit, config, distributor_type): """ :param repo_group: Pulp managed Yum repository :type repo_group: pulp.plugins.model.RepositoryGroup :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoGroupPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :param distributor_type: The type of the distributor that is being published :type distributor_type: str """ super(ExportRepoGroupPublisher, self).__init__(constants.PUBLISH_STEP_EXPORT_REPO_GROUP, repo_group, publish_conduit, config, working_dir=repo_group.working_dir, distributor_type=distributor_type) working_dir = self.get_working_dir() scratch_dir = os.path.join(working_dir, 'scratch') realized_dir = os.path.join(working_dir, 'realized') flat_config = config.flatten() export_dir = config.get(constants.EXPORT_DIRECTORY_KEYWORD) if export_dir: repo_config = config else: repo_config = PluginCallConfiguration(flat_config, {constants.EXPORT_DIRECTORY_KEYWORD: realized_dir}) query_manager = RepoQueryManager() repos = query_manager.find_by_id_list(repo_group.repo_ids) empty_repos = True for repo in repos: empty_repos = False repo = common_utils.to_transfer_repo(repo) # Make sure we only publish rpm repo's if repo.notes['_repo-type'] != 'rpm-repo': continue repo_config_copy = copy.deepcopy(repo_config) repo.working_dir = os.path.join(scratch_dir, repo.id) repo_conduit = RepoPublishConduit(repo.id, distributor_type) publisher = ExportRepoPublisher(repo, repo_conduit, repo_config_copy, distributor_type) publisher.description = _("Exporting Repo: %s") % repo.id self.add_child(publisher) if empty_repos: os.makedirs(realized_dir) self.add_child(GenerateListingFileStep(realized_dir, realized_dir)) # If we aren't exporting to a directory add the ISO create & publish steps if not export_dir: # Create the steps to generate the ISO and publish them to their final location output_dir = os.path.join(working_dir, 'output') self.add_child(CreateIsoStep(realized_dir, output_dir)) export_dirs = configuration.get_export_repo_group_publish_dirs(repo_group, config) publish_location = [('/', location) for location in export_dirs] master_dir = configuration.get_master_publish_dir(repo_group, distributor_type) self.add_child(AtomicDirectoryPublishStep(output_dir, publish_location, master_dir))
def test_process_main_multiple_targets(self): source_dir = os.path.join(self.working_directory, 'source') master_dir = os.path.join(self.working_directory, 'master') publish_dir = os.path.join(self.working_directory, 'publish', 'bar') publish_dir += '/' # create some files to test sub_file = os.path.join(source_dir, 'foo', 'bar.html') touch(sub_file) sub_file = os.path.join(source_dir, 'qux', 'quux.html') touch(sub_file) target_qux = os.path.join(self.working_directory, 'publish', 'qux.html') step = AtomicDirectoryPublishStep( source_dir, [('/', publish_dir), ('qux/quux.html', target_qux)], master_dir) step.parent = Mock(timestamp=str(time.time())) step.process_main() target_file = os.path.join(publish_dir, 'foo', 'bar.html') self.assertEquals(True, os.path.exists(target_file)) self.assertEquals(True, os.path.exists(target_qux))
def test_process_main_default_id(self): step = AtomicDirectoryPublishStep('foo', 'bar', 'baz') self.assertEquals(step.step_id, reporting_constants.PUBLISH_STEP_DIRECTORY)
def test_process_main_alternate_id(self): step = AtomicDirectoryPublishStep('foo', 'bar', 'baz', step_type='alternate') self.assertEquals(step.step_id, 'alternate')
def __init__(self, repo, publish_conduit, config, distributor_type, **kwargs): """ :param repo: Pulp managed Yum repository :type repo: pulp.plugins.model.Repository :param publish_conduit: Conduit providing access to relative Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :param distributor_type: The type of the distributor that is being published :type distributor_type: str """ repo_relative_path = configuration.get_repo_relative_path(repo, config) last_published = publish_conduit.last_publish() last_deleted = repo.last_unit_removed date_filter = None insert_step = None if last_published and \ ((last_deleted and last_published > last_deleted) or not last_deleted): # Add the step to copy the current published directory into place working_dir = repo.working_dir specific_master = None if config.get(constants.PUBLISH_HTTPS_KEYWORD): root_publish_dir = configuration.get_https_publish_dir(config) repo_publish_dir = os.path.join(root_publish_dir, repo_relative_path) specific_master = os.path.realpath(repo_publish_dir) if not specific_master and config.get(constants.PUBLISH_HTTP_KEYWORD): root_publish_dir = configuration.get_http_publish_dir(config) repo_publish_dir = os.path.join(root_publish_dir, repo_relative_path) specific_master = os.path.realpath(repo_publish_dir) # Only do an incremental publish if the previous publish can be found if os.path.exists(specific_master): insert_step = CopyDirectoryStep(specific_master, working_dir, preserve_symlinks=True) # Pass something useful to the super so that it knows the publish info string_date = dateutils.format_iso8601_datetime(last_published) date_filter = export_utils.create_date_range_filter( {constants.START_DATE_KEYWORD: string_date}) super(Publisher, self).__init__(repo, publish_conduit, config, distributor_type, association_filters=date_filter, **kwargs) if insert_step: self.insert_child(0, insert_step) self.rpm_step.fast_forward = True # Add the web specific directory publishing processing steps target_directories = [] # it's convenient to create these now, but we won't add them until later, # because we want them to run last listing_steps = [] if config.get(constants.PUBLISH_HTTPS_KEYWORD): root_publish_dir = configuration.get_https_publish_dir(config) repo_publish_dir = os.path.join(root_publish_dir, repo_relative_path) target_directories.append(['/', repo_publish_dir]) listing_steps.append(GenerateListingFileStep(root_publish_dir, repo_publish_dir)) if config.get(constants.PUBLISH_HTTP_KEYWORD): root_publish_dir = configuration.get_http_publish_dir(config) repo_publish_dir = os.path.join(root_publish_dir, repo_relative_path) target_directories.append(['/', repo_publish_dir]) listing_steps.append(GenerateListingFileStep(root_publish_dir, repo_publish_dir)) master_publish_dir = configuration.get_master_publish_dir(repo, distributor_type) atomic_publish_step = AtomicDirectoryPublishStep(self.get_working_dir(), target_directories, master_publish_dir) atomic_publish_step.description = _("Publishing files to web") self.add_child(atomic_publish_step) # add the listing file generation step(s) for step in listing_steps: self.add_child(step)