def perform_sync(self, repo, sync_conduit, config): """ Perform the sync operation accoring to the config for the given repo, and return a report. The sync progress will be reported through the sync_conduit. :param repo: Metadata describing the repository :type repo: pulp.server.plugins.model.Repository :param sync_conduit: The sync_conduit that gives us access to the local repository :type sync_conduit: pulp.server.conduits.repo_sync.RepoSyncConduit :param config: The configuration for the importer :type config: pulp.server.plugins.config.PluginCallConfiguration :return: The sync report :rtype: pulp.plugins.model.SyncReport """ # Build the progress report and set it to the running state progress_report = SyncProgressReport(sync_conduit) # Cast our config parameters to the correct types and use them to build an ISOBumper max_speed = config.get(constants.CONFIG_MAX_SPEED) if max_speed is not None: max_speed = float(max_speed) num_threads = config.get(constants.CONFIG_NUM_THREADS) if num_threads is not None: num_threads = int(num_threads) else: num_threads = constants.DEFAULT_NUM_THREADS progress_report.metadata_state = STATE_RUNNING progress_report.update_progress() self.bumper = ISOBumper( repo_url=encode_unicode(config.get(constants.CONFIG_FEED_URL)), working_path=repo.working_dir, max_speed=max_speed, num_threads=num_threads, ssl_client_cert=config.get(constants.CONFIG_SSL_CLIENT_CERT), ssl_client_key=config.get(constants.CONFIG_SSL_CLIENT_KEY), ssl_ca_cert=config.get(constants.CONFIG_SSL_CA_CERT), proxy_url=config.get(constants.CONFIG_PROXY_URL), proxy_port=config.get(constants.CONFIG_PROXY_PORT), proxy_user=config.get(constants.CONFIG_PROXY_USER), proxy_password=config.get(constants.CONFIG_PROXY_PASSWORD)) # Get the manifest and download the ISOs that we are missing manifest = self.bumper.get_manifest() progress_report.metadata_state = STATE_COMPLETE progress_report.modules_state = STATE_RUNNING progress_report.update_progress() missing_isos = self._filter_missing_isos(sync_conduit, manifest) new_isos = self.bumper.download_resources(missing_isos) # Move the downloaded stuff and junk to the permanent location self._create_units(sync_conduit, new_isos) # Report that we are finished progress_report.modules_state = STATE_COMPLETE report = progress_report.build_final_report() return report
class ISOSyncRun(object): """ This class maintains state for a single repository sync. We need to keep the state so that we can cancel a sync that is in progress. """ def cancel_sync(self): """ This method will cancel a sync that is in progress. """ self.bumper.cancel_download() def perform_sync(self, repo, sync_conduit, config): """ Perform the sync operation accoring to the config for the given repo, and return a report. The sync progress will be reported through the sync_conduit. :param repo: Metadata describing the repository :type repo: pulp.server.plugins.model.Repository :param sync_conduit: The sync_conduit that gives us access to the local repository :type sync_conduit: pulp.server.conduits.repo_sync.RepoSyncConduit :param config: The configuration for the importer :type config: pulp.server.plugins.config.PluginCallConfiguration :return: The sync report :rtype: pulp.plugins.model.SyncReport """ # Build the progress report and set it to the running state progress_report = SyncProgressReport(sync_conduit) # Cast our config parameters to the correct types and use them to build an ISOBumper max_speed = config.get(constants.CONFIG_MAX_SPEED) if max_speed is not None: max_speed = float(max_speed) num_threads = config.get(constants.CONFIG_NUM_THREADS) if num_threads is not None: num_threads = int(num_threads) else: num_threads = constants.DEFAULT_NUM_THREADS progress_report.metadata_state = STATE_RUNNING progress_report.update_progress() self.bumper = ISOBumper( repo_url=encode_unicode(config.get(constants.CONFIG_FEED_URL)), working_path=repo.working_dir, max_speed=max_speed, num_threads=num_threads, ssl_client_cert=config.get(constants.CONFIG_SSL_CLIENT_CERT), ssl_client_key=config.get(constants.CONFIG_SSL_CLIENT_KEY), ssl_ca_cert=config.get(constants.CONFIG_SSL_CA_CERT), proxy_url=config.get(constants.CONFIG_PROXY_URL), proxy_port=config.get(constants.CONFIG_PROXY_PORT), proxy_user=config.get(constants.CONFIG_PROXY_USER), proxy_password=config.get(constants.CONFIG_PROXY_PASSWORD)) # Get the manifest and download the ISOs that we are missing manifest = self.bumper.get_manifest() progress_report.metadata_state = STATE_COMPLETE progress_report.modules_state = STATE_RUNNING progress_report.update_progress() missing_isos = self._filter_missing_isos(sync_conduit, manifest) new_isos = self.bumper.download_resources(missing_isos) # Move the downloaded stuff and junk to the permanent location self._create_units(sync_conduit, new_isos) # Report that we are finished progress_report.modules_state = STATE_COMPLETE report = progress_report.build_final_report() return report def _filter_missing_isos(self, sync_conduit, manifest): """ Use the sync_conduit and the ISOBumper manifest to determine which ISOs are at the feed_url that are not in our local store. Return a subset of the given manifest that represents the missing ISOs. The manifest format is described in the docblock for pulp_rpm.plugins.importers.iso_importer.bumper.ISOBumper.manifest. :param sync_conduit: The sync_conduit that gives us access to the local repository :type sync_conduit: pulp.server.conduits.repo_sync.RepoSyncConduit :param manifest: A list of dictionaries that describe the ISOs that are available at the feed_url that we are syncing with :type manifest: list :return: A list of dictionaries that describe the ISOs that we should retrieve from the feed_url. These dictionaries are in the same format as they were in the manifest. :rtype: list """ def _unit_key_str(unit_key_dict): return '%s-%s-%s'%(unit_key_dict['name'], unit_key_dict['checksum'], unit_key_dict['size']) available_units_by_key = dict([(_unit_key_str(u), u) for u in manifest]) module_criteria = UnitAssociationCriteria(type_ids=[ids.TYPE_ID_ISO]) existing_isos = sync_conduit.get_units(criteria=module_criteria) existing_iso_keys = set([_unit_key_str(m.unit_key) for m in existing_isos]) available_iso_keys = set([_unit_key_str(u) for u in manifest]) missing_iso_keys = list(available_iso_keys - existing_iso_keys) missing_isos = [available_units_by_key[k] for k in missing_iso_keys] return missing_isos def _create_units(self, sync_conduit, new_isos): """ For each ISO specified in new_isos, create a new Pulp Unit and move the file from its temporary storage location to the storage location specified by the Unit. new_isos is a list of dictionaries that describe the isos that have been downloaded, and is the same format as the return value from pulp_rpm.plugins.importers.iso_importer.bumper.ISOBumper.download_resources. :param sync_conduit: The sync_conduit that gives us access to the local repository :type sync_conduit: pulp.server.conduits.repo_sync.RepoSyncConduit :param new_isos: A list of dictionaries describing the newly downloaded ISOs. :type new_isos: list """ for iso in new_isos: unit_key = {'name': iso['name'], 'size': iso['size'], 'checksum': iso['checksum']} metadata = {} relative_path = os.path.join(unit_key['name'], unit_key['checksum'], str(unit_key['size']), unit_key['name']) unit = sync_conduit.init_unit(ids.TYPE_ID_ISO, unit_key, metadata, relative_path) # Move the unit to the storage_path temporary_file_location = iso['destination'] permanent_file_location = unit.storage_path shutil.move(temporary_file_location, permanent_file_location) unit = sync_conduit.save_unit(unit)