def test_update_from_json(self): # Test metadata = RepositoryMetadata() metadata.update_from_json(VALID_REPO_METADATA_JSON) # Verify self.assertEqual(2, len(metadata.modules)) for m in metadata.modules: self.assertTrue(isinstance(m, Module)) sorted_modules = sorted(metadata.modules, key=lambda x: x.name) self.assertEqual(sorted_modules[0].name, 'common') self.assertEqual(sorted_modules[0].author, 'lab42') self.assertEqual(sorted_modules[0].version, '0.0.1') self.assertEqual(sorted_modules[0].tag_list, []) self.assertEqual(sorted_modules[0].description, None) self.assertEqual(sorted_modules[0].project_page, None) self.assertEqual(sorted_modules[1].name, 'postfix') self.assertEqual(sorted_modules[1].author, 'lab42') self.assertEqual(sorted_modules[1].version, '0.0.2') self.assertEqual(sorted_modules[1].tag_list, ['postfix', 'applications']) self.assertEqual(sorted_modules[1].description, None) self.assertEqual(sorted_modules[1].project_page, None)
def _generate_metadata(self, modules): """ Generates the repository metadata document for all modules in the :type modules: list of pulp.plugins.model.AssociatedUnit """ _logger.info('Generating metadata for repository <%s>' % self.repo.id) # Convert the Pulp data types into the local model metadata = RepositoryMetadata() for m in modules: combined = copy.copy(m.unit_key) combined.update(m.metadata) module = Module.from_dict(combined) metadata.modules.append(module) # Write the JSON representation of the metadata to the repository json_metadata = metadata.to_json() build_dir = self._build_dir() metadata_file = os.path.join(build_dir, constants.REPO_METADATA_FILENAME) f = open(metadata_file, 'w') f.write(json_metadata) f.close()
def test_to_json(self): # Setup metadata = RepositoryMetadata() metadata.update_from_json(VALID_REPO_METADATA_JSON) # Test serialized = metadata.to_json() # Verify parsed = json.loads(serialized) self.assertEqual(2, len(parsed)) sorted_modules = sorted(parsed, key=lambda x : x['name']) self.assertEqual(4, len(sorted_modules[0])) self.assertEqual(sorted_modules[0]['name'], 'common') self.assertEqual(sorted_modules[0]['author'], 'lab42') self.assertEqual(sorted_modules[0]['version'], '0.0.1') self.assertEqual(sorted_modules[0]['tag_list'], []) self.assertEqual(4, len(sorted_modules[1])) self.assertEqual(sorted_modules[1]['name'], 'postfix') self.assertEqual(sorted_modules[1]['author'], 'lab42') self.assertEqual(sorted_modules[1]['version'], '0.0.2') self.assertEqual(sorted_modules[1]['tag_list'], ['postfix', 'applications'])
def test_to_json(self): # Setup metadata = RepositoryMetadata() metadata.update_from_json(VALID_REPO_METADATA_JSON) # Test serialized = metadata.to_json() # Verify parsed = json.loads(serialized) self.assertEqual(2, len(parsed)) sorted_modules = sorted(parsed, key=lambda x: x['name']) self.assertEqual(4, len(sorted_modules[0])) self.assertEqual(sorted_modules[0]['name'], 'common') self.assertEqual(sorted_modules[0]['author'], 'lab42') self.assertEqual(sorted_modules[0]['version'], '0.0.1') self.assertEqual(sorted_modules[0]['tag_list'], []) self.assertEqual(4, len(sorted_modules[1])) self.assertEqual(sorted_modules[1]['name'], 'postfix') self.assertEqual(sorted_modules[1]['author'], 'lab42') self.assertEqual(sorted_modules[1]['version'], '0.0.2') self.assertEqual(sorted_modules[1]['tag_list'], ['postfix', 'applications'])
def test_update_from_json(self): # Test metadata = RepositoryMetadata() metadata.update_from_json(VALID_REPO_METADATA_JSON) # Verify self.assertEqual(2, len(metadata.modules)) for m in metadata.modules: self.assertTrue(isinstance(m, Module)) sorted_modules = sorted(metadata.modules, key=lambda x : x.name) self.assertEqual(sorted_modules[0].name, 'common') self.assertEqual(sorted_modules[0].author, 'lab42') self.assertEqual(sorted_modules[0].version, '0.0.1') self.assertEqual(sorted_modules[0].tags, []) self.assertEqual(sorted_modules[0].description, None) self.assertEqual(sorted_modules[0].project_page, None) self.assertEqual(sorted_modules[1].name, 'postfix') self.assertEqual(sorted_modules[1].author, 'lab42') self.assertEqual(sorted_modules[1].version, '0.0.2') self.assertEqual(sorted_modules[1].tags, ['postfix', 'applications']) self.assertEqual(sorted_modules[1].description, None) self.assertEqual(sorted_modules[1].project_page, None)
def _generate_metadata(self, modules): """ Generates the repository metadata document for all modules in the :type modules: list of pulp.plugins.model.AssociatedUnit """ _logger.info('Generating metadata for repository <%s>' % self.repo.id) # Convert the Pulp data types into the local model metadata = RepositoryMetadata() for m in modules: combined = copy.copy(m.unit_key) combined.update(m.metadata) module = Module.from_dict(combined) metadata.modules.append(module) # Write the JSON representation of the metadata to the repository json_metadata = metadata.to_json() build_dir = self._build_dir() metadata_file = os.path.join(build_dir, constants.REPO_METADATA_FILENAME) f = open(metadata_file, 'w') f.write(json_metadata) f.close()
def test_update_from_json(self): # Test metadata = RepositoryMetadata() metadata.update_from_json(VALID_REPO_METADATA_JSON) # Verify self.assertEqual(2, len(metadata.modules)) for m in metadata.modules: self.assertTrue(isinstance(m, Module)) sorted_modules = sorted(metadata.modules, key=lambda x: x.name) self.assertEqual(sorted_modules[0].name, "common") self.assertEqual(sorted_modules[0].author, "lab42") self.assertEqual(sorted_modules[0].version, "0.0.1") self.assertEqual(sorted_modules[0].tag_list, []) self.assertEqual(sorted_modules[0].description, None) self.assertEqual(sorted_modules[0].project_page, None) self.assertEqual(sorted_modules[1].name, "postfix") self.assertEqual(sorted_modules[1].author, "lab42") self.assertEqual(sorted_modules[1].version, "0.0.2") self.assertEqual(sorted_modules[1].tag_list, ["postfix", "applications"]) self.assertEqual(sorted_modules[1].description, None) self.assertEqual(sorted_modules[1].project_page, None)
def test_to_json(self): # Setup metadata = RepositoryMetadata() metadata.update_from_json(VALID_REPO_METADATA_JSON) # Test serialized = metadata.to_json() # Verify parsed = json.loads(serialized) self.assertEqual(2, len(parsed)) sorted_modules = sorted(parsed, key=lambda x: x["name"]) self.assertEqual(4, len(sorted_modules[0])) self.assertEqual(sorted_modules[0]["name"], "common") self.assertEqual(sorted_modules[0]["author"], "lab42") self.assertEqual(sorted_modules[0]["version"], "0.0.1") self.assertEqual(sorted_modules[0]["tag_list"], []) self.assertEqual(4, len(sorted_modules[1])) self.assertEqual(sorted_modules[1]["name"], "postfix") self.assertEqual(sorted_modules[1]["author"], "lab42") self.assertEqual(sorted_modules[1]["version"], "0.0.2") self.assertEqual(sorted_modules[1]["tag_list"], ["postfix", "applications"])
self.progress_report.metadata_traceback = sys.exc_info()[2] end_time = datetime.now() duration = end_time - start_time self.progress_report.metadata_execution_time = duration.seconds self.progress_report.update_progress() return None finally: self.downloader = None # Parse the retrieved metadata documents try: metadata = RepositoryMetadata() for doc in metadata_json_docs: metadata.update_from_json(doc) except Exception, e: _logger.exception('Exception parsing metadata for repository <%s>' % self.repo.id) self.progress_report.metadata_state = STATE_FAILED self.progress_report.metadata_error_message = _('Error parsing repository modules metadata document') self.progress_report.metadata_exception = e self.progress_report.metadata_traceback = sys.exc_info()[2] end_time = datetime.now() duration = end_time - start_time self.progress_report.metadata_execution_time = duration.seconds self.progress_report.update_progress()
self.progress_report.metadata_traceback = sys.exc_info()[2] end_time = datetime.now() duration = end_time - start_time self.progress_report.metadata_execution_time = duration.seconds self.progress_report.update_progress() return None finally: self.downloader = None # Parse the retrieved metadata documents try: metadata = RepositoryMetadata() for doc in metadata_json_docs: metadata.update_from_json(doc) except Exception, e: _logger.exception( 'Exception parsing metadata for repository <%s>' % self.repo.id) self.progress_report.metadata_state = STATE_FAILED self.progress_report.metadata_error_message = _( 'Error parsing repository modules metadata document') self.progress_report.metadata_exception = e self.progress_report.metadata_traceback = sys.exc_info()[2] end_time = datetime.now() duration = end_time - start_time self.progress_report.metadata_execution_time = duration.seconds
class PuppetModuleSyncRun(object): """ Used to perform a single sync of a puppet repository. This class will maintain state relevant to the run and should not be reused across runs. """ def __init__(self, repo, sync_conduit, config, is_cancelled_call): self.repo = repo self.sync_conduit = sync_conduit self.config = config self.is_cancelled_call = is_cancelled_call self.progress_report = SyncProgressReport(sync_conduit) def perform_sync(self): """ Performs the sync operation according to the configured state of the instance. The report to be sent back to Pulp is returned from this call. This call will make calls into the conduit's progress update as appropriate. This call executes serially. No threads are created by this call. It will not return until either a step fails or the entire sync is completed. :return: the report object to return to Pulp from the sync call :rtype: pulp.plugins.model.SyncReport """ _LOG.info('Beginning sync for repository <%s>' % self.repo.id) # quit now if there is no feed URL defined if not self.config.get(constants.CONFIG_FEED): self.progress_report.metadata_state = STATE_FAILED self.progress_report.metadata_error_message = _( 'Cannot perform repository sync on a repository with no feed') self.progress_report.update_progress() return self.progress_report.build_final_report() try: metadata = self._parse_metadata() if not metadata: report = self.progress_report.build_final_report() return report self._import_modules(metadata) finally: # One final progress update before finishing self.progress_report.update_progress() report = self.progress_report.build_final_report() return report def _parse_metadata(self): """ Takes the necessary actions (according to the run configuration) to retrieve and parse the repository's metadata. This call will return either the successfully parsed metadata or None if it could not be retrieved or parsed. The progress report will be updated with the appropriate description of what went wrong in the event of an error, so the caller should interpet a None return as an error occuring and not continue the sync. :return: object representation of the metadata :rtype: RepositoryMetadata """ _LOG.info('Beginning metadata retrieval for repository <%s>' % self.repo.id) self.progress_report.metadata_state = STATE_RUNNING self.progress_report.update_progress() start_time = datetime.now() # Retrieve the metadata from the source try: downloader = self._create_downloader() metadata_json_docs = downloader.retrieve_metadata( self.progress_report) except Exception, e: _LOG.exception( 'Exception while retrieving metadata for repository <%s>' % self.repo.id) self.progress_report.metadata_state = STATE_FAILED self.progress_report.metadata_error_message = _( 'Error downloading metadata') self.progress_report.metadata_exception = e self.progress_report.metadata_traceback = sys.exc_info()[2] end_time = datetime.now() duration = end_time - start_time self.progress_report.metadata_execution_time = duration.seconds self.progress_report.update_progress() return None # Parse the retrieved metadata documents try: metadata = RepositoryMetadata() for doc in metadata_json_docs: metadata.update_from_json(doc) except Exception, e: _LOG.exception('Exception parsing metadata for repository <%s>' % self.repo.id) self.progress_report.metadata_state = STATE_FAILED self.progress_report.metadata_error_message = _( 'Error parsing repository modules metadata document') self.progress_report.metadata_exception = e self.progress_report.metadata_traceback = sys.exc_info()[2] end_time = datetime.now() duration = end_time - start_time self.progress_report.metadata_execution_time = duration.seconds self.progress_report.update_progress() return None