def test_update_from_json(self):
# Test
metadata = RepositoryMetadata()
metadata.update_from_json(VALID_REPO_METADATA_JSON)
# Verify
self.assertEqual(2, len(metadata.modules))
for m in metadata.modules:
self.assertTrue(isinstance(m, Module))
sorted_modules = sorted(metadata.modules, key=lambda x: x.name)
self.assertEqual(sorted_modules[0].name, 'common')
self.assertEqual(sorted_modules[0].author, 'lab42')
self.assertEqual(sorted_modules[0].version, '0.0.1')
self.assertEqual(sorted_modules[0].tag_list, [])
self.assertEqual(sorted_modules[0].description, None)
self.assertEqual(sorted_modules[0].project_page, None)
self.assertEqual(sorted_modules[1].name, 'postfix')
self.assertEqual(sorted_modules[1].author, 'lab42')
self.assertEqual(sorted_modules[1].version, '0.0.2')
self.assertEqual(sorted_modules[1].tag_list,
['postfix', 'applications'])
self.assertEqual(sorted_modules[1].description, None)
self.assertEqual(sorted_modules[1].project_page, None)
def _generate_metadata(self, modules):
"""
Generates the repository metadata document for all modules in the
:type modules: list of pulp.plugins.model.AssociatedUnit
"""
_logger.info('Generating metadata for repository <%s>' % self.repo.id)
# Convert the Pulp data types into the local model
metadata = RepositoryMetadata()
for m in modules:
combined = copy.copy(m.unit_key)
combined.update(m.metadata)
module = Module.from_dict(combined)
metadata.modules.append(module)
# Write the JSON representation of the metadata to the repository
json_metadata = metadata.to_json()
build_dir = self._build_dir()
metadata_file = os.path.join(build_dir,
constants.REPO_METADATA_FILENAME)
f = open(metadata_file, 'w')
f.write(json_metadata)
f.close()
def test_to_json(self):
# Setup
metadata = RepositoryMetadata()
metadata.update_from_json(VALID_REPO_METADATA_JSON)
# Test
serialized = metadata.to_json()
# Verify
parsed = json.loads(serialized)
self.assertEqual(2, len(parsed))
sorted_modules = sorted(parsed, key=lambda x : x['name'])
self.assertEqual(4, len(sorted_modules[0]))
self.assertEqual(sorted_modules[0]['name'], 'common')
self.assertEqual(sorted_modules[0]['author'], 'lab42')
self.assertEqual(sorted_modules[0]['version'], '0.0.1')
self.assertEqual(sorted_modules[0]['tag_list'], [])
self.assertEqual(4, len(sorted_modules[1]))
self.assertEqual(sorted_modules[1]['name'], 'postfix')
self.assertEqual(sorted_modules[1]['author'], 'lab42')
self.assertEqual(sorted_modules[1]['version'], '0.0.2')
self.assertEqual(sorted_modules[1]['tag_list'], ['postfix', 'applications'])
def test_to_json(self):
# Setup
metadata = RepositoryMetadata()
metadata.update_from_json(VALID_REPO_METADATA_JSON)
# Test
serialized = metadata.to_json()
# Verify
parsed = json.loads(serialized)
self.assertEqual(2, len(parsed))
sorted_modules = sorted(parsed, key=lambda x: x['name'])
self.assertEqual(4, len(sorted_modules[0]))
self.assertEqual(sorted_modules[0]['name'], 'common')
self.assertEqual(sorted_modules[0]['author'], 'lab42')
self.assertEqual(sorted_modules[0]['version'], '0.0.1')
self.assertEqual(sorted_modules[0]['tag_list'], [])
self.assertEqual(4, len(sorted_modules[1]))
self.assertEqual(sorted_modules[1]['name'], 'postfix')
self.assertEqual(sorted_modules[1]['author'], 'lab42')
self.assertEqual(sorted_modules[1]['version'], '0.0.2')
self.assertEqual(sorted_modules[1]['tag_list'],
['postfix', 'applications'])
def test_update_from_json(self):
# Test
metadata = RepositoryMetadata()
metadata.update_from_json(VALID_REPO_METADATA_JSON)
# Verify
self.assertEqual(2, len(metadata.modules))
for m in metadata.modules:
self.assertTrue(isinstance(m, Module))
sorted_modules = sorted(metadata.modules, key=lambda x : x.name)
self.assertEqual(sorted_modules[0].name, 'common')
self.assertEqual(sorted_modules[0].author, 'lab42')
self.assertEqual(sorted_modules[0].version, '0.0.1')
self.assertEqual(sorted_modules[0].tags, [])
self.assertEqual(sorted_modules[0].description, None)
self.assertEqual(sorted_modules[0].project_page, None)
self.assertEqual(sorted_modules[1].name, 'postfix')
self.assertEqual(sorted_modules[1].author, 'lab42')
self.assertEqual(sorted_modules[1].version, '0.0.2')
self.assertEqual(sorted_modules[1].tags, ['postfix', 'applications'])
self.assertEqual(sorted_modules[1].description, None)
self.assertEqual(sorted_modules[1].project_page, None)
def _generate_metadata(self, modules):
"""
Generates the repository metadata document for all modules in the
:type modules: list of pulp.plugins.model.AssociatedUnit
"""
_logger.info('Generating metadata for repository <%s>' % self.repo.id)
# Convert the Pulp data types into the local model
metadata = RepositoryMetadata()
for m in modules:
combined = copy.copy(m.unit_key)
combined.update(m.metadata)
module = Module.from_dict(combined)
metadata.modules.append(module)
# Write the JSON representation of the metadata to the repository
json_metadata = metadata.to_json()
build_dir = self._build_dir()
metadata_file = os.path.join(build_dir, constants.REPO_METADATA_FILENAME)
f = open(metadata_file, 'w')
f.write(json_metadata)
f.close()
def test_update_from_json(self):
# Test
metadata = RepositoryMetadata()
metadata.update_from_json(VALID_REPO_METADATA_JSON)
# Verify
self.assertEqual(2, len(metadata.modules))
for m in metadata.modules:
self.assertTrue(isinstance(m, Module))
sorted_modules = sorted(metadata.modules, key=lambda x: x.name)
self.assertEqual(sorted_modules[0].name, "common")
self.assertEqual(sorted_modules[0].author, "lab42")
self.assertEqual(sorted_modules[0].version, "0.0.1")
self.assertEqual(sorted_modules[0].tag_list, [])
self.assertEqual(sorted_modules[0].description, None)
self.assertEqual(sorted_modules[0].project_page, None)
self.assertEqual(sorted_modules[1].name, "postfix")
self.assertEqual(sorted_modules[1].author, "lab42")
self.assertEqual(sorted_modules[1].version, "0.0.2")
self.assertEqual(sorted_modules[1].tag_list, ["postfix", "applications"])
self.assertEqual(sorted_modules[1].description, None)
self.assertEqual(sorted_modules[1].project_page, None)
def test_to_json(self):
# Setup
metadata = RepositoryMetadata()
metadata.update_from_json(VALID_REPO_METADATA_JSON)
# Test
serialized = metadata.to_json()
# Verify
parsed = json.loads(serialized)
self.assertEqual(2, len(parsed))
sorted_modules = sorted(parsed, key=lambda x: x["name"])
self.assertEqual(4, len(sorted_modules[0]))
self.assertEqual(sorted_modules[0]["name"], "common")
self.assertEqual(sorted_modules[0]["author"], "lab42")
self.assertEqual(sorted_modules[0]["version"], "0.0.1")
self.assertEqual(sorted_modules[0]["tag_list"], [])
self.assertEqual(4, len(sorted_modules[1]))
self.assertEqual(sorted_modules[1]["name"], "postfix")
self.assertEqual(sorted_modules[1]["author"], "lab42")
self.assertEqual(sorted_modules[1]["version"], "0.0.2")
self.assertEqual(sorted_modules[1]["tag_list"], ["postfix", "applications"])
self.progress_report.metadata_traceback = sys.exc_info()[2]
end_time = datetime.now()
duration = end_time - start_time
self.progress_report.metadata_execution_time = duration.seconds
self.progress_report.update_progress()
return None
finally:
self.downloader = None
# Parse the retrieved metadata documents
try:
metadata = RepositoryMetadata()
for doc in metadata_json_docs:
metadata.update_from_json(doc)
except Exception, e:
_logger.exception('Exception parsing metadata for repository <%s>' % self.repo.id)
self.progress_report.metadata_state = STATE_FAILED
self.progress_report.metadata_error_message = _('Error parsing repository modules metadata document')
self.progress_report.metadata_exception = e
self.progress_report.metadata_traceback = sys.exc_info()[2]
end_time = datetime.now()
duration = end_time - start_time
self.progress_report.metadata_execution_time = duration.seconds
self.progress_report.update_progress()
self.progress_report.metadata_traceback = sys.exc_info()[2]
end_time = datetime.now()
duration = end_time - start_time
self.progress_report.metadata_execution_time = duration.seconds
self.progress_report.update_progress()
return None
finally:
self.downloader = None
# Parse the retrieved metadata documents
try:
metadata = RepositoryMetadata()
for doc in metadata_json_docs:
metadata.update_from_json(doc)
except Exception, e:
_logger.exception(
'Exception parsing metadata for repository <%s>' %
self.repo.id)
self.progress_report.metadata_state = STATE_FAILED
self.progress_report.metadata_error_message = _(
'Error parsing repository modules metadata document')
self.progress_report.metadata_exception = e
self.progress_report.metadata_traceback = sys.exc_info()[2]
end_time = datetime.now()
duration = end_time - start_time
self.progress_report.metadata_execution_time = duration.seconds
class PuppetModuleSyncRun(object):
"""
Used to perform a single sync of a puppet repository. This class will
maintain state relevant to the run and should not be reused across runs.
"""
def __init__(self, repo, sync_conduit, config, is_cancelled_call):
self.repo = repo
self.sync_conduit = sync_conduit
self.config = config
self.is_cancelled_call = is_cancelled_call
self.progress_report = SyncProgressReport(sync_conduit)
def perform_sync(self):
"""
Performs the sync operation according to the configured state of the
instance. The report to be sent back to Pulp is returned from this
call. This call will make calls into the conduit's progress update
as appropriate.
This call executes serially. No threads are created by this call. It
will not return until either a step fails or the entire sync is
completed.
:return: the report object to return to Pulp from the sync call
:rtype: pulp.plugins.model.SyncReport
"""
_LOG.info('Beginning sync for repository <%s>' % self.repo.id)
# quit now if there is no feed URL defined
if not self.config.get(constants.CONFIG_FEED):
self.progress_report.metadata_state = STATE_FAILED
self.progress_report.metadata_error_message = _(
'Cannot perform repository sync on a repository with no feed')
self.progress_report.update_progress()
return self.progress_report.build_final_report()
try:
metadata = self._parse_metadata()
if not metadata:
report = self.progress_report.build_final_report()
return report
self._import_modules(metadata)
finally:
# One final progress update before finishing
self.progress_report.update_progress()
report = self.progress_report.build_final_report()
return report
def _parse_metadata(self):
"""
Takes the necessary actions (according to the run configuration) to
retrieve and parse the repository's metadata. This call will return
either the successfully parsed metadata or None if it could not
be retrieved or parsed. The progress report will be updated with the
appropriate description of what went wrong in the event of an error,
so the caller should interpet a None return as an error occuring and
not continue the sync.
:return: object representation of the metadata
:rtype: RepositoryMetadata
"""
_LOG.info('Beginning metadata retrieval for repository <%s>' %
self.repo.id)
self.progress_report.metadata_state = STATE_RUNNING
self.progress_report.update_progress()
start_time = datetime.now()
# Retrieve the metadata from the source
try:
downloader = self._create_downloader()
metadata_json_docs = downloader.retrieve_metadata(
self.progress_report)
except Exception, e:
_LOG.exception(
'Exception while retrieving metadata for repository <%s>' %
self.repo.id)
self.progress_report.metadata_state = STATE_FAILED
self.progress_report.metadata_error_message = _(
'Error downloading metadata')
self.progress_report.metadata_exception = e
self.progress_report.metadata_traceback = sys.exc_info()[2]
end_time = datetime.now()
duration = end_time - start_time
self.progress_report.metadata_execution_time = duration.seconds
self.progress_report.update_progress()
return None
# Parse the retrieved metadata documents
try:
metadata = RepositoryMetadata()
for doc in metadata_json_docs:
metadata.update_from_json(doc)
except Exception, e:
_LOG.exception('Exception parsing metadata for repository <%s>' %
self.repo.id)
self.progress_report.metadata_state = STATE_FAILED
self.progress_report.metadata_error_message = _(
'Error parsing repository modules metadata document')
self.progress_report.metadata_exception = e
self.progress_report.metadata_traceback = sys.exc_info()[2]
end_time = datetime.now()
duration = end_time - start_time
self.progress_report.metadata_execution_time = duration.seconds
self.progress_report.update_progress()
return None
