def test_publish_history_descending_sort(self):
"""
Tests use the sort parameter to sort the results in descending order by start time
"""
# Setup
self.repo_manager.create_repo("test_sort")
self.distributor_manager.add_distributor("test_sort", "mock-distributor", {}, True, distributor_id="test_dist")
# Create some consecutive publish entries
date_string = "2013-06-01T12:00:0%sZ"
for i in range(0, 10, 2):
r = RepoPublishResult.expected_result(
"test_sort",
"test_dist",
"bar",
date_string % str(i),
date_string % str(i + 1),
"test-summary",
"test-details",
RepoPublishResult.RESULT_SUCCESS,
)
RepoPublishResult.get_collection().insert(r, safe=True)
# Test that returned entries are in descending order by time
entries = self.publish_manager.publish_history("test_sort", "test_dist", sort=constants.SORT_DESCENDING)
self.assertEqual(5, len(entries))
for i in range(0, 4):
first = dateutils.parse_iso8601_datetime(entries[i]["started"])
second = dateutils.parse_iso8601_datetime(entries[i + 1]["started"])
self.assertTrue(first > second)
def test_publish_history_end_date(self):
# Setup
self.repo_manager.create_repo("test_date")
self.distributor_manager.add_distributor("test_date", "mock-distributor", {}, True, distributor_id="test_dist")
# Create three consecutive publish entries
date_string = "2013-06-01T12:00:0%sZ"
for i in range(0, 6, 2):
r = RepoPublishResult.expected_result(
"test_date",
"test_dist",
"bar",
date_string % str(i),
date_string % str(i + 1),
"test-summary",
"test-details",
RepoPublishResult.RESULT_SUCCESS,
)
RepoPublishResult.get_collection().insert(r, safe=True)
# Verify that all entries retrieved have dates prior to the given end date
end_date = "2013-06-01T12:00:03Z"
end_entries = self.publish_manager.publish_history("test_date", "test_dist", end_date=end_date)
# Confirm the dates of the retrieved entries are earlier than or equal to the requested date
self.assertEqual(2, len(end_entries))
for entries in end_entries:
retrieved = dateutils.parse_iso8601_datetime(entries["started"])
given_end = dateutils.parse_iso8601_datetime(end_date)
self.assertTrue(retrieved <= given_end)
def test_publish_history_start_date(self):
# Setup
self.repo_manager.create_repo('test_date')
self.distributor_manager.add_distributor('test_date', 'mock-distributor', {}, True,
distributor_id='test_dist')
# Create three consecutive publish entries
date_string = '2013-06-01T12:00:0%sZ'
for i in range(0, 6, 2):
r = RepoPublishResult.expected_result(
'test_date', 'test_dist', 'bar', date_string % str(i), date_string % str(i + 1),
'test-summary', 'test-details', RepoPublishResult.RESULT_SUCCESS)
RepoPublishResult.get_collection().insert(r, safe=True)
# Verify
self.assertEqual(3, len(self.publish_manager.publish_history('test_date', 'test_dist')))
start_date = '2013-06-01T12:00:02Z'
start_entries = self.publish_manager.publish_history('test_date', 'test_dist',
start_date=start_date)
# Confirm the dates of the retrieved entries are later than or equal to the requested date
self.assertEqual(2, len(start_entries))
for entries in start_entries:
retrieved = dateutils.parse_iso8601_datetime(entries['started'])
given_start = dateutils.parse_iso8601_datetime(start_date)
self.assertTrue(retrieved >= given_start)
def test_publish_history_end_date(self):
# Setup
self.repo_manager.create_repo('test_date')
self.distributor_manager.add_distributor('test_date',
'mock-distributor', {},
True,
distributor_id='test_dist')
# Create three consecutive publish entries
date_string = '2013-06-01T12:00:0%sZ'
for i in range(0, 6, 2):
r = RepoPublishResult.expected_result(
'test_date', 'test_dist', 'bar', date_string % str(i),
date_string % str(i + 1), 'test-summary', 'test-details',
RepoPublishResult.RESULT_SUCCESS)
RepoPublishResult.get_collection().insert(r, safe=True)
# Verify that all entries retrieved have dates prior to the given end date
end_date = '2013-06-01T12:00:03Z'
end_entries = self.publish_manager.publish_history('test_date',
'test_dist',
end_date=end_date)
# Confirm the dates of the retrieved entries are earlier than or equal to the requested date
self.assertEqual(2, len(end_entries))
for entries in end_entries:
retrieved = dateutils.parse_iso8601_datetime(entries['started'])
given_end = dateutils.parse_iso8601_datetime(end_date)
self.assertTrue(retrieved <= given_end)
def test_publish_history_ascending_sort(self):
"""
Tests use the sort parameter to sort the results in ascending order by start time
"""
# Setup
self.repo_manager.create_repo('test_sort')
self.distributor_manager.add_distributor('test_sort', 'mock-distributor', {}, True,
distributor_id='test_dist')
# Create some consecutive publish entries
date_string = '2013-06-01T12:00:0%sZ'
for i in range(0, 10, 2):
r = RepoPublishResult.expected_result(
'test_sort', 'test_dist', 'bar', date_string % str(i), date_string % str(i + 1),
'test-summary', 'test-details', RepoPublishResult.RESULT_SUCCESS)
RepoPublishResult.get_collection().insert(r, safe=True)
# Test that returned entries are in ascending order by time
entries = self.publish_manager.publish_history('test_sort', 'test_dist',
sort=constants.SORT_ASCENDING)
self.assertEqual(5, len(entries))
for i in range(0, 4):
first = dateutils.parse_iso8601_datetime(entries[i]['started'])
second = dateutils.parse_iso8601_datetime(entries[i + 1]['started'])
self.assertTrue(first < second)
def test_publish_history_descending_sort(self):
"""
Tests use the sort parameter to sort the results in descending order by start time
"""
# Setup
self.repo_manager.create_repo('test_sort')
self.distributor_manager.add_distributor('test_sort',
'mock-distributor', {},
True,
distributor_id='test_dist')
# Create some consecutive publish entries
date_string = '2013-06-01T12:00:0%sZ'
for i in range(0, 10, 2):
r = RepoPublishResult.expected_result(
'test_sort', 'test_dist', 'bar', date_string % str(i),
date_string % str(i + 1), 'test-summary', 'test-details',
RepoPublishResult.RESULT_SUCCESS)
RepoPublishResult.get_collection().insert(r, safe=True)
# Test that returned entries are in descending order by time
entries = self.publish_manager.publish_history(
'test_sort', 'test_dist', sort=constants.SORT_DESCENDING)
self.assertEqual(5, len(entries))
for i in range(0, 4):
first = dateutils.parse_iso8601_datetime(entries[i]['started'])
second = dateutils.parse_iso8601_datetime(entries[i +
1]['started'])
self.assertTrue(first > second)
def add_result(repo_id, dist_id, offset):
started = dateutils.now_utc_datetime_with_tzinfo()
completed = started + datetime.timedelta(days=offset)
r = RepoPublishResult.expected_result(
repo_id, dist_id, 'bar', dateutils.format_iso8601_datetime(started),
dateutils.format_iso8601_datetime(completed), 'test-summary', 'test-details',
RepoPublishResult.RESULT_SUCCESS)
RepoPublishResult.get_collection().insert(r, safe=True)
def add_result(repo_id, dist_id, offset):
started = dateutils.now_utc_datetime_with_tzinfo()
completed = started + datetime.timedelta(days=offset)
r = RepoPublishResult.expected_result(
repo_id, dist_id, 'bar', dateutils.format_iso8601_datetime(started),
dateutils.format_iso8601_datetime(completed), 'test-summary',
'test-details', RepoPublishResult.RESULT_SUCCESS)
RepoPublishResult.get_collection().insert(r, safe=True)
def add_result(repo_id, dist_id, offset):
started = datetime.datetime.now(dateutils.local_tz())
completed = started + datetime.timedelta(days=offset)
r = RepoPublishResult.expected_result(
repo_id,
dist_id,
"bar",
dateutils.format_iso8601_datetime(started),
dateutils.format_iso8601_datetime(completed),
"test-summary",
"test-details",
RepoPublishResult.RESULT_SUCCESS,
)
RepoPublishResult.get_collection().insert(r, safe=True)
class RepoPublishManager(object):
@staticmethod
def publish(repo_id, distributor_id, publish_config_override=None):
"""
Requests the given distributor publish the repository it is configured
on.
The publish operation is executed synchronously in the caller's thread
and will block until it is completed. The caller must take the necessary
steps to address the fact that a publish call may be time intensive.
@param repo_id: identifies the repo being published
@type repo_id: str
@param distributor_id: identifies the repo's distributor to publish
@type distributor_id: str
@param publish_config_override: optional config values to use for this
publish call only
@type publish_config_override: dict, None
:return: report of the details of the publish
:rtype: pulp.server.db.model.repository.RepoPublishResult
"""
repo_coll = Repo.get_collection()
distributor_coll = RepoDistributor.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
if repo_distributor is None:
raise MissingResource(repository=repo_id,
distributor=distributor_id)
distributor_instance, distributor_config = RepoPublishManager.\
_get_distributor_instance_and_config(repo_id, distributor_id)
# Assemble the data needed for the publish
conduit = RepoPublishConduit(repo_id, distributor_id)
call_config = PluginCallConfiguration(distributor_config,
repo_distributor['config'],
publish_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.distributor_working_dir(
repo_distributor['distributor_type_id'], repo_id, mkdir=True)
# Fire events describing the publish state
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_publish_started(repo_id, distributor_id)
result = RepoPublishManager._do_publish(repo, distributor_id,
distributor_instance,
transfer_repo, conduit,
call_config)
fire_manager.fire_repo_publish_finished(result)
return result
@staticmethod
def _get_distributor_instance_and_config(repo_id, distributor_id):
repo_distributor_manager = manager_factory.repo_distributor_manager()
repo_distributor = repo_distributor_manager.get_distributor(
repo_id, distributor_id)
distributor, config = plugin_api.get_distributor_by_id(
repo_distributor['distributor_type_id'])
return distributor, config
@staticmethod
def _do_publish(repo, distributor_id, distributor_instance, transfer_repo,
conduit, call_config):
distributor_coll = RepoDistributor.get_collection()
publish_result_coll = RepoPublishResult.get_collection()
repo_id = repo['id']
# Perform the publish
publish_start_timestamp = _now_timestamp()
try:
# Add the register_sigterm_handler decorator to the publish_repo call, so that we can
# respond to signals by calling the Distributor's cancel_publish_repo() method.
publish_repo = register_sigterm_handler(
distributor_instance.publish_repo,
distributor_instance.cancel_publish_repo)
publish_report = publish_repo(transfer_repo, conduit, call_config)
except Exception, e:
publish_end_timestamp = _now_timestamp()
# Reload the distributor in case the scratchpad is set by the plugin
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
repo_distributor['last_publish'] = publish_end_timestamp
distributor_coll.save(repo_distributor, safe=True)
# Add a publish history entry for the run
result = RepoPublishResult.error_result(
repo_id, repo_distributor['id'],
repo_distributor['distributor_type_id'],
publish_start_timestamp, publish_end_timestamp, e,
sys.exc_info()[2])
publish_result_coll.save(result, safe=True)
logger.exception(
_('Exception caught from plugin during publish for repo [%(r)s]'
% {'r': repo_id}))
raise
publish_end_timestamp = _now_timestamp()
# Reload the distributor in case the scratchpad is set by the plugin
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
repo_distributor['last_publish'] = _now_timestamp()
distributor_coll.save(repo_distributor, safe=True)
# Add a publish entry
if publish_report is not None and isinstance(publish_report,
PublishReport):
summary = publish_report.summary
details = publish_report.details
if publish_report.success_flag:
logger.debug(
'publish succeeded for repo [%s] with distributor ID [%s]'
% (repo_id, distributor_id))
result_code = RepoPublishResult.RESULT_SUCCESS
else:
logger.info(
'publish failed for repo [%s] with distributor ID [%s]' %
(repo_id, distributor_id))
logger.debug(
'summary for repo [%s] with distributor ID [%s]: %s' %
(repo_id, distributor_id, summary))
result_code = RepoPublishResult.RESULT_FAILED
else:
msg = _(
'Plugin type [%(type)s] on repo [%(repo)s] did not return a valid publish '
'report')
msg = msg % {
'type': repo_distributor['distributor_type_id'],
'repo': repo_id
}
logger.warn(msg)
summary = details = _('Unknown')
result_code = RepoPublishResult.RESULT_SUCCESS
result = RepoPublishResult.expected_result(
repo_id, repo_distributor['id'],
repo_distributor['distributor_type_id'], publish_start_timestamp,
publish_end_timestamp, summary, details, result_code)
publish_result_coll.save(result, safe=True)
return result
class RepoPublishManager(object):
def publish(self, repo_id, distributor_id, publish_config_override=None):
"""
Requests the given distributor publish the repository it is configured
on.
The publish operation is executed synchronously in the caller's thread
and will block until it is completed. The caller must take the necessary
steps to address the fact that a publish call may be time intensive.
@param repo_id: identifies the repo being published
@type repo_id: str
@param distributor_id: identifies the repo's distributor to publish
@type distributor_id: str
@param publish_config_override: optional config values to use for this
publish call only
@type publish_config_override: dict, None
"""
repo_coll = Repo.get_collection()
distributor_coll = RepoDistributor.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
if repo_distributor is None:
raise MissingResource(repository=repo_id,
distributor=distributor_id)
distributor_instance, distributor_config = self._get_distributor_instance_and_config(
repo_id, distributor_id)
if distributor_instance is None:
raise MissingResource(repo_id), None, sys.exc_info()[2]
dispatch_context = dispatch_factory.context()
dispatch_context.set_cancel_control_hook(
distributor_instance.cancel_publish_repo)
# Assemble the data needed for the publish
conduit = RepoPublishConduit(repo_id, distributor_id)
call_config = PluginCallConfiguration(distributor_config,
repo_distributor['config'],
publish_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.distributor_working_dir(
repo_distributor['distributor_type_id'], repo_id, mkdir=True)
# Fire events describing the publish state
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_publish_started(repo_id, distributor_id)
result = self._do_publish(repo, distributor_id, distributor_instance,
transfer_repo, conduit, call_config)
fire_manager.fire_repo_publish_finished(result)
dispatch_context.clear_cancel_control_hook()
def _get_distributor_instance_and_config(self, repo_id, distributor_id):
repo_distributor_manager = manager_factory.repo_distributor_manager()
try:
repo_distributor = repo_distributor_manager.get_distributor(
repo_id, distributor_id)
distributor, config = plugin_api.get_distributor_by_id(
repo_distributor['distributor_type_id'])
except (MissingResource, plugin_exceptions.PluginNotFound):
distributor = None
config = None
return distributor, config
def _do_publish(self, repo, distributor_id, distributor_instance,
transfer_repo, conduit, call_config):
distributor_coll = RepoDistributor.get_collection()
publish_result_coll = RepoPublishResult.get_collection()
repo_id = repo['id']
# Perform the publish
publish_start_timestamp = _now_timestamp()
try:
publish_report = distributor_instance.publish_repo(
transfer_repo, conduit, call_config)
except Exception, e:
publish_end_timestamp = _now_timestamp()
# Reload the distributor in case the scratchpad is set by the plugin
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
repo_distributor['last_publish'] = publish_end_timestamp
distributor_coll.save(repo_distributor, safe=True)
# Add a publish history entry for the run
result = RepoPublishResult.error_result(
repo_id, repo_distributor['id'],
repo_distributor['distributor_type_id'],
publish_start_timestamp, publish_end_timestamp, e,
sys.exc_info()[2])
publish_result_coll.save(result, safe=True)
_LOG.exception(
_('Exception caught from plugin during publish for repo [%(r)s]'
% {'r': repo_id}))
raise PulpExecutionException(), None, sys.exc_info()[2]
publish_end_timestamp = _now_timestamp()
# Reload the distributor in case the scratchpad is set by the plugin
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
repo_distributor['last_publish'] = _now_timestamp()
distributor_coll.save(repo_distributor, safe=True)
# Add a publish entry
if publish_report is not None and isinstance(publish_report,
PublishReport):
summary = publish_report.summary
details = publish_report.details
if publish_report.success_flag:
result_code = RepoPublishResult.RESULT_SUCCESS
else:
result_code = RepoPublishResult.RESULT_FAILED
else:
_LOG.warn(
'Plugin type [%s] on repo [%s] did not return a valid publish report'
% (repo_distributor['distributor_type_id'], repo_id))
summary = details = _('Unknown')
result_code = RepoPublishResult.RESULT_SUCCESS
result = RepoPublishResult.expected_result(
repo_id, repo_distributor['id'],
repo_distributor['distributor_type_id'], publish_start_timestamp,
publish_end_timestamp, summary, details, result_code)
publish_result_coll.save(result, safe=True)
return result
publish_end_timestamp = _now_timestamp()
# Reload the distributor in case the scratchpad is set by the plugin
repo_distributor = distributor_coll.find_one({'repo_id': repo_obj.repo_id, 'id': dist_id})
repo_distributor['last_publish'] = datetime.utcnow()
distributor_coll.save(repo_distributor, safe=True)
# Add a publish entry
summary = publish_report.summary
details = publish_report.details
_logger.debug('publish succeeded for repo [%s] with distributor ID [%s]' % (
repo_obj.repo_id, dist_id))
result_code = RepoPublishResult.RESULT_SUCCESS
result = RepoPublishResult.expected_result(
repo_obj.repo_id, repo_distributor['id'], repo_distributor['distributor_type_id'],
publish_start_timestamp, publish_end_timestamp, summary, details, result_code)
publish_result_coll.save(result, safe=True)
return result
def publish_history(start_date, end_date, repo_id, distributor_id):
"""
Returns a cursor containing the publish history entries for the given repo and distributor.
:param start_date: if specified, no events prior to this date will be returned.
:type start_date: iso8601 datetime string
:param end_date: if specified, no events after this date will be returned.
:type end_date: iso8601 datetime string
:param repo_id: identifies the repo
:type repo_id: str
# Reload the distributor in case the scratchpad is set by the plugin
repo_distributor = distributor_coll.find_one({
'repo_id': repo_obj.repo_id,
'id': dist_id
})
repo_distributor['last_publish'] = datetime.utcnow()
distributor_coll.save(repo_distributor, safe=True)
# Add a publish entry
summary = publish_report.summary
details = publish_report.details
_logger.debug('publish succeeded for repo [%s] with distributor ID [%s]' %
(repo_obj.repo_id, dist_id))
result_code = RepoPublishResult.RESULT_SUCCESS
result = RepoPublishResult.expected_result(
repo_obj.repo_id, repo_distributor['id'],
repo_distributor['distributor_type_id'], publish_start_timestamp,
publish_end_timestamp, summary, details, result_code)
publish_result_coll.save(result, safe=True)
return result
def publish_history(start_date, end_date, repo_id, distributor_id):
"""
Returns a cursor containing the publish history entries for the given repo and distributor.
:param start_date: if specified, no events prior to this date will be returned.
:type start_date: iso8601 datetime string
:param end_date: if specified, no events after this date will be returned.
:type end_date: iso8601 datetime string
:param repo_id: identifies the repo
:type repo_id: str
