def _do_sync(self, repo, importer_instance, transfer_repo, conduit, call_config):
"""
Once all of the preparation for a sync has taken place, this call
will perform the sync, making the necessary database updates. It returns
the sync result instance (already saved to the database). This call
does not have any behavior based on the success/failure of the sync;
it is up to the caller to raise an exception in the event of a failed
sync if that behavior is desired.
"""
importer_coll = RepoImporter.get_collection()
sync_result_coll = RepoSyncResult.get_collection()
repo_id = repo['id']
# Perform the sync
sync_start_timestamp = _now_timestamp()
try:
sync_report = importer_instance.sync_repo(transfer_repo, conduit, call_config)
except Exception, e:
# I really wish python 2.4 supported except and finally together
sync_end_timestamp = _now_timestamp()
# Reload the importer in case the plugin edits the scratchpad
repo_importer = importer_coll.find_one({'repo_id' : repo_id})
repo_importer['last_sync'] = sync_end_timestamp
importer_coll.save(repo_importer, safe=True)
# Add a sync history entry for this run
result = RepoSyncResult.error_result(repo_id, repo_importer['id'], repo_importer['importer_type_id'],
sync_start_timestamp, sync_end_timestamp, e, sys.exc_info()[2])
sync_result_coll.save(result, safe=True)
_LOG.exception(_('Exception caught from plugin during sync for repo [%(r)s]' % {'r' : repo_id}))
raise PulpExecutionException(), None, sys.exc_info()[2]
def add_result(repo_id, offset):
started = datetime.datetime.now(dateutils.local_tz())
completed = started + datetime.timedelta(days=offset)
r = RepoSyncResult.expected_result(repo_id, 'foo', 'bar', dateutils.format_iso8601_datetime(started),
dateutils.format_iso8601_datetime(completed), 1, 1, 1, '', '',
RepoSyncResult.RESULT_SUCCESS)
RepoSyncResult.get_collection().save(r, safe=True)
def test_sync_history_ascending_sort(self):
"""
Tests the sort functionality of sync_history
"""
# Setup
self.repo_manager.create_repo('test_sort')
date_string = '2013-06-01T12:00:0%sZ'
# Add some consecutive sync entries
for i in range(0, 10, 2):
r = RepoSyncResult.expected_result('test_sort', 'foo', 'bar',
date_string % str(i),
date_string % str(i + 1), 1, 1,
1, '', '',
RepoSyncResult.RESULT_SUCCESS)
RepoSyncResult.get_collection().save(r, safe=True)
# Test sort by ascending start date
entries = self.sync_manager.sync_history(repo_id='test_sort',
sort=constants.SORT_ASCENDING)
self.assertEqual(5, len(entries))
# Verify that each entry has a earlier completed date than the next one
for i in range(0, 4):
first = dateutils.parse_iso8601_datetime(entries[i]['started'])
second = dateutils.parse_iso8601_datetime(entries[i +
1]['started'])
self.assertTrue(first < second)
def _do_sync(self, repo, importer_instance, transfer_repo, conduit, call_config):
"""
Once all of the preparation for a sync has taken place, this call
will perform the sync, making the necessary database updates. It returns
the sync result instance (already saved to the database). This call
does not have any behavior based on the success/failure of the sync;
it is up to the caller to raise an exception in the event of a failed
sync if that behavior is desired.
"""
importer_coll = RepoImporter.get_collection()
sync_result_coll = RepoSyncResult.get_collection()
repo_id = repo['id']
repo_importer = importer_coll.find_one({'repo_id' : repo_id})
# Perform the sync
sync_start_timestamp = _now_timestamp()
sync_end_timestamp = None
result = None
try:
sync_report = importer_instance.sync_repo(transfer_repo, conduit, call_config)
except Exception, e:
sync_end_timestamp = _now_timestamp()
result = RepoSyncResult.error_result(repo_id, repo_importer['id'], repo_importer['importer_type_id'],
sync_start_timestamp, sync_end_timestamp, e, sys.exc_info()[2])
_LOG.exception(_('Exception caught from plugin during sync for repo [%(r)s]' % {'r' : repo_id}))
raise PulpExecutionException(), None, sys.exc_info()[2]
def _do_sync(repo, importer_instance, transfer_repo, conduit, call_config):
"""
Once all of the preparation for a sync has taken place, this call
will perform the sync, making the necessary database updates. It returns
the sync result instance (already saved to the database). This call
does not have any behavior based on the success/failure of the sync;
it is up to the caller to raise an exception in the event of a failed
sync if that behavior is desired.
"""
importer_coll = RepoImporter.get_collection()
sync_result_coll = RepoSyncResult.get_collection()
repo_id = repo['id']
repo_importer = importer_coll.find_one({'repo_id': repo_id})
# Perform the sync
sync_start_timestamp = _now_timestamp()
sync_end_timestamp = None
result = None
try:
# Replace the Importer's sync_repo() method with our register_sigterm_handler decorator,
# which will set up cancel_sync_repo() as the target for the signal handler
sync_repo = register_sigterm_handler(importer_instance.sync_repo,
importer_instance.cancel_sync_repo)
sync_report = sync_repo(transfer_repo, conduit, call_config)
except Exception, e:
sync_end_timestamp = _now_timestamp()
result = RepoSyncResult.error_result(
repo_id, repo_importer['id'], repo_importer['importer_type_id'],
sync_start_timestamp, sync_end_timestamp, e, sys.exc_info()[2])
raise
def test_sync_history_end_date(self):
"""
Tests the functionality of requesting sync history before a given date
"""
# Setup
self.repo_manager.create_repo('test_repo')
# A date string to fake some dates
date_string = '2013-06-01T12:00:0%sZ'
# Create 3 entries, with each date entry one second later
for i in range(0, 6, 2):
r = RepoSyncResult.expected_result('test_repo', 'foo', 'bar',
date_string % str(i),
date_string % str(i + 1), 1, 1,
1, '', '',
RepoSyncResult.RESULT_SUCCESS)
RepoSyncResult.get_collection().save(r, safe=True)
# Verify three entries in test_repo
self.assertEqual(3, len(self.sync_manager.sync_history('test_repo')))
# Retrieve the first two entries
end_date = '2013-06-01T12:00:03Z'
end_entries = self.sync_manager.sync_history('test_repo',
end_date=end_date)
# Confirm the dates of the retrieved entries are earlier than or equal to the requested date
self.assertEqual(2, len(end_entries))
for entry in end_entries:
retrieved = dateutils.parse_iso8601_datetime(entry['started'])
given_end = dateutils.parse_iso8601_datetime(end_date)
self.assertTrue(retrieved <= given_end)
def test_sync_history_end_date(self):
"""
Tests the functionality of requesting sync history before a given date
"""
# Setup
self.repo_manager.create_repo('test_repo')
# A date string to fake some dates
date_string = '2013-06-01T12:00:0%sZ'
# Create 3 entries, with each date entry one second later
for i in range(0, 6, 2):
r = RepoSyncResult.expected_result('test_repo', 'foo', 'bar', date_string % str(i),
date_string % str(i + 1), 1, 1, 1, '', '',
RepoSyncResult.RESULT_SUCCESS)
RepoSyncResult.get_collection().save(r, safe=True)
# Verify three entries in test_repo
self.assertEqual(3, len(self.sync_manager.sync_history('test_repo')))
# Retrieve the first two entries
end_date = '2013-06-01T12:00:03Z'
end_entries = self.sync_manager.sync_history('test_repo', end_date=end_date)
# Confirm the dates of the retrieved entries are earlier than or equal to the requested date
self.assertEqual(2, len(end_entries))
for entry in end_entries:
retrieved = dateutils.parse_iso8601_datetime(entry['started'])
given_end = dateutils.parse_iso8601_datetime(end_date)
self.assertTrue(retrieved <= given_end)
def sync(repo_id, sync_config_override=None):
"""
Performs a synchronize operation on the given repository and triggers publishs for distributors
with autopublish enabled.
The given repo must have an importer configured. This method is intentionally limited to
synchronizing a single repo. Performing multiple repository syncs concurrently will require a
more global view of the server and must be handled outside the scope of this class.
:param repo_id: identifies the repo to sync
:type repo_id: str
:param sync_config_override: optional config containing values to use for this sync only
:type sync_config_override: dict
:return: TaskResult containing sync results and a list of spawned tasks
:rtype: pulp.server.async.tasks.TaskResult
:raise pulp_exceptions.MissingResource: if specified repo does not exist, or it does not have
an importer and associated plugin
:raise pulp_exceptions.PulpExecutionException: if the task fails.
"""
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
transfer_repo = repo_obj.to_transfer_repo()
importer_collection = RepoImporter.get_collection()
repo_importer = importer_collection.find_one({'repo_id': repo_obj.repo_id})
if repo_importer is None:
raise pulp_exceptions.MissingResource(repository=repo_id)
try:
importer, imp_config = plugin_api.get_importer_by_id(repo_importer['importer_type_id'])
except plugin_exceptions.PluginNotFound:
raise pulp_exceptions.MissingResource(repository=repo_id)
call_config = PluginCallConfiguration(imp_config, repo_importer['config'], sync_config_override)
transfer_repo.working_dir = common_utils.get_working_directory()
conduit = RepoSyncConduit(repo_id, repo_importer['id'])
sync_result_collection = RepoSyncResult.get_collection()
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
# Perform the sync
sync_start_timestamp = _now_timestamp()
sync_result = None
try:
# Replace the Importer's sync_repo() method with our register_sigterm_handler decorator,
# which will set up cancel_sync_repo() as the target for the signal handler
sync_repo = register_sigterm_handler(importer.sync_repo, importer.cancel_sync_repo)
sync_report = sync_repo(transfer_repo, conduit, call_config)
except Exception, e:
sync_end_timestamp = _now_timestamp()
sync_result = RepoSyncResult.error_result(
repo_obj.repo_id, repo_importer['id'], repo_importer['importer_type_id'],
sync_start_timestamp, sync_end_timestamp, e, sys.exc_info()[2])
raise
def tearDown(self):
super(TestDoSync, self).tearDown()
mock_plugins.reset()
manager_factory.reset()
Repo.get_collection().remove()
RepoImporter.get_collection().remove()
RepoSyncResult.get_collection().remove()
MockRepoPublishManager.reset()
def add_result(repo_id, offset):
started = datetime.datetime.now(dateutils.local_tz())
completed = started + datetime.timedelta(days=offset)
r = RepoSyncResult.expected_result(
repo_id, 'foo', 'bar', dateutils.format_iso8601_datetime(started),
dateutils.format_iso8601_datetime(completed), 1, 1, 1, '', '',
RepoSyncResult.RESULT_SUCCESS)
RepoSyncResult.get_collection().save(r, safe=True)
def clean(self):
super(RepoSyncManagerTests, self).clean()
Repo.get_collection().remove()
RepoImporter.get_collection().remove()
RepoSyncResult.get_collection().remove()
# Reset the state of the mock's tracker variables
MockRepoPublishManager.reset()
def tearDown(self):
super(TestDoSync, self).tearDown()
mock_plugins.reset()
manager_factory.reset()
Repo.get_collection().remove()
RepoImporter.get_collection().remove()
RepoSyncResult.get_collection().remove()
MockRepoPublishManager.reset()
def clean(self):
super(RepoSyncManagerTests, self).clean()
Repo.get_collection().remove()
RepoImporter.get_collection().remove()
RepoSyncResult.get_collection().remove()
# Reset the state of the mock's tracker variables
MockRepoPublishManager.reset()
def sync(repo_id, sync_config_override=None, scheduled_call_id=None):
"""
Performs a synchronize operation on the given repository and triggers publishes for
distributors with auto-publish enabled.
The given repo must have an importer configured. This method is intentionally limited to
synchronizing a single repo. Performing multiple repository syncs concurrently will require a
more global view of the server and must be handled outside the scope of this class.
:param repo_id: identifies the repo to sync
:type repo_id: str
:param sync_config_override: optional config containing values to use for this sync only
:type sync_config_override: dict
:param scheduled_call_id: id of scheduled call that dispatched this task
:type scheduled_call_id: str
:return: TaskResult containing sync results and a list of spawned tasks
:rtype: pulp.server.async.tasks.TaskResult
:raise pulp_exceptions.MissingResource: if specified repo does not exist, or it does not have
an importer and associated plugin
:raise pulp_exceptions.PulpExecutionException: if the task fails.
"""
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
transfer_repo = repo_obj.to_transfer_repo()
repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id)
try:
importer, imp_config = plugin_api.get_importer_by_id(repo_importer.importer_type_id)
except plugin_exceptions.PluginNotFound:
raise pulp_exceptions.MissingResource(repository=repo_id)
call_config = PluginCallConfiguration(imp_config, repo_importer.config, sync_config_override)
transfer_repo.working_dir = common_utils.get_working_directory()
conduit = RepoSyncConduit(repo_id, repo_importer.importer_type_id, repo_importer.id)
sync_result_collection = RepoSyncResult.get_collection()
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
# Perform the sync
sync_start_timestamp = _now_timestamp()
sync_result = None
try:
# Replace the Importer's sync_repo() method with our register_sigterm_handler decorator,
# which will set up cancel_sync_repo() as the target for the signal handler
sync_repo = register_sigterm_handler(importer.sync_repo, importer.cancel_sync_repo)
sync_report = sync_repo(transfer_repo, conduit, call_config)
except Exception, e:
sync_end_timestamp = _now_timestamp()
sync_result = RepoSyncResult.error_result(
repo_obj.repo_id, repo_importer['id'], repo_importer['importer_type_id'],
sync_start_timestamp, sync_end_timestamp, e, sys.exc_info()[2])
raise
def sync_history(start_date, end_date, repo_id):
"""
Returns a cursor containing the sync history entries for the given repo.
:param start_date: if specified, no events prior to this date will be returned. Expected to be
an iso8601 datetime string.
:type start_date: str
:param end_date: if specified, no events after this date will be returned. Expected to be an
iso8601 datetime string.
:type end_date: str
:param repo_id: identifies the repo
:type repo_id: str
:return: object containing sync history results
:rtype: pymongo.cursor.Cursor
:raise MissingResource: if repo_id does not reference a valid repo
"""
model.Repository.objects.get_repo_or_missing_resource(repo_id)
search_params = {'repo_id': repo_id}
date_range = {}
if start_date:
date_range['$gte'] = start_date
if end_date:
date_range['$lte'] = end_date
if start_date or end_date:
search_params['started'] = date_range
return RepoSyncResult.get_collection().find(search_params)
def sync_history(self, repo_id, limit=None):
"""
Returns sync history entries for the given repo, sorted from most recent
to oldest. If there are no entries, an empty list is returned.
@param repo_id: identifies the repo
@type repo_id: str
@param limit: maximum number of results to return
@type limit: int
@return: list of sync history result instances
@rtype: list of L{pulp.server.db.model.repository.RepoSyncResult}
@raise MissingResource: if repo_id does not reference a valid repo
"""
# Validation
repo = Repo.get_collection().find_one({'id' : repo_id})
if repo is None:
raise MissingResource(repo_id)
if limit is None:
limit = 10 # default here for each of REST API calls into here
# Retrieve the entries
cursor = RepoSyncResult.get_collection().find({'repo_id' : repo_id})
cursor.limit(limit)
cursor.sort('completed', pymongo.DESCENDING)
return list(cursor)
def sync_history(start_date, end_date, repo_id):
"""
Returns a cursor containing the sync history entries for the given repo.
:param start_date: if specified, no events prior to this date will be returned. Expected to be
an iso8601 datetime string.
:type start_date: str
:param end_date: if specified, no events after this date will be returned. Expected to be an
iso8601 datetime string.
:type end_date: str
:param repo_id: identifies the repo
:type repo_id: str
:return: object containing sync history results
:rtype: pymongo.cursor.Cursor
:raise MissingResource: if repo_id does not reference a valid repo
"""
model.Repository.objects.get_repo_or_missing_resource(repo_id)
search_params = {'repo_id': repo_id}
date_range = {}
if start_date:
date_range['$gte'] = start_date
if end_date:
date_range['$lte'] = end_date
if start_date or end_date:
search_params['started'] = date_range
return RepoSyncResult.get_collection().find(search_params)
def sync_history(self, repo_id, limit=None):
"""
Returns sync history entries for the given repo, sorted from most recent
to oldest. If there are no entries, an empty list is returned.
@param repo_id: identifies the repo
@type repo_id: str
@param limit: maximum number of results to return
@type limit: int
@return: list of sync history result instances
@rtype: list of L{pulp.server.db.model.repository.RepoSyncResult}
@raise MissingResource: if repo_id does not reference a valid repo
"""
# Validation
repo = Repo.get_collection().find_one({'id' : repo_id})
if repo is None:
raise MissingResource(repo_id)
if limit is None:
limit = 10 # default here for each of REST API calls into here
# Retrieve the entries
cursor = RepoSyncResult.get_collection().find({'repo_id' : repo_id})
cursor.limit(limit)
cursor.sort('completed', pymongo.DESCENDING)
return list(cursor)
def test_sync_with_graceful_fail(self):
# Setup
sync_config = {'bruce': 'hulk', 'tony': 'ironman'}
self.repo_manager.create_repo('repo-1')
self.importer_manager.set_importer('repo-1', 'mock-importer',
sync_config)
mock_plugins.MOCK_IMPORTER.sync_repo.return_value = SyncReport(
False, 10, 5, 1, 'Summary of the sync', 'Details of the sync')
# Test
self.assertRaises(PulpExecutionException, self.sync_manager.sync,
'repo-1')
# Verify
history = list(RepoSyncResult.get_collection().find(
{'repo_id': 'repo-1'}))
self.assertEqual(1, len(history))
self.assertEqual('repo-1', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_FAILED, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
# Cleanup
mock_plugins.reset()
def test_sync(self, mock_finished, mock_started):
"""
Tests sync under normal conditions where everything is configured
correctly. No importer config is specified.
"""
# Setup
sync_config = {'bruce': 'hulk', 'tony': 'ironman'}
self.repo_manager.create_repo('repo-1')
self.importer_manager.set_importer('repo-1', 'mock-importer', sync_config)
# Test
self.sync_manager.sync('repo-1', sync_config_override=None)
# Verify
repo = Repo.get_collection().find_one({'id': 'repo-1'})
repo_importer = RepoImporter.get_collection().find_one({'repo_id': 'repo-1',
'id': 'mock-importer'})
# Database
self.assertTrue(repo_importer['last_sync'] is not None)
self.assertTrue(assert_last_sync_time(repo_importer['last_sync']))
# Call into the Importer
sync_args = mock_plugins.MOCK_IMPORTER.sync_repo.call_args[0]
self.assertEqual(repo['id'], sync_args[0].id)
self.assertTrue(sync_args[1] is not None)
self.assertEqual({}, sync_args[2].plugin_config)
self.assertEqual(sync_config, sync_args[2].repo_plugin_config)
self.assertEqual({}, sync_args[2].override_config)
# History Entry
history = list(RepoSyncResult.get_collection().find({'repo_id': 'repo-1'}))
self.assertEqual(1, len(history))
self.assertEqual('repo-1', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_SUCCESS, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
self.assertEqual(10, history[0]['added_count'])
self.assertEqual(1, history[0]['removed_count'])
self.assertTrue(history[0]['summary'] is not None)
self.assertTrue(history[0]['details'] is not None)
self.assertTrue(history[0]['error_message'] is None)
self.assertTrue(history[0]['exception'] is None)
self.assertTrue(history[0]['traceback'] is None)
self.assertEqual(1, mock_started.call_count)
self.assertEqual('repo-1', mock_started.call_args[0][0])
self.assertEqual(1, mock_finished.call_count)
self.assertEqual('repo-1', mock_finished.call_args[0][0]['repo_id'])
def test_sync_with_error(self):
"""
Tests a sync when the plugin raises an error.
"""
# Setup
class FakePluginException(Exception):
pass
error_msg = 'Error test'
mock_plugins.MOCK_IMPORTER.sync_repo.side_effect = FakePluginException(
error_msg)
self.repo_manager.create_repo('gonna-bail')
self.importer_manager.set_importer('gonna-bail', 'mock-importer', {})
# Test
self.assertRaises(Exception, self.sync_manager.sync, 'gonna-bail')
# Verify
# Database
repo_importer = RepoImporter.get_collection().find_one({
'repo_id':
'gonna-bail',
'id':
'mock-importer'
})
self.assertTrue(repo_importer['last_sync'] is not None)
self.assertTrue(assert_last_sync_time(repo_importer['last_sync']))
# History Entry
history = list(RepoSyncResult.get_collection().find(
{'repo_id': 'gonna-bail'}))
self.assertEqual(1, len(history))
self.assertEqual('gonna-bail', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_ERROR, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
self.assertTrue(history[0]['added_count'] is None)
self.assertTrue(history[0]['updated_count'] is None)
self.assertTrue(history[0]['removed_count'] is None)
self.assertTrue(history[0]['summary'] is None)
self.assertTrue(history[0]['details'] is None)
self.assertEqual(error_msg, history[0]['error_message'])
self.assertTrue('FakePluginException' in history[0]['exception'])
self.assertTrue(history[0]['traceback'] is not None)
# Cleanup
mock_plugins.MOCK_IMPORTER.sync_repo.side_effect = None
def test_sync_history_descending_sort(self):
# Setup
self.repo_manager.create_repo('test_sort')
date_string = '2013-06-01T12:00:0%sZ'
# Add some consecutive sync entries
for i in range(0, 10, 2):
r = RepoSyncResult.expected_result('test_sort', 'foo', 'bar', date_string % str(i),
date_string % str(i + 1), 1, 1, 1, '', '',
RepoSyncResult.RESULT_SUCCESS)
RepoSyncResult.get_collection().save(r, safe=True)
# Test sort by descending start date
entries = self.sync_manager.sync_history(repo_id='test_sort', sort=constants.SORT_DESCENDING)
self.assertEqual(5, len(entries))
# Verify that each entry has a later completed date than the next one
for i in range(0, 4):
first = dateutils.parse_iso8601_datetime(entries[i]['started'])
second = dateutils.parse_iso8601_datetime(entries[i + 1]['started'])
self.assertTrue(first > second)
def _do_sync(repo, importer_instance, transfer_repo, conduit, call_config):
"""
Once all of the preparation for a sync has taken place, this call
will perform the sync, making the necessary database updates. It returns
the sync result instance (already saved to the database). This call
does not have any behavior based on the success/failure of the sync;
it is up to the caller to raise an exception in the event of a failed
sync if that behavior is desired.
"""
importer_coll = RepoImporter.get_collection()
sync_result_coll = RepoSyncResult.get_collection()
repo_id = repo['id']
repo_importer = importer_coll.find_one({'repo_id': repo_id})
# Perform the sync
sync_start_timestamp = _now_timestamp()
sync_end_timestamp = None
result = None
try:
# Replace the Importer's sync_repo() method with our register_sigterm_handler decorator,
# which will set up cancel_sync_repo() as the target for the signal handler
sync_repo = register_sigterm_handler(
importer_instance.sync_repo,
importer_instance.cancel_sync_repo)
sync_report = sync_repo(transfer_repo, conduit, call_config)
except Exception, e:
sync_end_timestamp = _now_timestamp()
result = RepoSyncResult.error_result(
repo_id, repo_importer['id'],
repo_importer['importer_type_id'], sync_start_timestamp,
sync_end_timestamp, e,
sys.exc_info()[2])
_logger.exception(
_('Exception caught from plugin during sync for repo [%(r)s]' %
{'r': repo_id}))
raise
def test_sync_cancelled(self, mock_current_task_state):
"""
Test the repo sync result on a cancelled sync.
"""
repo_id = 'cancelled_repo'
self.repo_manager.create_repo(repo_id)
self.importer_manager.set_importer(repo_id, 'mock-importer', {})
mock_plugins.MOCK_IMPORTER.sync_repo.return_value = None
self.sync_manager.sync(repo_id)
sync_result = RepoSyncResult.get_collection().find_one({'repo_id': repo_id})
self.assertFalse(sync_result is None)
self.assertEqual(sync_result['result'], RepoSyncResult.RESULT_CANCELED)
self.assertEqual(mock_current_task_state.call_count, 1)
def test_sync_with_error(self):
"""
Tests a sync when the plugin raises an error.
"""
# Setup
class FakePluginException(Exception):
pass
error_msg = 'Error test'
mock_plugins.MOCK_IMPORTER.sync_repo.side_effect = FakePluginException(error_msg)
self.repo_manager.create_repo('gonna-bail')
self.importer_manager.set_importer('gonna-bail', 'mock-importer', {})
# Test
self.assertRaises(Exception, self.sync_manager.sync, 'gonna-bail')
# Database
repo_importer = RepoImporter.get_collection().find_one({'repo_id': 'gonna-bail',
'id': 'mock-importer'})
self.assertTrue(repo_importer['last_sync'] is not None)
self.assertTrue(assert_last_sync_time(repo_importer['last_sync']))
# History Entry
history = list(RepoSyncResult.get_collection().find({'repo_id': 'gonna-bail'}))
self.assertEqual(1, len(history))
self.assertEqual('gonna-bail', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_ERROR, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
self.assertTrue(history[0]['added_count'] is None)
self.assertTrue(history[0]['updated_count'] is None)
self.assertTrue(history[0]['removed_count'] is None)
self.assertTrue(history[0]['summary'] is None)
self.assertTrue(history[0]['details'] is None)
self.assertEqual(error_msg, history[0]['error_message'])
self.assertTrue('FakePluginException' in history[0]['exception'])
self.assertTrue(history[0]['traceback'] is not None)
# Cleanup
mock_plugins.MOCK_IMPORTER.sync_repo.side_effect = None
def test_sync_no_plugin_report(self):
"""
Tests synchronizing against a sloppy plugin that doesn't return a sync report.
"""
# Setup
self.repo_manager.create_repo('repo-1')
self.importer_manager.set_importer('repo-1', 'mock-importer', {})
mock_plugins.MOCK_IMPORTER.sync_repo.return_value = None # sloppy plugin
# Test
self.sync_manager.sync('repo-1')
# Verify
# History Entry
history = list(RepoSyncResult.get_collection().find(
{'repo_id': 'repo-1'}))
self.assertEqual(1, len(history))
self.assertEqual('repo-1', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_ERROR, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
self.assertEqual(-1, history[0]['added_count'])
self.assertEqual(-1, history[0]['updated_count'])
self.assertEqual(-1, history[0]['removed_count'])
expected_message = (
'Plugin type [mock-importer] on repo [repo-1] did not return a valid '
'sync report')
self.assertEqual(expected_message, history[0]['summary'])
self.assertEqual(expected_message, history[0]['details'])
self.assertTrue(history[0]['error_message'] is None)
self.assertTrue(history[0]['exception'] is None)
self.assertTrue(history[0]['traceback'] is None)
def test_sync_no_plugin_report(self):
"""
Tests synchronizing against a sloppy plugin that doesn't return a sync report.
"""
# Setup
self.repo_manager.create_repo('repo-1')
self.importer_manager.set_importer('repo-1', 'mock-importer', {})
mock_plugins.MOCK_IMPORTER.sync_repo.return_value = None # sloppy plugin
# Test
self.sync_manager.sync('repo-1')
# Verify
# History Entry
history = list(RepoSyncResult.get_collection().find({'repo_id' : 'repo-1'}))
self.assertEqual(1, len(history))
self.assertEqual('repo-1', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_ERROR, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
self.assertEqual(-1, history[0]['added_count'])
self.assertEqual(-1, history[0]['updated_count'])
self.assertEqual(-1, history[0]['removed_count'])
expected_message = ('Plugin type [mock-importer] on repo [repo-1] did not return a valid '
'sync report')
self.assertEqual(expected_message, history[0]['summary'])
self.assertEqual(expected_message, history[0]['details'])
self.assertTrue(history[0]['error_message'] is None)
self.assertTrue(history[0]['exception'] is None)
self.assertTrue(history[0]['traceback'] is None)
def test_sync_with_graceful_fail(self):
# Setup
sync_config = {'bruce' : 'hulk', 'tony' : 'ironman'}
self.repo_manager.create_repo('repo-1')
self.importer_manager.set_importer('repo-1', 'mock-importer', sync_config)
mock_plugins.MOCK_IMPORTER.sync_repo.return_value = SyncReport(False, 10, 5, 1, 'Summary of the sync', 'Details of the sync')
# Test
self.assertRaises(PulpExecutionException, self.sync_manager.sync, 'repo-1')
# Verify
history = list(RepoSyncResult.get_collection().find({'repo_id' : 'repo-1'}))
self.assertEqual(1, len(history))
self.assertEqual('repo-1', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_FAILED, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
# Cleanup
mock_plugins.reset()
elif sync_report.success_flag:
result_code = RepoSyncResult.RESULT_SUCCESS
else:
result_code = RepoSyncResult.RESULT_FAILED
else:
_LOG.warn('Plugin type [%s] on repo [%s] did not return a valid sync report' % (repo_importer['importer_type_id'], repo_id))
added_count = updated_count = removed_count = -1 # None?
summary = details = _('Unknown')
result_code = RepoSyncResult.RESULT_ERROR # RESULT_UNKNOWN?
result = RepoSyncResult.expected_result(repo_id, repo_importer['id'], repo_importer['importer_type_id'],
sync_start_timestamp, sync_end_timestamp, added_count, updated_count,
removed_count, summary, details, result_code)
finally:
# Do an update instead of a save in case the importer has changed the scratchpad
importer_coll.update({'repo_id': repo_id}, {'$set': {'last_sync': sync_end_timestamp}}, safe=True)
# Add a sync history entry for this run
sync_result_coll.save(result, safe=True)
return result
def sync_history(self, repo_id, limit=None):
"""
Returns sync history entries for the given repo, sorted from most recent
to oldest. If there are no entries, an empty list is returned.
error_tuples.append((_('Filesystem Cleanup Error'), e.args))
# Database Updates
try:
Repo.get_collection().remove({'id': repo_id}, safe=True)
# Remove all importers and distributors from the repo
# This is likely already done by the calls to other methods in
# this manager, but in case those failed we still want to attempt
# to keep the database clean
RepoDistributor.get_collection().remove({'repo_id': repo_id},
safe=True)
RepoImporter.get_collection().remove({'repo_id': repo_id},
safe=True)
RepoSyncResult.get_collection().remove({'repo_id': repo_id},
safe=True)
RepoPublishResult.get_collection().remove({'repo_id': repo_id},
safe=True)
# Remove all associations from the repo
RepoContentUnit.get_collection().remove({'repo_id': repo_id},
safe=True)
except Exception, e:
_LOG.exception(
'Error updating one or more database collections while removing repo [%s]'
% repo_id)
error_tuples.append((_('Database Removal Error'), e.args))
# remove the repo from any groups it was a member of
group_manager = manager_factory.repo_group_manager()
group_manager.remove_repo_from_groups(repo_id)
'Error received removing distributor [%s] from repo [%s]' %
(repo_distributor['id'], repo_id))
error_tuples.append(e)
# Database Updates
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
repo.delete()
try:
# Remove all importers and distributors from the repo. This is likely already done by the
# calls to other methods in this manager, but in case those failed we still want to attempt
# to keep the database clean.
RepoDistributor.get_collection().remove({'repo_id': repo_id},
safe=True)
RepoImporter.get_collection().remove({'repo_id': repo_id}, safe=True)
RepoSyncResult.get_collection().remove({'repo_id': repo_id}, safe=True)
RepoPublishResult.get_collection().remove({'repo_id': repo_id},
safe=True)
RepoContentUnit.get_collection().remove({'repo_id': repo_id},
safe=True)
except Exception, e:
msg = _(
'Error updating one or more database collections while removing repo [%(r)s]'
)
msg = msg % {'r': repo_id}
_logger.exception(msg)
error_tuples.append(e)
# remove the repo from any groups it was a member of
group_manager = manager_factory.repo_group_manager()
group_manager.remove_repo_from_groups(repo_id)
try:
importer, config = self.sync_manager._get_importer_instance_and_config('gonna-bail')
self.sync_manager.sync('gonna-bail', importer, config)
except repo_sync_manager.PulpExecutionException, e:
print(e) # for coverage
# Verify
# Database
repo_importer = RepoImporter.get_collection().find_one({'repo_id' : 'gonna-bail', 'id' : 'mock-importer'})
self.assertTrue(repo_importer['last_sync'] is not None)
self.assertTrue(assert_last_sync_time(repo_importer['last_sync']))
# History Entry
history = list(RepoSyncResult.get_collection().find({'repo_id' : 'gonna-bail'}))
self.assertEqual(1, len(history))
self.assertEqual('gonna-bail', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_ERROR, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
self.assertTrue(history[0]['added_count'] is None)
self.assertTrue(history[0]['updated_count'] is None)
self.assertTrue(history[0]['removed_count'] is None)
self.assertTrue(history[0]['summary'] is None)
self.assertTrue(history[0]['details'] is None)
self.assertEqual(error_msg, history[0]['error_message'])
result_code = RepoSyncResult.RESULT_SUCCESS
else:
result_code = RepoSyncResult.RESULT_FAILED
else:
msg = _('Plugin type [%s] on repo [%s] did not return a valid sync report')
msg = msg % (repo_importer['importer_type_id'], repo_id)
logger.warn(msg)
added_count = updated_count = removed_count = -1 # None?
summary = details = msg
result_code = RepoSyncResult.RESULT_ERROR # RESULT_UNKNOWN?
result = RepoSyncResult.expected_result(
repo_id, repo_importer['id'], repo_importer['importer_type_id'],
sync_start_timestamp, sync_end_timestamp, added_count, updated_count, removed_count,
summary, details, result_code)
finally:
# Do an update instead of a save in case the importer has changed the scratchpad
importer_coll.update({'repo_id': repo_id}, {'$set': {'last_sync': sync_end_timestamp}},
safe=True)
# Add a sync history entry for this run
sync_result_coll.save(result, safe=True)
return result
def sync_history(self, repo_id, limit=None, sort=constants.SORT_DESCENDING, start_date=None,
end_date=None):
"""
Returns sync history entries for the given repo, sorted from most recent
# Verify
# Database
repo_importer = RepoImporter.get_collection().find_one({
'repo_id':
'gonna-bail',
'id':
'mock-importer'
})
self.assertTrue(repo_importer['last_sync'] is not None)
self.assertTrue(assert_last_sync_time(repo_importer['last_sync']))
# History Entry
history = list(RepoSyncResult.get_collection().find(
{'repo_id': 'gonna-bail'}))
self.assertEqual(1, len(history))
self.assertEqual('gonna-bail', history[0]['repo_id'])
self.assertEqual(RepoSyncResult.RESULT_ERROR, history[0]['result'])
self.assertEqual('mock-importer', history[0]['importer_id'])
self.assertEqual('mock-importer', history[0]['importer_type_id'])
self.assertTrue(history[0]['started'] is not None)
self.assertTrue(history[0]['completed'] is not None)
self.assertTrue(history[0]['added_count'] is None)
self.assertTrue(history[0]['updated_count'] is None)
self.assertTrue(history[0]['removed_count'] is None)
self.assertTrue(history[0]['summary'] is None)
self.assertTrue(history[0]['details'] is None)
self.assertEqual(error_msg, history[0]['error_message'])
def sync_history(self, repo_id, limit=None, sort=constants.SORT_DESCENDING, start_date=None,
end_date=None):
"""
Returns sync history entries for the given repo, sorted from most recent
to oldest. If there are no entries, an empty list is returned.
:param repo_id: identifies the repo
:type repo_id: str
:param limit: if specified, the query will only return up to this amount of
entries; default is to return the entire sync history
:type limit: int
:param sort: Indicates the sort direction of the results, which are sorted by start date. Options
are "ascending" and "descending". Descending is the default.
:type sort: str
:param start_date: if specified, no events prior to this date will be returned. Expected to be an
iso8601 datetime string.
:type start_date: str
:param end_date: if specified, no events after this date will be returned. Expected to be an
iso8601 datetime string.
:type end_date: str
:return: list of sync history result instances
:rtype: list
:raise MissingResource: if repo_id does not reference a valid repo
:raise InvalidValue: if one or more options are invalid
"""
# Validation
repo = Repo.get_collection().find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
invalid_values = []
# Verify the limit makes sense
if limit is not None:
try:
limit = int(limit)
if limit < 1:
invalid_values.append('limit')
except ValueError:
invalid_values.append('limit')
# Verify the sort direction is valid
if sort not in constants.SORT_DIRECTION:
invalid_values.append('sort')
# Verify that start_date and end_date is valid
if start_date is not None:
try:
dateutils.parse_iso8601_datetime(start_date)
except (ValueError, isodate.ISO8601Error):
invalid_values.append('start_date')
if end_date is not None:
try:
dateutils.parse_iso8601_datetime(end_date)
except (ValueError, isodate.ISO8601Error):
invalid_values.append('end_date')
# Report any invalid values
if invalid_values:
raise InvalidValue(invalid_values)
# Assemble the mongo search parameters
search_params = {'repo_id': repo_id}
# Add in date range limits if specified
date_range = {}
if start_date:
date_range['$gte'] = start_date
if end_date:
date_range['$lte'] = end_date
if len(date_range) > 0:
search_params['started'] = date_range
# Retrieve the entries
cursor = RepoSyncResult.get_collection().find(search_params)
# Sort the results on the 'started' field. By default, descending order is used
cursor.sort('started', direction=constants.SORT_DIRECTION[sort])
if limit is not None:
cursor.limit(limit)
return list(cursor)
def sync_history(self,
repo_id,
limit=None,
sort=constants.SORT_DESCENDING,
start_date=None,
end_date=None):
"""
Returns sync history entries for the given repo, sorted from most recent
to oldest. If there are no entries, an empty list is returned.
:param repo_id: identifies the repo
:type repo_id: str
:param limit: if specified, the query will only return up to this amount of
entries; default is to return the entire sync history
:type limit: int
:param sort: Indicates the sort direction of the results, which are sorted by
start date. Options are "ascending" and "descending". Descending is
the default.
:type sort: str
:param start_date: if specified, no events prior to this date will be returned.
Expected to be an iso8601 datetime string.
:type start_date: str
:param end_date: if specified, no events after this date will be returned. Expected
to be an iso8601 datetime string.
:type end_date: str
:return: list of sync history result instances
:rtype: list
:raise MissingResource: if repo_id does not reference a valid repo
:raise InvalidValue: if one or more options are invalid
"""
# Validation
repo = Repo.get_collection().find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
invalid_values = []
# Verify the limit makes sense
if limit is not None:
try:
limit = int(limit)
if limit < 1:
invalid_values.append('limit')
except ValueError:
invalid_values.append('limit')
# Verify the sort direction is valid
if sort not in constants.SORT_DIRECTION:
invalid_values.append('sort')
# Verify that start_date and end_date is valid
if start_date is not None:
try:
dateutils.parse_iso8601_datetime(start_date)
except (ValueError, isodate.ISO8601Error):
invalid_values.append('start_date')
if end_date is not None:
try:
dateutils.parse_iso8601_datetime(end_date)
except (ValueError, isodate.ISO8601Error):
invalid_values.append('end_date')
# Report any invalid values
if invalid_values:
raise InvalidValue(invalid_values)
# Assemble the mongo search parameters
search_params = {'repo_id': repo_id}
# Add in date range limits if specified
date_range = {}
if start_date:
date_range['$gte'] = start_date
if end_date:
date_range['$lte'] = end_date
if len(date_range) > 0:
search_params['started'] = date_range
# Retrieve the entries
cursor = RepoSyncResult.get_collection().find(search_params)
# Sort the results on the 'started' field. By default, descending order is used
cursor.sort('started', direction=constants.SORT_DIRECTION[sort])
if limit is not None:
cursor.limit(limit)
return list(cursor)
logger.exception('Error while deleting repo working dir [%s] for repo [%s]' % (
repo_working_dir, repo_id))
error_tuples.append(e)
# Database Updates
try:
Repo.get_collection().remove({'id' : repo_id}, safe=True)
# Remove all importers and distributors from the repo
# This is likely already done by the calls to other methods in
# this manager, but in case those failed we still want to attempt
# to keep the database clean
RepoDistributor.get_collection().remove({'repo_id' : repo_id}, safe=True)
RepoImporter.get_collection().remove({'repo_id' : repo_id}, safe=True)
RepoSyncResult.get_collection().remove({'repo_id' : repo_id}, safe=True)
RepoPublishResult.get_collection().remove({'repo_id' : repo_id}, safe=True)
# Remove all associations from the repo
RepoContentUnit.get_collection().remove({'repo_id' : repo_id}, safe=True)
except Exception, e:
msg = _('Error updating one or more database collections while removing repo [%(r)s]')
msg = msg % {'r': repo_id}
logger.exception(msg)
error_tuples.append(e)
# remove the repo from any groups it was a member of
group_manager = manager_factory.repo_group_manager()
group_manager.remove_repo_from_groups(repo_id)
if len(error_tuples) > 0:
def _validate_repo_sync_result():
objectdb = RepoSyncResult.get_collection()
reference = RepoSyncResult('', '', '', '', '', '')
return _validate_model(RepoSyncResult.__name__, objectdb, reference)
except Exception, e:
_logger.exception('Error received removing distributor [%s] from repo [%s]' % (
repo_distributor['id'], repo_id))
error_tuples.append(e)
# Database Updates
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
repo.delete()
try:
# Remove all importers and distributors from the repo. This is likely already done by the
# calls to other methods in this manager, but in case those failed we still want to attempt
# to keep the database clean.
RepoDistributor.get_collection().remove({'repo_id': repo_id}, safe=True)
RepoImporter.get_collection().remove({'repo_id': repo_id}, safe=True)
RepoSyncResult.get_collection().remove({'repo_id': repo_id}, safe=True)
RepoPublishResult.get_collection().remove({'repo_id': repo_id}, safe=True)
RepoContentUnit.get_collection().remove({'repo_id': repo_id}, safe=True)
except Exception, e:
msg = _('Error updating one or more database collections while removing repo [%(r)s]')
msg = msg % {'r': repo_id}
_logger.exception(msg)
error_tuples.append(e)
# remove the repo from any groups it was a member of
group_manager = manager_factory.repo_group_manager()
group_manager.remove_repo_from_groups(repo_id)
if len(error_tuples) > 0:
pe = pulp_exceptions.PulpExecutionException()
pe.child_exceptions = error_tuples