def test_upgrade(self):
# Setup
coll = Bind.get_collection()
for counter in range(0, 3):
bind_dict = {
'consumer_id': 'consumer_%s' % counter,
'repo_id': 'repo_%s' % counter,
'distributor_id': 'distributor_%s' % counter,
}
coll.insert(bind_dict)
# Test
module = MigrationModule(
'pulp.server.db.migrations.0003_bind_additions')._module
module.migrate()
# Verify
bindings = coll.find()
for b in bindings:
self.assertTrue('notify_agent' in b)
self.assertEqual(b['notify_agent'], True)
self.assertTrue('binding_config' in b)
self.assertEqual(b['binding_config'], None)
def test_upgrade_idempotency(self):
"""
Simplest way to check the migration can run twice is simply to run it twice. The
primary goal is to make sure an exception isn't raised.
"""
# Setup
coll = Bind.get_collection()
for counter in range(0, 3):
bind_dict = {
'consumer_id' : 'consumer_%s' % counter,
'repo_id' : 'repo_%s' % counter,
'distributor_id' : 'distributor_%s' % counter,
}
coll.insert(bind_dict, safe=True)
# Test
module = MigrationModule('pulp.server.db.migrations.0003_bind_additions')._module
module.migrate()
module.migrate()
# Verify
bindings = coll.find()
for b in bindings:
self.assertTrue('notify_agent' in b)
self.assertEqual(b['notify_agent'], True)
self.assertTrue('binding_config' in b)
self.assertEqual(b['binding_config'], None)
def test__del_queue_catch_no_queue_in_use_exception_calls_add_queue(self):
fake_broker = Mock()
mock_name = Mock()
migration = MigrationModule(MIGRATION)._module
migration._del_queue_catch_queue_in_use_exception(
fake_broker, mock_name)
fake_broker.delQueue.assert_called_once_with(mock_name)
def test_migration(self):
# setup
collection = Bind.get_collection()
for n in range(0, MAX_BINDINGS):
if n % 2 == 0:
conf = {ID: n}
else:
conf = None
binding = {
ID: n,
CONSUMER_ID: n,
REPO_ID: n,
DISTRIBUTOR_ID: n,
BINDING_CONFIG: conf,
NOTIFY_AGENT: True,
}
collection.save(binding, safe=True)
# migrate
module = MigrationModule(MIGRATION)._module
module.migrate()
# verify
bindings = list(collection.find({}))
self.assertEqual(len(bindings), MAX_BINDINGS)
for binding in bindings:
conf = binding[BINDING_CONFIG]
bind_id = binding[ID]
if bind_id % 2 == 0:
# untouched
self.assertEqual(conf, {ID: bind_id})
else:
# fixed
self.assertEqual(conf, {})
def test_migrate(self, m_get_collection, now_utc_datetime):
"""
Test last_updated and last_override_config fields added.
"""
collection = Mock()
found = [
{LAST_SYNC: '2016-05-04T18:19:01Z', LAST_UPDATED: '2016-05-03T18:19:01Z'},
{LAST_SYNC: '2016-05-04T18:20:01Z'},
{},
]
collection.find.return_value = deepcopy(found)
m_get_collection.return_value = collection
# test
module = MigrationModule(MIGRATION)._module
module.migrate()
# validation
m_get_collection.assert_called_once_with('repo_importers')
collection.find.assert_called_once_with()
now_utc_datetime.assert_called_once_with()
self.assertTrue(LAST_UPDATED in dist for dist in collection.save.call_args_list)
self.assertTrue(LAST_OVERRIDE_CONFIG in dist for dist in collection.save.call_args_list)
self.assertEqual(
len(collection.save.call_args_list), 2)
def test_idempotence(self, mock_connection):
"""
Test the idempotence of the migration
"""
permissions_schema = [{
"resource": "/",
"id": "5356d55b37382030f4a80b5e",
"users": {
"admin": [0, 1, 2, 3, 4]
}
}]
new_schema = [{
"resource":
"/",
"id":
"5356d55b37382030f4a80b5e",
"users": [{
"username": "******",
"permissions": [0, 1, 2, 3, 4]
}]
}]
migration = MigrationModule(MIGRATION)._module
collection = mock_connection.get_collection.return_value
collection.find.return_value = permissions_schema
migration.migrate()
self.assertEquals(permissions_schema, new_schema)
migration.migrate()
self.assertEquals(permissions_schema, new_schema)
def test_migrate(self, m_get_collection, now_utc_datetime):
"""
Test last_updated field is set.
"""
collection = Mock()
found = [
{
LAST_UPDATED: '2016-05-03T18:19:01Z'
},
{
LAST_UPDATED: None
},
{},
]
collection.find.return_value = deepcopy(found)
m_get_collection.return_value = collection
# test
module = MigrationModule(MIGRATION)._module
module.migrate()
# validation
m_get_collection.assert_called_once_with('repo_distributors')
collection.find.assert_called_once_with()
self.assertEqual(now_utc_datetime.call_count, 2)
self.assertTrue(
dist.get(LAST_UPDATED) for dist in collection.save.call_args_list)
self.assertEqual(len(collection.save.call_args_list), 2)
def test_update_called(self, mock_get_collection):
module = MigrationModule('pulp.server.db.migrations.0002_rename_http_notifier')._module
module.migrate()
# make sure the correct mongo query is being passed down
mock_get_collection.return_value.update.assert_called_once_with(
{'notifier_type_id': 'rest-api'}, {'$set': {'notifier_type_id': 'http'}}
)
def test_time_to_utc_on_collection_skips_utc(self, mock_connection):
migration = MigrationModule(MIGRATION)._module
collection = mock_connection.get_collection.return_value
unit = {'bar': '2014-07-09T11:09:07Z'}
collection.find.return_value = [unit]
migration.update_time_to_utc_on_collection('foo', 'bar')
mock_connection.get_collection.assert_called_once_with('foo')
self.assertFalse(collection.save.called)
def test__del_queue_catch_no_queue_in_use_exception_calls_passed_through_other_exceptions(
self):
fake_broker = Mock()
mock_name = Mock()
fake_broker.delQueue.side_effect = IOError()
migration = MigrationModule(MIGRATION)._module
self.assertRaises(IOError,
migration._del_queue_catch_queue_in_use_exception,
fake_broker, mock_name)
def test_del_agent_queues_skips_existing_queues(self):
self.fake_broker.getQueue.side_effect = [None, None]
# test
migration = MigrationModule(MIGRATION)._module
migration._del_agent_queues(self.fake_broker)
self.fake_broker.getQueue.assert_has_calls([call('dog'), call('cat')])
self.assertTrue(not self.mock_del_queue_catch_queue_in_use_exc.called)
def test_migrate(self, mock_connection):
"""
Test the schema change happens like it should.
"""
role_schema = copy.deepcopy(CURRENT)
migration = MigrationModule(MIGRATION)._module
collection = mock_connection.get_collection.return_value
collection.find.return_value = role_schema
migration.migrate()
self.assertEquals(role_schema, TARGET)
def test_idempotence(self, mock_connection):
"""
Test the idempotence of the migration
"""
role_schema = copy.deepcopy(TARGET)
migration = MigrationModule(MIGRATION)._module
collection = mock_connection.get_collection.return_value
collection.find.return_value = role_schema
migration.migrate()
self.assertFalse(collection.save.called)
self.assertEquals(role_schema, TARGET)
def test_migrate_agent_queues(self, fake_add_agent_queues,
fake_del_agent_queues):
fake_broker = Mock()
# test
migration = MigrationModule(MIGRATION)._module
migration._migrate_agent_queues(fake_broker)
# validation
fake_add_agent_queues.assert_called_with(fake_broker)
fake_del_agent_queues.assert_called_with(fake_broker)
def test_migrate(self, mock_connection, mock_update):
"""
Verify that only known & valid collections are updated
"""
migration = MigrationModule(MIGRATION)._module
collection_list = ['repo_distributors']
mock_connection.get_database.return_value.collection_names.return_value = collection_list
migration.migrate()
mock_update.assert_called_once_with('repo_distributors', 'last_publish')
def test_time_to_utc_on_collection(self, mock_connection):
migration = MigrationModule(MIGRATION)._module
collection = mock_connection.get_collection.return_value
unit = {'bar': '2014-07-09T11:09:07-04:00'}
utc_value = '2014-07-09T15:09:07Z'
collection.find.return_value = [unit]
migration.update_time_to_utc_on_collection('foo', 'bar')
mock_connection.get_collection.assert_called_once_with('foo')
self.assertEquals(unit['bar'], utc_value)
collection.save.assert_called_once_with(unit, safe=True)
def test_del_agent_queues_deletes_all_existing_queues(self):
self.fake_broker.getQueue.side_effect = [Mock(), Mock()]
# test
migration = MigrationModule(MIGRATION)._module
migration._del_agent_queues(self.fake_broker)
expected_calls = [
call(self.fake_broker, 'dog'),
call(self.fake_broker, 'cat')
]
self.mock_del_queue_catch_queue_in_use_exc.assert_has_calls(
expected_calls)
def test_migrate_reply_queue_not_found_does_not_delete_or_add_reply_queue(
self):
fake_broker = Mock()
fake_broker.getQueue.return_value = None
# test
migration = MigrationModule(MIGRATION)._module
migration._migrate_reply_queue(fake_broker)
# validation
fake_broker.getQueue.assert_called_with(Services.REPLY_QUEUE)
self.assertTrue(not self.mock_del_queue_catch_queue_in_use_exc.called)
self.assertTrue(not fake_broker.addQueue.called)
def setUp(self):
# Remove dependencies on qpidtoollibs and qpid.messaging
self.migration = MigrationModule(MIGRATION)._module
if self.migration.QPIDTOOLLIBS_AVAILABLE:
self.broker = self.migration.BrokerAgent
if self.migration.QPID_MESSAGING_AVAILABLE:
self.connection = self.migration.Connection
self.qpidtoollibs = self.migration.QPIDTOOLLIBS_AVAILABLE
self.qpid_messaging = self.migration.QPID_MESSAGING_AVAILABLE
self.migration.QPIDTOOLLIBS_AVAILABLE = True
self.migration.QPID_MESSAGING_AVAILABLE = True
self.migration.BrokerAgent = Mock()
self.migration.Connection = Mock()
def test_migrate_not_qpid(self, fake_conf, fake_migrate_agent_queues,
fake_migrate_reply_queue, fake_connection,
fake_broker):
fake_conf.get.return_value = 'not-qpid'
# test
migration = MigrationModule(MIGRATION)._module
migration.migrate()
# validation
self.assertFalse(fake_connection.called)
self.assertFalse(fake_broker.called)
self.assertFalse(fake_migrate_reply_queue.called)
self.assertFalse(fake_migrate_agent_queues.called)
def test_migrate_reply_queue_not_exclusive_does_not_delete_or_add_reply_queue(
self):
fake_queue = Mock()
fake_queue.values = {'exclusive': False, 'arguments': {}}
fake_broker = Mock()
fake_broker.getQueue.return_value = fake_queue
# test
migration = MigrationModule(MIGRATION)._module
migration._migrate_reply_queue(fake_broker)
# validation
fake_broker.getQueue.assert_called_with(Services.REPLY_QUEUE)
self.assertTrue(not self.mock_del_queue_catch_queue_in_use_exc.called)
self.assertTrue(not fake_broker.addQueue.called)
def test(self):
# migrate
module = MigrationModule(MIGRATION)._module
module.migrate()
# validation
for collection in [
connection.get_collection(n) for n in TEST_COLLECTIONS
]:
for unit in collection.find({}):
self.assertTrue(LAST_UPDATED in unit)
unit_id = unit[ID]
last_updated = unit[LAST_UPDATED]
if unit_id % 2 == 0:
self.assertEqual(last_updated, 1)
else:
self.assertTrue(isinstance(last_updated, float))
def test_migrate(self, m_get_collection):
"""
Test last_updated and last_override_config fields added.
"""
collection = Mock()
m_get_collection.return_value = collection
# test
module = MigrationModule(MIGRATION)._module
module.migrate()
# validation
m_get_collection.assert_called_once_with('repo_importers')
# can't do much more than see that update was called
# for each key to be removed (2 total calls)
self.assertEqual(len(collection.update.call_args_list), 2)
def test_migrate_reply_queue_checks_deleted_and_recreates_reply_queue(
self):
fake_queue = Mock()
fake_queue.values = {'exclusive': True, 'arguments': {}}
fake_broker = Mock()
fake_broker.getQueue.return_value = fake_queue
# test
migration = MigrationModule(MIGRATION)._module
migration._migrate_reply_queue(fake_broker)
# validation
fake_broker.getQueue.assert_called_once_with(Services.REPLY_QUEUE)
expected_call = call(fake_broker, Services.REPLY_QUEUE)
self.mock_del_queue_catch_queue_in_use_exc.assert_has_calls(
expected_call)
fake_broker.addQueue.assert_called_once_with(Services.REPLY_QUEUE,
durable=True)
def test_database_integration(self):
# make sure the migration works on a live document in mongo
collection = EventListener.get_collection()
event_listener_id = str(collection.insert({
'notifier_type_id': 'rest-api',
'event_types': ['*'],
'notifier_config': {},
}))
event_listener_factory = managers.factory.event_listener_manager()
module = MigrationModule('pulp.server.db.migrations.0002_rename_http_notifier')._module
module.migrate()
event_listener = event_listener_factory.get(event_listener_id)
self.assertEqual(event_listener['notifier_type_id'], 'http')
# cleanup
collection.remove()
def test__del_queue_catch_no_queue_in_use_exception_catches_cannot_delete_queue(
self):
fake_broker = Mock()
mock_name = Mock()
exc_to_raise = Exception("Cannot delete queue celery; queue in use")
fake_broker.delQueue.side_effect = exc_to_raise
migration = MigrationModule(MIGRATION)._module
try:
migration._del_queue_catch_queue_in_use_exception(
fake_broker, mock_name)
self.fail('An exception should have been raised, and was not.')
except Exception as error:
string_a = 'Consumers are still bound to the queue'
string_b = 'All consumers must be unregistered, upgraded, or off before you can continue'
if string_a not in error.message or string_b not in error.message:
self.fail(
"Migration 0009 does not handle a 'queue in use' exception"
)
def test_add_agent_queues(self, fake_get):
fake_collection = Mock()
fake_collection.find = Mock(return_value=[{
'id': 'dog'
}, {
'id': 'cat'
}])
fake_get.return_value = fake_collection
fake_broker = Mock()
fake_broker.getQueue.side_effect = [None, None]
# test
migration = MigrationModule(MIGRATION)._module
migration._add_agent_queues(fake_broker)
fake_broker.getQueue.assert_any('pulp.agent.dog')
fake_broker.getQueue.assert_any('pulp.agent.cat')
fake_broker.addQueue.assert_any('pulp.agent.dog')
fake_broker.addQueue.assert_any('pulp.agent.cat')
def setUp(self):
super(TestMigrationAddMetadata, self).setUp()
self.module = MigrationModule(
'pulp_rpm.plugins.migrations.0005_rpm_changelog_files')._module
self.fake_unit = {'_storage_path': '/tmp/foo'}
self.mock_ts = mock.MagicMock()
self.mock_collection = mock.MagicMock()
self.mock_package = mock.MagicMock()
self.mock_package.changelog = [
('now', '*****@*****.**', 'description'),
]
self.mock_package.filelist = ['/a', '/b', '/c']
self.mock_package.files = {
'ghost': ['/xyz'],
'dir': ['/foo', '/bar'],
'file': ['/a', '/b', '/c'],
}
def test_migrate(self, fake_migrate_agent_queues, fake_migrate_reply_queue,
fake_connection, fake_broker):
# test
migration = MigrationModule(MIGRATION)._module
migration.migrate()
# validation
fake_connection.assert_called_with(host='myhost',
port=1234,
transport='tcp',
reconnect=False,
ssl_certfile='TEST-CERTIFICATE',
ssl_skip_hostname_check=True)
fake_connection().attach.assert_called_with()
fake_broker.assert_called_with(fake_connection())
fake_migrate_reply_queue.assert_called_with(fake_broker())
fake_migrate_agent_queues.assert_called_with(fake_broker())
fake_connection().detach.assert_called_with()
def test_migrate(self, m_get_collection, parse_iso8601_datetime):
collection = Mock()
found = [
{
LAST_PUBLISH: '2015-04-28T18:19:01Z'
},
{
LAST_PUBLISH: datetime.now()
},
{
LAST_PUBLISH: '2015-04-28T18:20:01Z'
},
{
LAST_PUBLISH: datetime.now()
},
]
parsed = [1, 2]
collection.find.return_value = deepcopy(found)
m_get_collection.return_value = collection
parse_iso8601_datetime.side_effect = parsed
# test
module = MigrationModule(MIGRATION)._module
module.migrate()
# validation
m_get_collection.assert_called_once_with('repo_distributors')
collection.find.assert_called_once_with()
self.assertEqual(parse_iso8601_datetime.call_args_list, [
call(found[0][LAST_PUBLISH]),
call(found[2][LAST_PUBLISH]),
])
self.assertEqual(
collection.save.call_args_list,
[call({LAST_PUBLISH: parsed[0]}),
call({LAST_PUBLISH: parsed[1]})])
