def test_using_admin_party_source_and_target(self):
m_admin_party_client = self.setUpClientMocks(admin_party=True)
m_replicator = mock.MagicMock()
type(m_replicator).creds = mock.PropertyMock(return_value=None)
m_admin_party_client.__getitem__.return_value = m_replicator
# create source/target databases
src = CouchDatabase(m_admin_party_client, self.source_db)
tgt = CouchDatabase(m_admin_party_client, self.target_db)
# trigger replication
rep = Replicator(m_admin_party_client)
rep.create_replication(src, tgt, repl_id=self.repl_id)
kcall = m_replicator.create_document.call_args_list
self.assertEquals(len(kcall), 1)
args, kwargs = kcall[0]
self.assertEquals(len(args), 1)
expected_doc = {
'_id': self.repl_id,
'source': {
'url': '/'.join((self.server_url, self.source_db))
},
'target': {
'url': '/'.join((self.server_url, self.target_db))
}
}
self.assertDictEqual(args[0], expected_doc)
self.assertTrue(kwargs['throw_on_exists'])
def test_create_replication(self):
"""test create_replication method"""
with mock.patch('cloudant.database.CouchDatabase.create_document') as mock_create:
mock_target = mock.Mock()
mock_target.database_url = "http://bob.cloudant.com/target"
mock_target.creds = {'basic_auth': "target_auth"}
mock_source = mock.Mock()
mock_source.database_url = "http://bob.cloudant.com/source"
mock_source.creds = {'basic_auth': "source_auth"}
repl = Replicator(self.mock_account)
repl.create_replication(mock_source, mock_target, "REPLID")
self.assertTrue(mock_create.called)
repl_doc = mock_create.call_args[0][0]
self.assertTrue('source' in repl_doc)
self.assertTrue('target' in repl_doc)
self.assertEqual(repl_doc['_id'], 'REPLID')
self.assertEqual(
repl_doc['source']['url'],
'http://bob.cloudant.com/source'
)
self.assertEqual(
repl_doc['target']['url'],
'http://bob.cloudant.com/target'
)
self.assertEqual(
repl_doc['target']['headers']['Authorization'],
'target_auth'
)
self.assertEqual(
repl_doc['source']['headers']['Authorization'],
'source_auth'
)
def test_replication_with_generated_id(self):
clone = Replicator(self.client)
repl_id = clone.create_replication(
self.db,
self.target_db
)
self.replication_ids.append(repl_id['_id'])
def test_replication_state(self):
"""
_test_replication_state_
Verify that we can get the replication state.
"""
dbsource = u"test_replication_state_source_{}".format(
unicode(uuid.uuid4()))
dbtarget = u"test_replication_state_target_{}".format(
unicode(uuid.uuid4()))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document(
{"_id": "doc1", "testing": "document 1"}
)
doc2 = dbs.create_document(
{"_id": "doc2", "testing": "document 1"}
)
doc3 = dbs.create_document(
{"_id": "doc3", "testing": "document 1"}
)
replicator = Replicator(c)
repl_id = u"test_replication_state_{}".format(
unicode(uuid.uuid4()))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
replication_state = "not_yet_set"
while True:
# Verify that replication_state returns either None
# (if the field doesn't exist yet), or a valid
# replication state.
replication_state = replicator.replication_state(repl_id)
if replication_state is not None:
self.assertTrue(
replication_state in [
'completed',
'error',
'triggered'
]
)
if replication_state in ('error', 'completed'):
break
LOG.debug("got replication state: {}".format(
replication_state))
time.sleep(1)
def test_using_iam_auth_source_and_target(self):
m_iam_auth_client = self.setUpClientMocks(iam_api_key=MOCK_API_KEY)
m_replicator = mock.MagicMock()
m_iam_auth_client.__getitem__.return_value = m_replicator
# create source/target databases
src = CouchDatabase(m_iam_auth_client, self.source_db)
tgt = CouchDatabase(m_iam_auth_client, self.target_db)
# trigger replication
rep = Replicator(m_iam_auth_client)
rep.create_replication(src,
tgt,
repl_id=self.repl_id,
user_ctx=self.user_ctx)
kcall = m_replicator.create_document.call_args_list
self.assertEquals(len(kcall), 1)
args, kwargs = kcall[0]
self.assertEquals(len(args), 1)
expected_doc = {
'_id': self.repl_id,
'user_ctx': self.user_ctx,
'source': {
'auth': {
'iam': {
'api_key': MOCK_API_KEY
}
},
'url': '/'.join((self.server_url, self.source_db))
},
'target': {
'auth': {
'iam': {
'api_key': MOCK_API_KEY
}
},
'url': '/'.join((self.server_url, self.target_db))
}
}
self.assertDictEqual(args[0], expected_doc)
self.assertTrue(kwargs['throw_on_exists'])
def test_using_basic_auth_source_and_target(self):
test_basic_auth_header = 'abc'
m_basic_auth_client = self.setUpClientMocks()
m_replicator = mock.MagicMock()
m_basic_auth_client.__getitem__.return_value = m_replicator
m_basic_auth_client.basic_auth_str.return_value = test_basic_auth_header
# create source/target databases
src = CouchDatabase(m_basic_auth_client, self.source_db)
tgt = CouchDatabase(m_basic_auth_client, self.target_db)
# trigger replication
rep = Replicator(m_basic_auth_client)
rep.create_replication(src,
tgt,
repl_id=self.repl_id,
user_ctx=self.user_ctx)
kcall = m_replicator.create_document.call_args_list
self.assertEquals(len(kcall), 1)
args, kwargs = kcall[0]
self.assertEquals(len(args), 1)
expected_doc = {
'_id': self.repl_id,
'user_ctx': self.user_ctx,
'source': {
'headers': {
'Authorization': test_basic_auth_header
},
'url': '/'.join((self.server_url, self.source_db))
},
'target': {
'headers': {
'Authorization': test_basic_auth_header
},
'url': '/'.join((self.server_url, self.target_db))
}
}
self.assertDictEqual(args[0], expected_doc)
self.assertTrue(kwargs['throw_on_exists'])
def test_replication_state(self):
"""
_test_replication_state_
Verify that we can get the replication state.
"""
dbsource = unicode_("test_replication_state_source_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_replication_state_target_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_replication_state_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
replication_state = "not_yet_set"
while True:
# Verify that replication_state returns either None
# (if the field doesn't exist yet), or a valid
# replication state.
replication_state = replicator.replication_state(repl_id)
if replication_state is not None:
self.assertTrue(replication_state in
['completed', 'error', 'triggered'])
if replication_state in ('error', 'completed'):
break
LOG.debug(
"got replication state: {}".format(replication_state))
time.sleep(1)
def test_follow_replication_with_errors(self):
"""
_test_follow_replication_with_errors_
Test to make sure that we exit the follow loop when we submit
a bad replication.
"""
dbsource = unicode_("test_follow_replication_source_error_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_follow_replication_target_error_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_follow_replication_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
# Deliberately override these good params with bad params
source=dbsource + "foo",
target=dbtarget + "foo",
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'error')
def test_follow_replication_with_errors(self):
"""
_test_follow_replication_with_errors_
Test to make sure that we exit the follow loop when we submit
a bad replication.
"""
dbsource = u"test_follow_replication_source_error_{}".format(
unicode(uuid.uuid4()))
dbtarget = u"test_follow_replication_target_error_{}".format(
unicode(uuid.uuid4()))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document(
{"_id": "doc1", "testing": "document 1"}
)
doc2 = dbs.create_document(
{"_id": "doc2", "testing": "document 1"}
)
doc3 = dbs.create_document(
{"_id": "doc3", "testing": "document 1"}
)
replicator = Replicator(c)
repl_id = u"test_follow_replication_{}".format(
unicode(uuid.uuid4()))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
# Deliberately override these good params with bad params
source=dbsource + "foo",
target=dbtarget + "foo",
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'error')
def setup_replication(self,
source_db_name,
target,
repl_id=None,
use_log=None,
**kwargs):
"""
Set up replication of a database.
Args:
source_db_name: Source database
target: Target database URL or :py:class:`djali.couchdb.CloudiControl` instance
repl_id (str, optional): replication document ID
use_log (logging.Logger): Override logger instance
Keyword Args:
continuous(bool): Continuous replication (defaults to ``False``)
Returns:
cloudant.document.Document: replication document
"""
if use_log is None:
use_log = self.log
client = CouchDB(self._root_auth[0],
self._root_auth[1],
url=self.instance_url,
connect=True,
auto_renew=True)
replicator_obj = Replicator(client)
try:
target_database = target.database
except Exception:
target = CloudiControl(target, create=True)
target_database = target.database
source_database = client[source_db_name]
use_log.info("Setting up replication: {!s} -> {!s}".format(
source_database, target_database))
return replicator_obj.create_replication(source_database,
target_database,
repl_id=repl_id,
create_target=True,
continuous=kwargs.get(
"continuous", False))
def test_follow_replication(self):
"""
_test_follow_replication_
Test to make sure that we can follow a replication.
"""
dbsource = unicode_("test_follow_replication_source_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_follow_replication_target_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_follow_replication_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'completed')
def test_list_replications(self):
"""
_test_list_replications_
Verify that we get a list of replications documents back when
we got to list replications.
"""
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
repl_ids = []
num_reps = 3
for i in range(0, num_reps):
tag = "{0}_{1}".format(i, unicode(uuid.uuid4()))
dbsource = u"test_list_repl_src_{}".format(tag)
dbtarget = u"test_list_repl_tgt_{}".format(tag)
self.dbs.append(dbsource)
self.dbs.append(dbtarget)
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document(
{"_id": "doc1", "testing": "document 1"}
)
repl_id = u"test_create_replication_{}".format(tag)
self.replication_ids.append(repl_id)
repl_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False
)
replications = replicator.list_replications()
ids = [doc['_id'] for doc in replications]
found_ids = [i for i in ids if i in repl_ids]
self.assertEqual(num_reps, len(found_ids))
def test_list_replications(self):
"""
_test_list_replications_
Verify that we get a list of replications documents back when
we got to list replications.
"""
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
repl_ids = []
num_reps = 3
for i in range(0, num_reps):
tag = "{0}_{1}".format(i, unicode_(uuid.uuid4()))
dbsource = unicode_("test_list_repl_src_{}".format(tag))
dbtarget = unicode_("test_list_repl_tgt_{}".format(tag))
self.dbs.append(dbsource)
self.dbs.append(dbtarget)
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
repl_id = unicode_("test_create_replication_{}".format(tag))
self.replication_ids.append(repl_id)
repl_ids.append(repl_id)
ret = replicator.create_replication(source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False)
replications = replicator.list_replications()
ids = [doc['_id'] for doc in replications]
found_ids = [i for i in ids if i in repl_ids]
self.assertEqual(num_reps, len(found_ids))
def test_follow_replication(self):
"""
_test_follow_replication_
Test to make sure that we can follow a replication.
"""
dbsource = u"test_follow_replication_source_{}".format(
unicode(uuid.uuid4()))
dbtarget = u"test_follow_replication_target_{}".format(
unicode(uuid.uuid4()))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document(
{"_id": "doc1", "testing": "document 1"}
)
doc2 = dbs.create_document(
{"_id": "doc2", "testing": "document 1"}
)
doc3 = dbs.create_document(
{"_id": "doc3", "testing": "document 1"}
)
replicator = Replicator(c)
repl_id = u"test_follow_replication_{}".format(
unicode(uuid.uuid4()))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'completed')
class ReplicatorTests(UnitTestDbBase):
"""
Replicator unit tests
"""
def setUp(self):
"""
Set up test attributes
"""
super(ReplicatorTests, self).setUp()
self.db_set_up()
self.test_target_dbname = self.dbname()
self.target_db = self.client._DATABASE_CLASS(
self.client,
self.test_target_dbname
)
self.target_db.create()
self.replicator = Replicator(self.client)
self.replication_ids = []
def tearDown(self):
"""
Reset test attributes
"""
self.target_db.delete()
del self.test_target_dbname
del self.target_db
for rep_id in self.replication_ids:
max_retry = 5
while True:
try:
self.replicator.stop_replication(rep_id)
break
except requests.HTTPError as ex:
# Retry failed attempt to delete replication document. It's
# likely in an error state and receiving constant updates
# via the replicator.
max_retry -= 1
if ex.response.status_code != 409 or max_retry == 0:
raise
del self.replicator
self.db_tear_down()
super(ReplicatorTests, self).tearDown()
def test_constructor(self):
"""
Test constructing a Replicator
"""
self.assertIsInstance(self.replicator, Replicator)
self.assertIsInstance(
self.replicator.database,
self.client._DATABASE_CLASS
)
self.assertEqual(self.replicator.database, self.client['_replicator'])
def test_constructor_failure(self):
"""
Test that constructing a Replicator will not work
without a valid client.
"""
repl = None
try:
self.client.disconnect()
repl = Replicator(self.client)
self.fail('Above statement should raise a CloudantException')
except CloudantClientException as err:
self.assertEqual(
str(err),
'Database _replicator does not exist. '
'Verify that the client is valid and try again.'
)
finally:
self.assertIsNone(repl)
self.client.connect()
def test_replication_with_generated_id(self):
clone = Replicator(self.client)
repl_id = clone.create_replication(
self.db,
self.target_db
)
self.replication_ids.append(repl_id['_id'])
@flaky(max_runs=3)
def test_create_replication(self):
"""
Test that the replication document gets created and that the
replication is successful.
"""
self.populate_db_with_documents(3)
repl_id = 'test-repl-{}'.format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replication_ids.append(repl_id)
# Test that the replication document was created
expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx']
# If Admin Party mode then user_ctx will not be in the key list
if self.client.admin_party or self.client.is_iam_authenticated:
expected_keys.pop()
self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys))
self.assertEqual(repl_doc['_id'], repl_id)
self.assertTrue(repl_doc['_rev'].startswith('1-'))
# Now that we know that the replication document was created,
# check that the replication occurred.
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') not in ('completed', 'error'):
changes = self.replicator.database.changes(
feed='continuous',
heartbeat=1000)
beats = 0
for change in changes:
if beats == 300:
changes.stop()
if not change:
beats += 1
continue
elif change.get('id') == repl_id:
beats = 0
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') in ('completed', 'error'):
changes.stop()
self.assertEqual(repl_doc.get('_replication_state'), 'completed')
self.assertEqual(self.db.all_docs(), self.target_db.all_docs())
self.assertTrue(
all(x in self.target_db.keys(True) for x in [
'julia000',
'julia001',
'julia002'
])
)
def test_timeout_in_create_replication(self):
"""
Test that a read timeout exception is thrown when creating a
replicator with a timeout value of 500 ms.
"""
# Setup client with a timeout
self.set_up_client(auto_connect=True, timeout=.5)
self.db = self.client[self.test_target_dbname]
self.target_db = self.client[self.test_dbname]
# Construct a replicator with the updated client
self.replicator = Replicator(self.client)
repl_id = 'test-repl-{}'.format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replication_ids.append(repl_id)
# Test that the replication document was created
expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx']
# If Admin Party mode then user_ctx will not be in the key list
if self.client.admin_party or self.client.is_iam_authenticated:
expected_keys.pop()
self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys))
self.assertEqual(repl_doc['_id'], repl_id)
self.assertTrue(repl_doc['_rev'].startswith('1-'))
# Now that we know that the replication document was created,
# check that the replication timed out.
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') not in ('completed', 'error'):
# assert that a connection error is thrown because the read timed out
with self.assertRaises(ConnectionError) as cm:
changes = self.replicator.database.changes(
feed='continuous')
for change in changes:
continue
self.assertTrue(str(cm.exception).endswith('Read timed out.'))
def test_create_replication_without_a_source(self):
"""
Test that the replication document is not created and fails as expected
when no source database is provided.
"""
try:
repl_doc = self.replicator.create_replication()
self.fail('Above statement should raise a CloudantException')
except CloudantReplicatorException as err:
self.assertEqual(
str(err),
'You must specify either a source_db Database '
'object or a manually composed \'source\' string/dict.'
)
def test_create_replication_without_a_target(self):
"""
Test that the replication document is not created and fails as expected
when no target database is provided.
"""
try:
repl_doc = self.replicator.create_replication(self.db)
self.fail('Above statement should raise a CloudantException')
except CloudantReplicatorException as err:
self.assertEqual(
str(err),
'You must specify either a target_db Database '
'object or a manually composed \'target\' string/dict.'
)
def test_list_replications(self):
"""
Test that a list of Document wrapped objects are returned.
"""
self.populate_db_with_documents(3)
repl_ids = ['test-repl-{}'.format(
unicode_(uuid.uuid4())
) for _ in range(3)]
repl_docs = [self.replicator.create_replication(
self.db,
self.target_db,
repl_id
) for repl_id in repl_ids]
self.replication_ids.extend(repl_ids)
replications = self.replicator.list_replications()
all_repl_ids = [doc['_id'] for doc in replications]
match = [repl_id for repl_id in all_repl_ids if repl_id in repl_ids]
self.assertEqual(set(repl_ids), set(match))
def test_retrieve_replication_state(self):
"""
Test that the replication state can be retrieved for a replication
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replication_ids.append(repl_id)
repl_state = None
valid_states = ['completed', 'error', 'triggered', 'running', None]
finished = False
for _ in range(300):
repl_state = self.replicator.replication_state(repl_id)
self.assertTrue(repl_state in valid_states)
if repl_state in ('error', 'completed'):
finished = True
break
time.sleep(1)
self.assertTrue(finished)
def test_retrieve_replication_state_using_invalid_id(self):
"""
Test that replication_state(...) raises an exception as expected
when an invalid replication id is provided.
"""
repl_id = 'fake-repl-id-{}'.format(unicode_(uuid.uuid4()))
repl_state = None
try:
self.replicator.replication_state(repl_id)
self.fail('Above statement should raise a CloudantException')
except CloudantReplicatorException as err:
self.assertEqual(
str(err),
'Replication with id {} not found.'.format(repl_id)
)
self.assertIsNone(repl_state)
def test_stop_replication(self):
"""
Test that a replication can be stopped.
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
max_retry = 3
while True:
try:
max_retry -= 1
self.replicator.stop_replication(repl_id)
break
except requests.HTTPError as err:
self.assertEqual(err.response.status_code, 409)
if max_retry == 0:
self.fail('Failed to stop replication: {0}'.format(err))
try:
# The .fetch() will fail since the replication has been stopped
# and the replication document has been removed from the db.
repl_doc.fetch()
self.fail('Above statement should raise a CloudantException')
except requests.HTTPError as err:
self.assertEqual(err.response.status_code, 404)
def test_stop_replication_using_invalid_id(self):
"""
Test that stop_replication(...) raises an exception as expected
when an invalid replication id is provided.
"""
repl_id = 'fake-repl-id-{}'.format(unicode_(uuid.uuid4()))
try:
self.replicator.stop_replication(repl_id)
self.fail('Above statement should raise a CloudantException')
except CloudantReplicatorException as err:
self.assertEqual(
str(err),
'Replication with id {} not found.'.format(repl_id)
)
def test_follow_replication(self):
"""
Test that follow_replication(...) properly iterates updated
replication documents while the replication is executing.
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replication_ids.append(repl_id)
valid_states = ('completed', 'error', 'triggered', 'running', None)
repl_states = []
if 'scheduler' in self.client.features():
state_key = 'state'
else:
state_key = '_replication_state'
for doc in self.replicator.follow_replication(repl_id):
self.assertIn(doc.get(state_key), valid_states)
repl_states.append(doc.get(state_key))
self.assertTrue(len(repl_states) > 0)
self.assertEqual(repl_states[-1], 'completed')
self.assertNotIn('error', repl_states)
def test_replication_with_generated_id(self):
clone = Replicator(self.client)
clone.create_replication(self.db, self.target_db)
def test_create_replication(self):
"""
_test_create_replication_
Make a couple of test databases, and confirm that docs from
one get transferred to t'other.
"""
dbsource = u"test_create_replication_source_{}".format(
unicode(uuid.uuid4()))
dbtarget = u"test_create_replication_target_{}".format(
unicode(uuid.uuid4()))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document(
{"_id": "doc1", "testing": "document 1"}
)
doc2 = dbs.create_document(
{"_id": "doc2", "testing": "document 1"}
)
doc3 = dbs.create_document(
{"_id": "doc3", "testing": "document 1"}
)
replicator = Replicator(c)
repl_id = u"test_create_replication_{}".format(
unicode(uuid.uuid4()))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False
)
try:
repl_doc = replicator[repl_id]
except KeyError:
repl_doc = None
if not repl_doc or not (repl_doc.get(
'_replication_state', "none") in ('completed', 'error')):
for change in replicator.changes():
if change.get('id') == repl_id:
try:
repl_doc = replicator[repl_id]
repl_doc.fetch()
except KeyError:
pass
if repl_doc and (repl_doc.get(
'_replication_state',
"none") in ('completed', 'error')):
break
else:
LOG.debug(
u"Waiting for replication to complete "
u"(repl_doc: {})".format(repl_doc)
)
self.assertTrue(repl_doc)
self.assertEqual(repl_doc.get('_replication_state'), 'completed')
for d in ['doc1', 'doc2', 'doc3']:
self.assertTrue(dbt[d])
self.assertEqual(dbt[d]['testing'], dbs[d]['testing'])
def test_create_replication(self):
"""
_test_create_replication_
Make a couple of test databases, and confirm that docs from
one get transferred to t'other.
"""
dbsource = unicode_("test_create_replication_source_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_create_replication_target_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_create_replication_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False)
try:
repl_doc = replicator[repl_id]
except KeyError:
repl_doc = None
if not repl_doc or not (repl_doc.get('_replication_state', "none")
in ('completed', 'error')):
for change in replicator.changes():
if change.get('id') == repl_id:
try:
repl_doc = replicator[repl_id]
repl_doc.fetch()
except KeyError:
pass
if repl_doc and (repl_doc.get('_replication_state',
"none")
in ('completed', 'error')):
break
else:
LOG.debug(
unicode_("Waiting for replication to complete "
"(repl_doc: {})".format(repl_doc)))
self.assertTrue(repl_doc)
self.assertEqual(repl_doc.get('_replication_state'), 'completed')
for d in ['doc1', 'doc2', 'doc3']:
self.assertTrue(dbt[d])
self.assertEqual(dbt[d]['testing'], dbs[d]['testing'])
def test_replication_with_generated_id(self):
clone = Replicator(self.client)
clone.create_replication(self.db, self.target_db)
class ReplicatorTests(UnitTestDbBase):
"""
Replicator unit tests
"""
def setUp(self):
"""
Set up test attributes
"""
super(ReplicatorTests, self).setUp()
self.db_set_up()
self.test_target_dbname = self.dbname()
self.target_db = self.client._DATABASE_CLASS(self.client,
self.test_target_dbname)
self.target_db.create()
self.replicator = Replicator(self.client)
self.replication_ids = []
def tearDown(self):
"""
Reset test attributes
"""
self.target_db.delete()
del self.test_target_dbname
del self.target_db
while self.replication_ids:
self.replicator.stop_replication(self.replication_ids.pop())
del self.replicator
self.db_tear_down()
super(ReplicatorTests, self).tearDown()
def test_constructor(self):
"""
Test constructing a Replicator
"""
self.assertIsInstance(self.replicator, Replicator)
self.assertIsInstance(self.replicator.database,
self.client._DATABASE_CLASS)
self.assertEqual(self.replicator.database, self.client['_replicator'])
def test_constructor_failure(self):
"""
Test that constructing a Replicator will not work
without a valid client.
"""
repl = None
try:
self.client.disconnect()
repl = Replicator(self.client)
self.fail('Above statement should raise a CloudantException')
except CloudantException as err:
self.assertEqual(
str(err), 'Unable to acquire _replicator database. '
'Verify that the client is valid and try again.')
finally:
self.assertIsNone(repl)
self.client.connect()
def test_create_replication(self):
"""
Test that the replication document gets created and that the
replication is successful.
"""
self.populate_db_with_documents(3)
repl_id = 'test-repl-{}'.format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(self.db, self.target_db,
repl_id)
self.replication_ids.append(repl_id)
# Test that the replication document was created
expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx']
# If Admin Party mode then user_ctx will not be in the key list
if self.client.admin_party:
expected_keys.pop()
self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys))
self.assertEqual(repl_doc['_id'], repl_id)
self.assertTrue(repl_doc['_rev'].startswith('1-'))
# Now that we know that the replication document was created,
# check that the replication occurred.
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') not in ('completed', 'error'):
changes = self.replicator.database.changes(feed='continuous',
heartbeat=1000)
beats = 0
for change in changes:
if beats == 300:
changes.stop()
if not change:
beats += 1
continue
elif change.get('id') == repl_id:
beats = 0
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') in ('completed',
'error'):
changes.stop()
self.assertEqual(repl_doc.get('_replication_state'), 'completed')
self.assertEqual(self.db.all_docs(), self.target_db.all_docs())
self.assertTrue(
all(x in self.target_db.keys(True)
for x in ['julia000', 'julia001', 'julia002']))
def test_create_replication_without_a_source(self):
"""
Test that the replication document is not created and fails as expected
when no source database is provided.
"""
try:
repl_doc = self.replicator.create_replication()
self.fail('Above statement should raise a CloudantException')
except CloudantException as err:
self.assertEqual(
str(err), 'You must specify either a source_db Database '
'object or a manually composed \'source\' string/dict.')
def test_create_replication_without_a_target(self):
"""
Test that the replication document is not created and fails as expected
when no target database is provided.
"""
try:
repl_doc = self.replicator.create_replication(self.db)
self.fail('Above statement should raise a CloudantException')
except CloudantException as err:
self.assertEqual(
str(err), 'You must specify either a target_db Database '
'object or a manually composed \'target\' string/dict.')
def test_list_replications(self):
"""
Test that a list of Document wrapped objects are returned.
"""
self.populate_db_with_documents(3)
repl_ids = [
'test-repl-{}'.format(unicode_(uuid.uuid4())) for _ in range(3)
]
repl_docs = [
self.replicator.create_replication(self.db, self.target_db,
repl_id) for repl_id in repl_ids
]
self.replication_ids.extend(repl_ids)
replications = self.replicator.list_replications()
all_repl_ids = [doc['_id'] for doc in replications]
match = [repl_id for repl_id in all_repl_ids if repl_id in repl_ids]
self.assertEqual(set(repl_ids), set(match))
def test_retrieve_replication_state(self):
"""
Test that the replication state can be retrieved for a replication
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(self.db, self.target_db,
repl_id)
self.replication_ids.append(repl_id)
repl_state = None
valid_states = ['completed', 'error', 'triggered', None]
finished = False
for _ in range(300):
repl_state = self.replicator.replication_state(repl_id)
self.assertTrue(repl_state in valid_states)
if repl_state in ('error', 'completed'):
finished = True
break
time.sleep(1)
self.assertTrue(finished)
def test_retrieve_replication_state_using_invalid_id(self):
"""
Test that replication_state(...) raises an exception as expected
when an invalid replication id is provided.
"""
repl_id = 'fake-repl-id-{}'.format(unicode_(uuid.uuid4()))
repl_state = None
try:
self.replicator.replication_state(repl_id)
self.fail('Above statement should raise a CloudantException')
except CloudantException as err:
self.assertEqual(str(err),
'Replication {} not found'.format(repl_id))
self.assertIsNone(repl_state)
def test_stop_replication(self):
"""
Test that a replication can be stopped.
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(self.db, self.target_db,
repl_id)
self.replicator.stop_replication(repl_id)
try:
# The .fetch() will fail since the replication has been stopped
# and the replication document has been removed from the db.
repl_doc.fetch()
self.fail('Above statement should raise a CloudantException')
except requests.HTTPError as err:
self.assertEqual(err.response.status_code, 404)
def test_stop_replication_using_invalid_id(self):
"""
Test that stop_replication(...) raises an exception as expected
when an invalid replication id is provided.
"""
repl_id = 'fake-repl-id-{}'.format(unicode_(uuid.uuid4()))
try:
self.replicator.stop_replication(repl_id)
self.fail('Above statement should raise a CloudantException')
except CloudantException as err:
self.assertEqual(
str(err),
'Could not find replication with id {}'.format(repl_id))
def test_follow_replication(self):
"""
Test that follow_replication(...) properly iterates updated
replication documents while the replication is executing.
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(self.db, self.target_db,
repl_id)
self.replication_ids.append(repl_id)
valid_states = ('completed', 'error', 'triggered', None)
repl_states = []
for doc in self.replicator.follow_replication(repl_id):
self.assertIn(doc.get('_replication_state'), valid_states)
repl_states.append(doc.get('_replication_state'))
self.assertTrue(len(repl_states) > 0)
self.assertEqual(repl_states[-1], 'completed')
self.assertNotIn('error', repl_states)
class ReplicatorTests(UnitTestDbBase):
"""
Replicator unit tests
"""
def setUp(self):
"""
Set up test attributes
"""
super(ReplicatorTests, self).setUp()
self.db_set_up()
self.test_target_dbname = self.dbname()
self.target_db = self.client._DATABASE_CLASS(
self.client,
self.test_target_dbname
)
self.target_db.create()
self.replicator = Replicator(self.client)
self.replication_ids = []
def tearDown(self):
"""
Reset test attributes
"""
self.target_db.delete()
del self.test_target_dbname
del self.target_db
for rep_id in self.replication_ids:
max_retry = 5
while True:
try:
self.replicator.stop_replication(rep_id)
break
except requests.HTTPError as ex:
# Retry failed attempt to delete replication document. It's
# likely in an error state and receiving constant updates
# via the replicator.
max_retry -= 1
if ex.response.status_code != 409 or max_retry == 0:
raise
del self.replicator
self.db_tear_down()
super(ReplicatorTests, self).tearDown()
def test_constructor(self):
"""
Test constructing a Replicator
"""
self.assertIsInstance(self.replicator, Replicator)
self.assertIsInstance(
self.replicator.database,
self.client._DATABASE_CLASS
)
self.assertEqual(self.replicator.database, self.client['_replicator'])
def test_constructor_failure(self):
"""
Test that constructing a Replicator will not work
without a valid client.
"""
repl = None
try:
self.client.disconnect()
repl = Replicator(self.client)
self.fail('Above statement should raise a CloudantException')
except CloudantClientException as err:
self.assertEqual(
str(err),
'Database _replicator does not exist. '
'Verify that the client is valid and try again.'
)
finally:
self.assertIsNone(repl)
self.client.connect()
def test_replication_with_generated_id(self):
clone = Replicator(self.client)
clone.create_replication(self.db, self.target_db)
@flaky(max_runs=3)
def test_create_replication(self):
"""
Test that the replication document gets created and that the
replication is successful.
"""
self.populate_db_with_documents(3)
repl_id = 'test-repl-{}'.format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replication_ids.append(repl_id)
# Test that the replication document was created
expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx']
# If Admin Party mode then user_ctx will not be in the key list
if self.client.admin_party:
expected_keys.pop()
self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys))
self.assertEqual(repl_doc['_id'], repl_id)
self.assertTrue(repl_doc['_rev'].startswith('1-'))
# Now that we know that the replication document was created,
# check that the replication occurred.
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') not in ('completed', 'error'):
changes = self.replicator.database.changes(
feed='continuous',
heartbeat=1000)
beats = 0
for change in changes:
if beats == 300:
changes.stop()
if not change:
beats += 1
continue
elif change.get('id') == repl_id:
beats = 0
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') in ('completed', 'error'):
changes.stop()
self.assertEqual(repl_doc.get('_replication_state'), 'completed')
self.assertEqual(self.db.all_docs(), self.target_db.all_docs())
self.assertTrue(
all(x in self.target_db.keys(True) for x in [
'julia000',
'julia001',
'julia002'
])
)
def test_timeout_in_create_replication(self):
"""
Test that a read timeout exception is thrown when creating a
replicator with a timeout value of 500 ms.
"""
# Setup client with a timeout
self.set_up_client(auto_connect=True, timeout=.5)
self.db = self.client[self.test_target_dbname]
self.target_db = self.client[self.test_dbname]
# Construct a replicator with the updated client
self.replicator = Replicator(self.client)
repl_id = 'test-repl-{}'.format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replication_ids.append(repl_id)
# Test that the replication document was created
expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx']
# If Admin Party mode then user_ctx will not be in the key list
if self.client.admin_party:
expected_keys.pop()
self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys))
self.assertEqual(repl_doc['_id'], repl_id)
self.assertTrue(repl_doc['_rev'].startswith('1-'))
# Now that we know that the replication document was created,
# check that the replication timed out.
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') not in ('completed', 'error'):
# assert that a connection error is thrown because the read timed out
with self.assertRaises(ConnectionError) as cm:
changes = self.replicator.database.changes(
feed='continuous')
for change in changes:
continue
self.assertTrue(str(cm.exception).endswith('Read timed out.'))
def test_create_replication_without_a_source(self):
"""
Test that the replication document is not created and fails as expected
when no source database is provided.
"""
try:
repl_doc = self.replicator.create_replication()
self.fail('Above statement should raise a CloudantException')
except CloudantReplicatorException as err:
self.assertEqual(
str(err),
'You must specify either a source_db Database '
'object or a manually composed \'source\' string/dict.'
)
def test_create_replication_without_a_target(self):
"""
Test that the replication document is not created and fails as expected
when no target database is provided.
"""
try:
repl_doc = self.replicator.create_replication(self.db)
self.fail('Above statement should raise a CloudantException')
except CloudantReplicatorException as err:
self.assertEqual(
str(err),
'You must specify either a target_db Database '
'object or a manually composed \'target\' string/dict.'
)
def test_list_replications(self):
"""
Test that a list of Document wrapped objects are returned.
"""
self.populate_db_with_documents(3)
repl_ids = ['test-repl-{}'.format(
unicode_(uuid.uuid4())
) for _ in range(3)]
repl_docs = [self.replicator.create_replication(
self.db,
self.target_db,
repl_id
) for repl_id in repl_ids]
self.replication_ids.extend(repl_ids)
replications = self.replicator.list_replications()
all_repl_ids = [doc['_id'] for doc in replications]
match = [repl_id for repl_id in all_repl_ids if repl_id in repl_ids]
self.assertEqual(set(repl_ids), set(match))
def test_retrieve_replication_state(self):
"""
Test that the replication state can be retrieved for a replication
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replication_ids.append(repl_id)
repl_state = None
valid_states = ['completed', 'error', 'triggered', None]
finished = False
for _ in range(300):
repl_state = self.replicator.replication_state(repl_id)
self.assertTrue(repl_state in valid_states)
if repl_state in ('error', 'completed'):
finished = True
break
time.sleep(1)
self.assertTrue(finished)
def test_retrieve_replication_state_using_invalid_id(self):
"""
Test that replication_state(...) raises an exception as expected
when an invalid replication id is provided.
"""
repl_id = 'fake-repl-id-{}'.format(unicode_(uuid.uuid4()))
repl_state = None
try:
self.replicator.replication_state(repl_id)
self.fail('Above statement should raise a CloudantException')
except CloudantReplicatorException as err:
self.assertEqual(
str(err),
'Replication with id {} not found.'.format(repl_id)
)
self.assertIsNone(repl_state)
def test_stop_replication(self):
"""
Test that a replication can be stopped.
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replicator.stop_replication(repl_id)
try:
# The .fetch() will fail since the replication has been stopped
# and the replication document has been removed from the db.
repl_doc.fetch()
self.fail('Above statement should raise a CloudantException')
except requests.HTTPError as err:
self.assertEqual(err.response.status_code, 404)
def test_stop_replication_using_invalid_id(self):
"""
Test that stop_replication(...) raises an exception as expected
when an invalid replication id is provided.
"""
repl_id = 'fake-repl-id-{}'.format(unicode_(uuid.uuid4()))
try:
self.replicator.stop_replication(repl_id)
self.fail('Above statement should raise a CloudantException')
except CloudantReplicatorException as err:
self.assertEqual(
str(err),
'Replication with id {} not found.'.format(repl_id)
)
def test_follow_replication(self):
"""
Test that follow_replication(...) properly iterates updated
replication documents while the replication is executing.
"""
self.populate_db_with_documents(3)
repl_id = "test-repl-{}".format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(
self.db,
self.target_db,
repl_id
)
self.replication_ids.append(repl_id)
valid_states = ('completed', 'error', 'triggered', None)
repl_states = []
for doc in self.replicator.follow_replication(repl_id):
self.assertIn(doc.get('_replication_state'), valid_states)
repl_states.append(doc.get('_replication_state'))
self.assertTrue(len(repl_states) > 0)
self.assertEqual(repl_states[-1], 'completed')
self.assertNotIn('error', repl_states)