def test_using_admin_party_source_and_target(self):
m_admin_party_client = self.setUpClientMocks(admin_party=True)
m_replicator = mock.MagicMock()
type(m_replicator).creds = mock.PropertyMock(return_value=None)
m_admin_party_client.__getitem__.return_value = m_replicator
# create source/target databases
src = CouchDatabase(m_admin_party_client, self.source_db)
tgt = CouchDatabase(m_admin_party_client, self.target_db)
# trigger replication
rep = Replicator(m_admin_party_client)
rep.create_replication(src, tgt, repl_id=self.repl_id)
kcall = m_replicator.create_document.call_args_list
self.assertEquals(len(kcall), 1)
args, kwargs = kcall[0]
self.assertEquals(len(args), 1)
expected_doc = {
'_id': self.repl_id,
'source': {
'url': '/'.join((self.server_url, self.source_db))
},
'target': {
'url': '/'.join((self.server_url, self.target_db))
}
}
self.assertDictEqual(args[0], expected_doc)
self.assertTrue(kwargs['throw_on_exists'])
def test_timeout_in_create_replication(self):
"""
Test that a read timeout exception is thrown when creating a
replicator with a timeout value of 500 ms.
"""
# Setup client with a timeout
self.set_up_client(auto_connect=True, timeout=.5)
self.db = self.client[self.test_target_dbname]
self.target_db = self.client[self.test_dbname]
# Construct a replicator with the updated client
self.replicator = Replicator(self.client)
repl_id = 'test-repl-{}'.format(unicode_(uuid.uuid4()))
repl_doc = self.replicator.create_replication(self.db, self.target_db,
repl_id)
self.replication_ids.append(repl_id)
# Test that the replication document was created
expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx']
# If Admin Party mode then user_ctx will not be in the key list
if self.client.admin_party:
expected_keys.pop()
self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys))
self.assertEqual(repl_doc['_id'], repl_id)
self.assertTrue(repl_doc['_rev'].startswith('1-'))
# Now that we know that the replication document was created,
# check that the replication timed out.
repl_doc = Document(self.replicator.database, repl_id)
repl_doc.fetch()
if repl_doc.get('_replication_state') not in ('completed', 'error'):
# assert that a connection error is thrown because the read timed out
with self.assertRaises(ConnectionError) as cm:
changes = self.replicator.database.changes(feed='continuous')
for change in changes:
continue
self.assertTrue(str(cm.exception).endswith('Read timed out.'))
def test_replication_with_generated_id(self):
clone = Replicator(self.client)
repl_id = clone.create_replication(
self.db,
self.target_db
)
self.replication_ids.append(repl_id['_id'])
def tearDown(self):
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
while self.replication_ids:
replicator.stop_replication(self.replication_ids.pop())
while self.dbs:
c.delete_database(self.dbs.pop())
def test_replication_state(self):
"""
_test_replication_state_
Verify that we can get the replication state.
"""
dbsource = unicode_("test_replication_state_source_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_replication_state_target_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_replication_state_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
replication_state = "not_yet_set"
while True:
# Verify that replication_state returns either None
# (if the field doesn't exist yet), or a valid
# replication state.
replication_state = replicator.replication_state(repl_id)
if replication_state is not None:
self.assertTrue(replication_state in
['completed', 'error', 'triggered'])
if replication_state in ('error', 'completed'):
break
LOG.debug(
"got replication state: {}".format(replication_state))
time.sleep(1)
def test_init(self):
"""
_test_init_
Verify that we can init our database object.
"""
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
replicator.all_docs()
def setUp(self):
"""
Set up test attributes
"""
super(ReplicatorTests, self).setUp()
self.db_set_up()
self.test_target_dbname = self.dbname()
self.target_db = self.client._DATABASE_CLASS(self.client,
self.test_target_dbname)
self.target_db.create()
self.replicator = Replicator(self.client)
self.replication_ids = []
def test_follow_replication_with_errors(self):
"""
_test_follow_replication_with_errors_
Test to make sure that we exit the follow loop when we submit
a bad replication.
"""
dbsource = unicode_("test_follow_replication_source_error_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_follow_replication_target_error_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_follow_replication_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
# Deliberately override these good params with bad params
source=dbsource + "foo",
target=dbtarget + "foo",
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'error')
def setup_replication(self,
source_db_name,
target,
repl_id=None,
use_log=None,
**kwargs):
"""
Set up replication of a database.
Args:
source_db_name: Source database
target: Target database URL or :py:class:`djali.couchdb.CloudiControl` instance
repl_id (str, optional): replication document ID
use_log (logging.Logger): Override logger instance
Keyword Args:
continuous(bool): Continuous replication (defaults to ``False``)
Returns:
cloudant.document.Document: replication document
"""
if use_log is None:
use_log = self.log
client = CouchDB(self._root_auth[0],
self._root_auth[1],
url=self.instance_url,
connect=True,
auto_renew=True)
replicator_obj = Replicator(client)
try:
target_database = target.database
except Exception:
target = CloudiControl(target, create=True)
target_database = target.database
source_database = client[source_db_name]
use_log.info("Setting up replication: {!s} -> {!s}".format(
source_database, target_database))
return replicator_obj.create_replication(source_database,
target_database,
repl_id=repl_id,
create_target=True,
continuous=kwargs.get(
"continuous", False))
def test_follow_replication(self):
"""
_test_follow_replication_
Test to make sure that we can follow a replication.
"""
dbsource = unicode_("test_follow_replication_source_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_follow_replication_target_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_follow_replication_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'completed')
def test_constructor_failure(self):
"""
Test that constructing a Replicator will not work
without a valid client.
"""
repl = None
try:
self.client.disconnect()
repl = Replicator(self.client)
self.fail('Above statement should raise a CloudantException')
except CloudantClientException as err:
self.assertEqual(
str(err), 'Database _replicator does not exist. '
'Verify that the client is valid and try again.')
finally:
self.assertIsNone(repl)
self.client.connect()
def configure_replicator(client, source_db, target_db):
"""
Currently we try and start a replication, if that fails that means one already exists. So we stop it and create a new one.
https://python-cloudant.readthedocs.io/en/latest/replicator.html#module-cloudant.replicator
How can we configure couchdb to delete this data...
"""
replicator = Replicator(client)
replication_id = f"{source_db.database_name}_to_{target_db.database_name}"
try:
replication_doc = start_replication(replicator, source_db, target_db,
replication_id)
except:
replicator.stop_replication(repl_id=replication_id)
replication_doc = start_replication(replicator, source_db, target_db,
replication_id)
def test_list_replications(self):
"""
_test_list_replications_
Verify that we get a list of replications documents back when
we got to list replications.
"""
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
repl_ids = []
num_reps = 3
for i in range(0, num_reps):
tag = "{0}_{1}".format(i, unicode_(uuid.uuid4()))
dbsource = unicode_("test_list_repl_src_{}".format(tag))
dbtarget = unicode_("test_list_repl_tgt_{}".format(tag))
self.dbs.append(dbsource)
self.dbs.append(dbtarget)
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
repl_id = unicode_("test_create_replication_{}".format(tag))
self.replication_ids.append(repl_id)
repl_ids.append(repl_id)
ret = replicator.create_replication(source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False)
replications = replicator.list_replications()
ids = [doc['_id'] for doc in replications]
found_ids = [i for i in ids if i in repl_ids]
self.assertEqual(num_reps, len(found_ids))
def test_using_iam_auth_source_and_target(self):
m_iam_auth_client = self.setUpClientMocks(iam_api_key=MOCK_API_KEY)
m_replicator = mock.MagicMock()
m_iam_auth_client.__getitem__.return_value = m_replicator
# create source/target databases
src = CouchDatabase(m_iam_auth_client, self.source_db)
tgt = CouchDatabase(m_iam_auth_client, self.target_db)
# trigger replication
rep = Replicator(m_iam_auth_client)
rep.create_replication(src,
tgt,
repl_id=self.repl_id,
user_ctx=self.user_ctx)
kcall = m_replicator.create_document.call_args_list
self.assertEquals(len(kcall), 1)
args, kwargs = kcall[0]
self.assertEquals(len(args), 1)
expected_doc = {
'_id': self.repl_id,
'user_ctx': self.user_ctx,
'source': {
'auth': {
'iam': {
'api_key': MOCK_API_KEY
}
},
'url': '/'.join((self.server_url, self.source_db))
},
'target': {
'auth': {
'iam': {
'api_key': MOCK_API_KEY
}
},
'url': '/'.join((self.server_url, self.target_db))
}
}
self.assertDictEqual(args[0], expected_doc)
self.assertTrue(kwargs['throw_on_exists'])
def test_using_basic_auth_source_and_target(self):
test_basic_auth_header = 'abc'
m_basic_auth_client = self.setUpClientMocks()
m_replicator = mock.MagicMock()
m_basic_auth_client.__getitem__.return_value = m_replicator
m_basic_auth_client.basic_auth_str.return_value = test_basic_auth_header
# create source/target databases
src = CouchDatabase(m_basic_auth_client, self.source_db)
tgt = CouchDatabase(m_basic_auth_client, self.target_db)
# trigger replication
rep = Replicator(m_basic_auth_client)
rep.create_replication(src,
tgt,
repl_id=self.repl_id,
user_ctx=self.user_ctx)
kcall = m_replicator.create_document.call_args_list
self.assertEquals(len(kcall), 1)
args, kwargs = kcall[0]
self.assertEquals(len(args), 1)
expected_doc = {
'_id': self.repl_id,
'user_ctx': self.user_ctx,
'source': {
'headers': {
'Authorization': test_basic_auth_header
},
'url': '/'.join((self.server_url, self.source_db))
},
'target': {
'headers': {
'Authorization': test_basic_auth_header
},
'url': '/'.join((self.server_url, self.target_db))
}
}
self.assertDictEqual(args[0], expected_doc)
self.assertTrue(kwargs['throw_on_exists'])
def test_create_replication(self):
"""
_test_create_replication_
Make a couple of test databases, and confirm that docs from
one get transferred to t'other.
"""
dbsource = unicode_("test_create_replication_source_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_create_replication_target_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_create_replication_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False)
try:
repl_doc = replicator[repl_id]
except KeyError:
repl_doc = None
if not repl_doc or not (repl_doc.get('_replication_state', "none")
in ('completed', 'error')):
for change in replicator.changes():
if change.get('id') == repl_id:
try:
repl_doc = replicator[repl_id]
repl_doc.fetch()
except KeyError:
pass
if repl_doc and (repl_doc.get('_replication_state',
"none")
in ('completed', 'error')):
break
else:
LOG.debug(
unicode_("Waiting for replication to complete "
"(repl_doc: {})".format(repl_doc)))
self.assertTrue(repl_doc)
self.assertEqual(repl_doc.get('_replication_state'), 'completed')
for d in ['doc1', 'doc2', 'doc3']:
self.assertTrue(dbt[d])
self.assertEqual(dbt[d]['testing'], dbs[d]['testing'])
def test_replication_with_generated_id(self):
clone = Replicator(self.client)
clone.create_replication(self.db, self.target_db)