def test_replicate_upsert(self, mock_post):
self.assertFalse(ReplicationStack.get_instance().contains_data())
self.query_manager.upsert('col', [{
'id': 1,
'first_name': 'fn',
'last_name': 'ln'
}])
while ReplicationStack.get_instance().contains_data() is True:
time.sleep(DatabaseContext.THREADS_CYCLE)
self.assertEqual(len(ReplicationStack.get_instance().errors), 0)
self.assertIn(mock.call(url='localhost:5002/admin/replicate/auth'),
mock_post.call_args_list)
self.assertIn(
mock.call(url='localhost:5001/admin/replicate/sync',
data={
'collection':
'col',
'docs': [{
'id': 1,
'first_name': 'fn',
'last_name': 'ln'
}]
},
headers={'Authorization': 'Bearer first_token'}),
mock_post.call_args_list)
self.assertIn(
mock.call(url='localhost:5002/admin/replicate/sync',
data={
'collection':
'col',
'docs': [{
'id': 1,
'first_name': 'fn',
'last_name': 'ln'
}]
},
headers={'Authorization': 'Bearer second_token'}),
mock_post.call_args_list)
self.assertEqual(len(mock_post.call_args_list), 3)
def run(self):
try:
url = self.item['url']
del self.item['url']
self.auth(url)
response = self.make_query(url)
if response.status_code == 401 or response.status_code == 403:
self.auth(url)
response = self.make_query(url)
if response.status_code != 200:
self.make_query(url)
except:
try:
self.make_query(url)
except Exception as e:
print(f'Replication thread failed with {e}')
self.item['error'] = str(e)
ReplicationStack.get_instance().push_error(self.item)
def test_replicate_delete(self, mock_delete, mock_post):
self.assertFalse(ReplicationStack.get_instance().contains_data())
self.query_manager.delete('col', {'$filter': {'id': 3}})
while ReplicationStack.get_instance().contains_data() is True:
time.sleep(DatabaseContext.THREADS_CYCLE)
self.assertEqual(len(ReplicationStack.get_instance().errors), 0)
self.assertIn(mock.call(url='localhost:5002/admin/replicate/auth'),
mock_post.call_args_list)
self.assertIn(
mock.call(url='localhost:5001/admin/replicate/sync',
data={
'collection': 'col',
'search_query': {
'$filter': {
'id': 3
}
}
},
headers={'Authorization': 'Bearer first_token'}),
mock_delete.call_args_list)
self.assertIn(
mock.call(url='localhost:5002/admin/replicate/sync',
data={
'collection': 'col',
'search_query': {
'$filter': {
'id': 3
}
}
},
headers={'Authorization': 'Bearer second_token'}),
mock_delete.call_args_list)
self.assertEqual(len(mock_delete.call_args_list), 2)
self.assertEqual(len(mock_post.call_args_list), 1)
def get_status(self):
cols = []
for f in os.listdir(DatabaseContext.DATA_FOLDER):
col_meta_data = CollectionMetaData(f)
cols.append({
'collection': f,
'count': self.collections_service.count(col_meta_data),
'size (bytes)': self.collection_size(col_meta_data)
})
return {
'collections': cols,
'cleaning_operations': CleaningStack.get_instance().get_details(),
'query_operations': QueryStack.get_instance().get_details(),
'replication_operations':
ReplicationStack.get_instance().get_details()
}
def run(self):
available_threads = DatabaseContext.MAX_THREADS
while DatabaseContext.THREADS_MANAGER_CYCLING:
try:
if len(
DatabaseContext.SLAVES
) > 0 and ReplicationStack.get_instance().contains_data():
ReplicationThread().start()
if CleaningStack.get_instance().contains_data():
CleaningThread().start()
threads_by_query = QueryStack.get_instance().threads_needed()
if threads_by_query is not None:
# copy the list to iterate over all the elements while first finish sooner
threads = list(threads_by_query['threads'])
for t in threads:
QueryThread(threads_by_query['query_id'], t).start()
time.sleep(DatabaseContext.THREADS_CYCLE)
except Exception as e:
print(f'Threads manager failed with {e}')
def __init__(self):
Thread.__init__(self)
self.item = ReplicationStack.get_instance().pop()
def delete(self, collection, search_query):
query_id = QueryStack.get_instance().push_delete(collection, search_query)
ReplicationStack.get_instance().push_delete(collection, search_query)
return QueryStack.get_instance().pop_results(query_id)
def patch(self, collection, previous_doc, doc):
query_id = QueryStack.get_instance().push_patch(collection, previous_doc, doc)
ReplicationStack.get_instance().push_patch(collection, previous_doc, doc)
return QueryStack.get_instance().pop_results(query_id)
def upsert(self, collection, docs):
query_id = QueryStack.get_instance().push_upsert(collection, docs)
ReplicationStack.get_instance().push_upsert(collection, docs)
return QueryStack.get_instance().pop_results(query_id)