def test_index_git_dict_key_hash_index(self):
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path, factory=TestRootFactory)
db.init_index()
db["_index"]["_git"] = churrodb.GitObjectHashIndex(name="git_root")
db["a"] = IndexedCollection()
db["a"].init_index()
db["a"]["_index"]["_a"] = churrodb.GitDictKeyHashIndex(
name="git_a", supply="git_root", dict_key="k")
db["a"]["b"] = churro.PersistentDict()
db["a"]["b"]["c"] = "d"
db["a"]["b"]["k"] = "1"
db.save()
self.assertDictEqual({
"c": "d",
"k": "1"
}, dict(object_by_hash(self.churrodb_path,
db.idx_find("1")[0])))
db = churrodb.ChurroDb(self.churrodb_path, factory=TestRootFactory)
self.assertDictEqual({
"c": "d",
"k": "1"
}, dict(object_by_hash(self.churrodb_path,
db.idx_find("1")[0])))
def test_index_git_index_key_persistence(self):
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = IndexedCollection()
db["a"].init_index()
db["a"]["_index"]["_git"] = churrodb.GitDictKeyHashIndex()
db["a"]["b"] = churro.PersistentDict({"id": "1", "c": "d"})
db["a"]["x"] = churro.PersistentDict({"id": "3", "h": "i"})
db.save()
self.assertEqual("ae1bc6576ccca4b0c416af2004971674e90fd501",
db["a"].idx_find_first("1"))
self.assertEqual("cc885e9312bc6b3a4006bf455f037ecbf0af6162",
db["a"].idx_find_first("3"))
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path)
self.assertEqual("ae1bc6576ccca4b0c416af2004971674e90fd501",
db["a"].idx_find_first("1"))
self.assertEqual("cc885e9312bc6b3a4006bf455f037ecbf0af6162",
db["a"].idx_find_first("3"))
del db["a"]["b"]
db["a"]["x"]["h"] = "j"
db["a"]["e"] = churro.PersistentDict({"id": "2", "f": "g"})
db.save()
tx = transaction.begin()
self.assertEqual("ae1bc6576ccca4b0c416af2004971674e90fd501",
db["a"].idx_find_first("1"))
self.assertEqual("de056b94c551a1b3413c9a740c6d07f163aee8ea",
db["a"].idx_find_first("2"))
self.assertEqual("2c45a823da31fc40936714d3e3e10e838acc9ad2",
db["a"].idx_find_first("3"))
tx.abort()
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path)
self.assertEqual("ae1bc6576ccca4b0c416af2004971674e90fd501",
db["a"].idx_find_first("1"))
self.assertEqual("de056b94c551a1b3413c9a740c6d07f163aee8ea",
db["a"].idx_find_first("2"))
self.assertEqual("2c45a823da31fc40936714d3e3e10e838acc9ad2",
db["a"].idx_find_first("3"))
tx.abort()
def test_index_update_on_abort(self):
tx = transaction.begin()
class TransMan(TestTransactionManager):
def tpc_vote(self, tx):
raise Exception
mock_index_factory = unittest.mock.MagicMock()
mock_index_prop = unittest.mock.MagicMock()
class Abc(churrodb.IndexMixin, churro.PersistentDict):
index_factory = mock_index_factory
tx.join(TransMan())
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Abc()
db["a"]._index = mock_index_prop
db["a"].init_index()
db["a"]["b"] = "c"
try:
tx.commit()
except Exception:
tx.abort()
self.assertEqual(1, mock_index_factory.call_count)
self.assertEqual(1, mock_index_prop.idx_update.call_count)
def test_index_indexes_folder(self):
tx = transaction.begin()
mock_index_a = unittest.mock.MagicMock()
mock_index_b = unittest.mock.MagicMock()
class Abc(churrodb.IndexMixin, churro.PersistentDict):
index_factory = churrodb.IndexesFolder
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Abc()
db["a"].init_index()
db["a"]["_index"]["idx_a"] = mock_index_a
db["a"]["_index"]["idx_b"] = mock_index_b
db["a"]["b"] = "c"
tx.commit()
self.assertEqual(1, mock_index_a.idx_update.call_count)
db["a"].idx_find("")
db["a"].idx_find("", "idx_b")
self.assertRaisesRegex(Exception, "there is no index called 'idx_c'",
lambda: db["a"].idx_find("", "idx_c"))
self.assertEqual(1, mock_index_a.idx_find.call_count)
self.assertEqual(2, mock_index_b.idx_find.call_count)
def test_concurrent_conflict_not_mergeable_existing_object(
self, unique_branch_name):
unique_branch_name.return_value = "conflict"
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Dummy()
db.save()
env_a = unittest.mock.MagicMock()
env_b = unittest.mock.MagicMock()
env_a.exception = None
def action_a(env):
db = env.action_result = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Dummy()
db["a"].value = "c"
def action_b(env):
db = env.action_result = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Dummy()
db["a"].value = "b"
def save(env):
env.action_result.save()
thread_b = threading.Thread(target=threader,
kwargs={
"env": env_b,
"action": action_b,
"save": save,
})
thread_a = threading.Thread(target=threader,
kwargs={
"env": env_a,
"action": action_a,
"save": save,
"other": thread_b
})
# with Python < 3.4 we probably have to use testfixtures.log_capture decorator (see below)
with self.assertLogs("churrodb", level="ERROR") as logged:
thread_a.start()
thread_a.join()
branches = get_branches(self.churrodb_path)
conflict_content = subprocess.check_output(
["git", "show", "conflict:a.churro"], cwd=self.churrodb_path)
conflict_json = json.loads(conflict_content.decode("utf-8"))
master_json = read_json(os.path.join(self.churrodb_path, "a.churro"))
self.assertIsInstance(env_a.exception, acidfs.ConflictError)
self.assertListEqual(["conflict", "* master"], branches)
self.assertEqual("b", master_json["__churro_data__"]["value"])
self.assertEqual("c", conflict_json["__churro_data__"]["value"])
self.assertListEqual(logged.output, [
"ERROR:churrodb:trying to commit this transaction caused a conflict",
"ERROR:churrodb:wrote problematic changeset to branch 'conflict', you have to resolve this yourself"
])
def test_transparent_access_to_churro_instance(self):
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = churro.Persistent()
root = db.root()
self.assertIs(db._churro.root(), root)
self.assertTrue(root._dirty)
db.flush()
self.assertFalse(root._dirty)
transaction.abort()
def test_index_git_index_persistence(self):
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path, factory=TestRootFactory)
db.init_index()
db["_index"]["_git"] = churrodb.GitObjectHashIndex(name="root_git")
db["a"] = GitIndexedCollection(idx_name="abc", idx_supply="root_git")
db["a"].init_index()
db["a"]["x"] = churro.PersistentDict({"a": "b"})
db.save()
self.assertEqual("d6e9f0f90bf4fe6acdc173607d70d8a8d7a0f612",
db.idx_find_first("x"))
db = churrodb.ChurroDb(self.churrodb_path)
self.assertEqual("d6e9f0f90bf4fe6acdc173607d70d8a8d7a0f612",
db.idx_find_first("x"))
def test_index_git_index_root_factory(self):
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path, factory=TestRootFactory)
db.init_index()
db["_index"]["_git"] = churrodb.GitObjectHashIndex(name="root_git",
inverse=True)
db["coll"] = GitIndexedCollection(idx_name="coll",
idx_supply="root_git")
db["coll"].init_index()
db["a"] = Dummy("c")
db["coll"]["x"] = Dummy("e")
db.save()
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path, factory=TestRootFactory)
self.assertEqual(
"c", db[db.idx_find("46d49b1a588f3684e0dc9f5ea6426a60512fd89d")
[0]].value)
self.assertEqual(
"e", db["coll"][db.idx_find(
"c6326067e195c781848cdca50797407bdbc6faeb")[0]].value)
db["c"] = Dummy("e")
db["coll"]["y"] = Dummy("f")
db.save()
db = churrodb.ChurroDb(self.churrodb_path, factory=TestRootFactory)
self.assertEqual(
"c", db[db.idx_find("46d49b1a588f3684e0dc9f5ea6426a60512fd89d")
[0]].value)
self.assertEqual(
"e", db[db.idx_find("c6326067e195c781848cdca50797407bdbc6faeb")
[0]].value)
self.assertEqual(
"f", db["coll"][db.idx_find(
"3534c705a203d4ef38e2e4d1b6b6d2a63dd3866d")[0]].value)
def test_index_full(self):
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = IndexedCollection()
db["a"].init_index()
db["a"]["_index"]["idx_a"] = TestDictIndex()
db["a"]["b"] = churro.Persistent()
db.save()
path = os.path.join(self.churrodb_path, "a", "_index", "idx_a.churro")
self.assertTrue(os.path.exists(path))
self.assertDictEqual(
{
"__churro_class__": "churrodb.tests.TestDictIndex",
"__churro_data__": {
"data": {
"_index": "indexed",
"b": "indexed"
}
}
}, read_json(path))
db = churrodb.ChurroDb(self.churrodb_path)
db["a"]["c"] = churro.Persistent()
db.save()
self.assertDictEqual(
{
"__churro_class__": "churrodb.tests.TestDictIndex",
"__churro_data__": {
"data": {
"_index": "indexed",
"b": "indexed",
"c": "indexed"
}
}
}, read_json(path))
def test_saving(self):
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = churro.Persistent()
db.save()
result = read_json(os.path.join(self.churrodb_path, "a.churro"))
self.assertDictEqual(
{
"__churro_data__": {},
"__churro_class__": "churro.Persistent"
}, result)
def test_concurrent_noconflict(self):
# initial commit so that we do not generate a false conflict
# strange, but see acidfs code for reference
db = churrodb.ChurroDb(self.churrodb_path)
db.save()
env_a = unittest.mock.MagicMock()
env_b = unittest.mock.MagicMock()
env_a.exception = None
def action_a(env):
db = env.action_result = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Dummy()
db["a"].value = "b"
env.db = db
def action_b(env):
db = env.action_result = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Dummy()
db["a"].value = "b"
db["b"] = Dummy()
db["b"].value = "c"
def save(env):
env.action_result.save()
thread_b = threading.Thread(target=threader,
kwargs={
"env": env_b,
"action": action_b,
"save": save,
})
thread_a = threading.Thread(target=threader,
kwargs={
"env": env_a,
"action": action_a,
"save": save,
"other": thread_b
})
thread_a.start()
thread_a.join()
json_a = read_json(os.path.join(self.churrodb_path, "a.churro"))
json_b = read_json(os.path.join(self.churrodb_path, "b.churro"))
self.assertIsNone(env_a.exception)
self.assertEqual("b", json_a["__churro_data__"]["value"])
self.assertEqual("c", json_b["__churro_data__"]["value"])
def test_index_abort_transaction_on_fail(self):
mock_index_factory = unittest.mock.MagicMock()
mock_index_prop = unittest.mock.MagicMock()
mock_index_prop.idx_update.side_effect = Exception
class Abc(churrodb.IndexMixin, churro.PersistentDict):
index_factory = mock_index_factory
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Abc()
db["a"]._index = mock_index_prop
db["a"]["b"] = "c"
self.assertRaises(Exception, lambda: transaction.commit())
mock_index_prop.idx_update.side_effect = None
transaction.commit()
self.assertEqual(2, mock_index_prop.idx_update.call_count)
def test_index_git_index_mixin(self):
tx = transaction.begin()
class MyCollectionA(churrodb.GitIndexMixin, churro.PersistentFolder):
_index = churro.PersistentProperty()
def git_index_key_mapper(k, v):
return k
class MyCollectionB(churrodb.GitIndexMixin, churro.PersistentFolder):
@staticmethod
def git_index_key_mapper(k, v):
return v.get("key")
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = MyCollectionA()
db["a"].init_index()
db["b"] = MyCollectionB()
db["b"]["_index"] = churro.Persistent()
db["b"].init_index()
db["a"]["x"] = Dummy("c")
db["b"]["y"] = churro.PersistentDict({"d": "e", "key": "identifier"})
db["b"]["z"] = churro.PersistentDict({"f": "g"})
db.save()
a_path = os.path.join(self.churrodb_path, "a", "__folder__.churro")
b_index_path = os.path.join(self.churrodb_path, "b", "_index.churro")
dict_a = read_json(a_path)
dict_b = read_json(b_index_path)
self.assertDictEqual(
{"x": "46d49b1a588f3684e0dc9f5ea6426a60512fd89d"},
dict_a["__churro_data__"]["_index"]["__churro_data__"]["data"])
self.assertDictEqual(
{"identifier": "ce7dbc999655ca73f204629774cda385a77599a4"},
dict_b["__churro_data__"]["data"])
self.assertEqual(
"identifier",
object_by_hash(self.churrodb_path,
"ce7dbc999655ca73f204629774cda385a77599a4")["key"])
def test_index_update_on_commit(self):
tx = transaction.begin()
mock_index_factory = unittest.mock.MagicMock()
mock_index_prop = unittest.mock.MagicMock()
class Abc(churrodb.IndexMixin, churro.PersistentDict):
index_factory = mock_index_factory
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Abc()
db["a"]._index = mock_index_prop
db["a"].init_index()
db["a"]["b"] = "c"
tx.commit()
self.assertEqual(1, mock_index_factory.call_count)
self.assertEqual(1, mock_index_prop.idx_update.call_count)
self.assertEqual(1, mock_index_prop.idx_validate.call_count)
def test_changing_branches(self):
tx = transaction.begin()
class Prop(churro.Persistent):
c = churro.PersistentProperty()
db = churrodb.ChurroDb(self.churrodb_path)
db["folder"] = churro.PersistentFolder()
db["folder"]["a"] = Prop()
db["folder"]["b"] = Prop()
tx.commit()
tx = transaction.begin()
db.switch("alternate")
tx.commit()
tx = transaction.begin()
gitout = subprocess.check_output(
["git", "show", "master:folder/a.churro"], cwd=self.churrodb_path)
master_json = json.loads(gitout.decode("utf-8"))
gitout = subprocess.check_output(
["git", "show", "alternate:folder/a.churro"],
cwd=self.churrodb_path)
alternate_json = json.loads(gitout.decode("utf-8"))
self.assertDictEqual(master_json, alternate_json)
db.switch("HEAD")
db["folder"]["a"].c = "d"
tx.commit()
gitout = subprocess.check_output(
["git", "show", "master:folder/a.churro"], cwd=self.churrodb_path)
master_json = json.loads(gitout.decode("utf-8"))
gitout = subprocess.check_output(
["git", "show", "alternate:folder/a.churro"],
cwd=self.churrodb_path)
alternate_json = json.loads(gitout.decode("utf-8"))
self.assertTrue(master_json != alternate_json)
def test_index_root_factory(self):
tx = transaction.begin()
class RootFactory(churrodb.IndexMixin, churro.PersistentFolder):
pass
db = churrodb.ChurroDb(self.churrodb_path, factory=RootFactory)
db.init_index()
db["_index"]["_a"] = churrodb.AbstractDictIndex({"a": "b"})
db["a"] = Dummy()
db.save()
a_path = os.path.join(self.churrodb_path, "a.churro")
index_path = os.path.join(self.churrodb_path, "_index")
self.assertEqual(["b"], db.idx_find("a"))
self.assertTrue(os.path.isdir(index_path))
self.assertDictEqual(
{
"__churro_data__": {
"value": None
},
"__churro_class__": "churrodb.tests.Dummy"
}, read_json(a_path))
def test_transparent_access_to_churro_root(self):
db = churrodb.ChurroDb(self.churrodb_path)
t = transaction.begin()
MockDb = unittest.mock.create_autospec(churrodb.ChurroDb)
mock_db = MockDb(self.churrodb_path)
prop_a = churro.Persistent()
prop_b = churro.Persistent()
prop_c = churro.Persistent()
# __setitem__
mock_db["a"] = prop_a
db["a"] = prop_a
db["b"] = prop_b
db["c"] = prop_c
keys = ["a", "b", "c"]
values = [prop_a, prop_b, prop_c]
# __nonzero__ (apparently Python 3 doesn't call __nonzero__ for bool()
# we include it here for reasons of integrity
# bool(mock_db)
# self.assertTrue(mock_db.__nonzero__.called)
self.assertTrue(db)
self.assertTrue(db._data)
self.assertTrue(db.root())
# __len__
len(mock_db)
self.assertEqual(3, len(db))
self.assertEqual(3, len(db._data))
self.assertEqual(3, len(db.root()))
# __getitem__
mock_db["a"]
self.assertIs(prop_a, db["a"])
self.assertIs(prop_b, db["b"])
self.assertIs(prop_c, db["c"])
self.assertIs(prop_a, db._data["a"])
self.assertIs(prop_b, db._data["b"])
self.assertIs(prop_c, db._data["c"])
self.assertIs(prop_a, db.root()["a"])
self.assertIs(prop_b, db.root()["b"])
self.assertIs(prop_c, db.root()["c"])
# get
mock_db.get("a")
self.assertIs(prop_a, db.get("a"))
self.assertIs(prop_b, db.get("b"))
self.assertIs(prop_c, db.get("c"))
# keys
mock_db.keys()
self.assertListEqual(keys, sorted(list(db.keys())))
# values
mock_db.values()
for value in db.values():
self.assertTrue(value in values)
# __contains__
"a" in mock_db
for key in keys:
self.assertTrue(key in db)
# __iter__
for k in mock_db:
pass
for key in db:
keys.remove(key)
self.assertEqual(0, len(keys))
# items
mock_db.items()
for key, value in db.items():
values.remove(value)
self.assertEqual(0, len(values))
# __delitem__
del mock_db["a"]
del db["a"]
# pop
mock_db.pop("a")
db.pop("b")
# remove
mock_db.remove("a")
db.remove("c")
self.assertFalse("a" in db)
self.assertFalse("b" in db)
self.assertFalse("c" in db)
self.assertFalse("a" in db._data)
self.assertFalse("b" in db._data)
self.assertFalse("c" in db._data)
self.assertFalse("a" in db.root())
self.assertFalse("b" in db.root())
self.assertFalse("c" in db.root())
self.assertEqual(1, mock_db.__setitem__.call_count)
self.assertEqual(1, mock_db.__len__.call_count)
self.assertEqual(1, mock_db.__getitem__.call_count)
self.assertEqual(1, mock_db.get.call_count)
self.assertEqual(1, mock_db.keys.call_count)
self.assertEqual(1, mock_db.values.call_count)
self.assertEqual(1, mock_db.__contains__.call_count)
self.assertEqual(1, mock_db.__iter__.call_count)
self.assertEqual(1, mock_db.items.call_count)
self.assertEqual(1, mock_db.__delitem__.call_count)
self.assertEqual(1, mock_db.pop.call_count)
self.assertEqual(1, mock_db.remove.call_count)
t.abort()
def test_concurrent_conflict_not_mergeable_nonexisting_object(self, mock):
mock.return_value = "conflict"
# this test is not final, see below for AcidFS git problem
# initial commit so that we do not generate a false conflict
# strange, but see acidfs code for reference
db = churrodb.ChurroDb(self.churrodb_path)
db.save()
env_a = unittest.mock.MagicMock()
env_b = unittest.mock.MagicMock()
env_a.exception = None
def action_a(env):
db = env.action_result = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Dummy()
db["a"].value = "c"
def action_b(env):
db = env.action_result = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Dummy()
db["a"].value = "b"
def save(env):
env.action_result.save()
thread_b = threading.Thread(target=threader,
kwargs={
"env": env_b,
"action": action_b,
"save": save,
})
thread_a = threading.Thread(target=threader,
kwargs={
"env": env_a,
"action": action_a,
"save": save,
"other": thread_b
})
with self.assertLogs("churrodb", level="ERROR"):
thread_a.start()
thread_a.join()
branches = get_branches(self.churrodb_path)
conflict_content = subprocess.check_output(
["git", "show", "conflict:a.churro"], cwd=self.churrodb_path)
conflict_json = json.loads(conflict_content.decode("utf-8"))
master_json = read_json(os.path.join(self.churrodb_path, "a.churro"))
# AcidFS git segfault problem (see https://github.com/Pylons/acidfs/issues/3)
# strangely enough, the bug doesn't always affect this test
# so we log in an INFO message whether this was the case
if isinstance(env_a.exception, subprocess.CalledProcessError):
log.info("git segfault bug detected")
self.assertIsInstance(env_a.exception, Exception)
self.assertListEqual(["conflict", "* master"], branches)
self.assertEqual("b", master_json["__churro_data__"]["value"])
self.assertEqual("c", conflict_json["__churro_data__"]["value"])
def action_b(env):
db = env.action_result = churrodb.ChurroDb(self.churrodb_path)
db["a"] = Dummy()
db["a"].value = "b"
def test_index_git_object_hash(self):
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path)
db["a"] = IndexedCollection()
db["a"].init_index()
db["a"]["_index"]["_a"] = churrodb.GitObjectHashIndex(True)
db["a"]["_index"]["_b"] = churrodb.GitObjectHashIndex()
db["a"]["b"] = Dummy("c")
db["a"]["c"] = Dummy("d")
db["a"]["d"] = Dummy("e")
db["a"]["e"] = Dummy("f")
db.save()
db = churro.Churro(self.churrodb_path)
db = db.root()
coll = db.get("a")
self.assertEqual("46d49b1a588f3684e0dc9f5ea6426a60512fd89d",
coll.idx_find("b")[0])
self.assertEqual("ab3a9aad770bc930fef6b1fd4eb03ad6d67fd407",
coll.idx_find("c")[0])
self.assertEqual("c6326067e195c781848cdca50797407bdbc6faeb",
coll.idx_find("d")[0])
self.assertEqual("3534c705a203d4ef38e2e4d1b6b6d2a63dd3866d",
coll.idx_find("e")[0])
self.assertEqual(
"c", coll[coll.idx_find("46d49b1a588f3684e0dc9f5ea6426a60512fd89d")
[0]].value)
self.assertEqual(
"d", coll[coll.idx_find("ab3a9aad770bc930fef6b1fd4eb03ad6d67fd407")
[0]].value)
self.assertEqual(
"e", coll[coll.idx_find("c6326067e195c781848cdca50797407bdbc6faeb")
[0]].value)
self.assertEqual(
"f", coll[coll.idx_find("3534c705a203d4ef38e2e4d1b6b6d2a63dd3866d")
[0]].value)
self.assertEqual(
coll[coll.idx_find("46d49b1a588f3684e0dc9f5ea6426a60512fd89d")
[0]].value, coll[coll["_index"]["_a"].idx_find_first(
"46d49b1a588f3684e0dc9f5ea6426a60512fd89d")].value)
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path)
db["a"]["b"] = Dummy("x")
db.save()
db = churro.Churro(self.churrodb_path)
db = db.root()
coll = db.get("a")
self.assertEqual(
"x", coll[coll.idx_find("a58a8f0b987cbb685ac125a060fb4ad0be7e76a0")
[0]].value)
tx = transaction.begin()
db = churrodb.ChurroDb(self.churrodb_path)
db["a"]["f"] = Dummy("d")
self.assertRaises(churrodb.IndexUpdateError, tx.commit)
transaction.abort()