def setUp(self):
super(TestRdirBackend, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path}
self.rdir = RdirBackend(self.conf)
self.volume = random_id(32)
self.rdir.create(self.volume)
self.container_0 = '0' + random_id(63)
self.container_1 = '1' + random_id(63)
self.container_2 = '2' + random_id(63)
self.content_0 = '0' + random_id(31)
self.content_1 = '1' + random_id(31)
self.chunk_0 = random_id(64)
def create_app(conf, **kwargs):
app = flask.Flask(__name__)
app.register_blueprint(rdir_api)
app.backend = RdirBackend(conf)
app.ns = conf['namespace']
# we want exceptions to be logged
if conf.get('log_level') == 'DEBUG':
app.config['PROPAGATE_EXCEPTIONS'] = True
return app
def create_app(conf, **kwargs):
backend = RdirBackend(conf)
app = Rdir(conf, backend)
# default to sync worker for rdir
if not conf.get('worker_class'):
conf['worker_class'] = 'sync'
# default to 1 worker (concurrency issue with leveldb)
if not conf.get('workers'):
conf['workers'] = 1
return app
def create_app(conf, **kwargs):
app = flask.Flask(__name__)
app.register_blueprint(rdir_api)
app.backend = RdirBackend(conf)
app.ns = conf['namespace']
# default to sync worker for rdir
if not conf.get('worker_class'):
conf['worker_class'] = 'sync'
# default to 1 worker (concurrency issue with leveldb)
if not conf.get('workers'):
conf['workers'] = 1
# we want exceptions to be logged
if conf.get('log_level') == 'DEBUG':
app.config['PROPAGATE_EXCEPTIONS'] = True
return app
def setUp(self):
super(TestRdirBackend, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path}
self.rdir = RdirBackend(self.conf)
class TestRdirBackend(unittest.TestCase):
def setUp(self):
super(TestRdirBackend, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path}
self.rdir = RdirBackend(self.conf)
def tearDown(self):
super(TestRdirBackend, self).tearDown()
del self.rdir
shutil.rmtree(self.db_path)
def test_chunk_push_mtime(self):
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
mtime=1234)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 1234})
])
def test_chunk_push_rtime(self):
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
rtime=5555)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 5555, 'rtime': 5555})
])
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
mtime=4444, rtime=5555)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 4444, 'rtime': 5555})
])
def test_chunk_push_no_rtime_no_mtime(self):
self.assertRaises(ServerException,
self.rdir.chunk_push,
"myvolume", "mycontainer",
"mycontent", "mychunk")
def test_push_allowed_tokens(self):
data_put = {
'content_version': 1,
'content_nbchunks': 3,
'content_path': "path",
'content_size': 1234,
'chunk_hash': "1234567890ABCDEF",
'chunk_position': "1",
'chunk_size': 123,
'mtime': 123456,
'rtime': 456
}
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
**data_put)
data_fetch = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data_fetch, [
("mycontainer|mycontent|mychunk", data_put)
])
def test_chunk_push_update_data(self):
# initial push
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
rtime=5555, mtime=6666)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 6666, 'rtime': 5555})
])
# update mtime and rtime
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
mtime=1111, rtime=2222)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 1111, 'rtime': 2222})
])
# update only mtime
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
mtime=9999)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 9999, 'rtime': 2222})
])
# update only rtime
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
rtime=7777)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 9999, 'rtime': 7777})
])
def test_chunk_delete(self):
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
rtime=5555, mtime=6666)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 6666, 'rtime': 5555})
])
self.rdir.chunk_delete("myvolume", "mycontainer", "mycontent",
"mychunk")
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [])
def test_fetch(self):
# initial push (container name are unordered)
self.rdir.chunk_push("myvolume", "mycontainer0", "mycontent2",
"mychunk",
mtime=2)
self.rdir.chunk_push("myvolume", "mycontainer0", "mycontent1",
"mychunk",
mtime=1)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent4",
"mychunk",
mtime=10)
self.rdir.chunk_push("myvolume", "mycontainer1", "mycontent3",
"mychunk",
mtime=3, rtime=4)
# fetch all data (leveldb sorted entries by key)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer0|mycontent1|mychunk", {"mtime": 1}),
("mycontainer0|mycontent2|mychunk", {"mtime": 2}),
("mycontainer1|mycontent3|mychunk", {"mtime": 3, "rtime": 4}),
("mycontainer2|mycontent4|mychunk", {"mtime": 10})
])
# fetch 0 records max
data = self.rdir.chunk_fetch("myvolume", limit=0)
self.assertEqual(data, [])
# fetch 2 records max
data = self.rdir.chunk_fetch("myvolume", limit=2)
self.assertEqual(data, [
("mycontainer0|mycontent1|mychunk", {"mtime": 1}),
("mycontainer0|mycontent2|mychunk", {"mtime": 2})
])
# fetch 5 records max from record number 3
data = self.rdir.chunk_fetch("myvolume", limit=5,
start_after="mycontainer0|"
"mycontent2|mychunk")
self.assertEqual(data, [
("mycontainer1|mycontent3|mychunk", {"mtime": 3, "rtime": 4}),
("mycontainer2|mycontent4|mychunk", {"mtime": 10})
])
# fetch 5 records max from last record
data = self.rdir.chunk_fetch("myvolume", limit=2,
start_after="mycontainer2|"
"mycontent4|mychunk")
self.assertEqual(data, [])
# fetch all data from record number 2
data = self.rdir.chunk_fetch("myvolume", start_after="mycontainer0|"
"mycontent2|"
"mychunk")
self.assertEqual(data, [
("mycontainer1|mycontent3|mychunk", {"mtime": 3, "rtime": 4}),
("mycontainer2|mycontent4|mychunk", {"mtime": 10})
])
# rebuild mode: no incident date so no entry
data = self.rdir.chunk_fetch("myvolume", rebuild=True)
self.assertEqual(data, [])
# rebuild mode: with incident date
self.rdir.admin_set_incident_date("myvolume", 6)
data = self.rdir.chunk_fetch("myvolume", rebuild=True)
self.assertEqual(data, [
("mycontainer0|mycontent1|mychunk", {"mtime": 1}),
("mycontainer0|mycontent2|mychunk", {"mtime": 2}),
])
def test_rdir_status(self):
# initial pushes
self.rdir.chunk_push("myvolume", "mycontainer0", "mycontent1",
"mychunk", mtime=10)
self.rdir.chunk_push("myvolume", "mycontainer1", "mycontent2",
"mychunk", mtime=10)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent4",
"mychunk", mtime=10)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent5",
"mychunk", mtime=20)
data = self.rdir.chunk_status("myvolume")
self.assertEqual(data,
{
'chunk': {
'total': 4
},
'container': {
'mycontainer0': {
'total': 1
},
'mycontainer1': {
'total': 1
},
'mycontainer2': {
'total': 2
},
}
})
def test_rdir_status_rebuild(self):
self.rdir.admin_set_incident_date("myvolume", 30)
# initial pushes
self.rdir.chunk_push("myvolume", "mycontainer0", "mycontent1",
"mychunk", mtime=10)
self.rdir.chunk_push("myvolume", "mycontainer1", "mycontent2",
"mychunk", mtime=10, rtime=20)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent4",
"mychunk", mtime=10)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent5",
"mychunk", mtime=20, rtime=30)
data = self.rdir.chunk_status("myvolume")
self.assertEqual(data,
{
'rebuild': {
'incident_date': 30
},
'chunk': {
'total': 4,
'rebuilt': 2
},
'container': {
'mycontainer0': {
'total': 1,
'rebuilt': 0
},
'mycontainer1': {
'total': 1,
'rebuilt': 1
},
'mycontainer2': {
'total': 2,
'rebuilt': 1
},
}
})
def test_status(self):
self.assertEqual(self.rdir.status(), {'opened_db_count': 0})
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
rtime=5555, mtime=6666)
self.assertEqual(self.rdir.status(), {'opened_db_count': 1})
def test_multi_volume(self):
self.rdir.chunk_push("myvolume1", "mycontainer", "mycontent",
"mychunk",
mtime=1111)
self.rdir.chunk_push("myvolume2", "mycontainer", "mycontent",
"mychunk",
mtime=2222)
data = self.rdir.chunk_fetch("myvolume1")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 1111})
])
data = self.rdir.chunk_fetch("myvolume2")
self.assertEqual(data, [
("mycontainer|mycontent|mychunk", {'mtime': 2222})
])
def test_push_mix_container_content_chunk(self):
self.rdir.chunk_push("myvolume", "mycontainer1", "mycontent",
"mychunk",
mtime=1)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent1",
"mychunk",
mtime=2)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent2",
"mychunk1",
mtime=3)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent2",
"mychunk2",
mtime=4)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer1|mycontent|mychunk", {'mtime': 1}),
("mycontainer2|mycontent1|mychunk", {'mtime': 2}),
("mycontainer2|mycontent2|mychunk1", {'mtime': 3}),
("mycontainer2|mycontent2|mychunk2", {'mtime': 4})
])
def test_admin_incident_date(self):
# no incident date
date = self.rdir.admin_get_incident_date("myvolume")
self.assertEqual(date, None)
# add incident date
self.rdir.admin_set_incident_date("myvolume", 1234)
self.assertEqual(self.rdir.admin_get_incident_date("myvolume"), 1234)
# update incident date
self.rdir.admin_set_incident_date("myvolume", 5555)
self.assertEqual(self.rdir.admin_get_incident_date("myvolume"), 5555)
def test_admin_lock_unlock(self):
# unlock without lock
self.rdir.admin_unlock("myvolume")
# lock
res = self.rdir.admin_lock("myvolume", "a functionnal test")
self.assertEqual(res, None)
# double lock
res = self.rdir.admin_lock("myvolume", "an other functionnal test")
self.assertEqual(res, "a functionnal test")
# unlock
self.rdir.admin_unlock("myvolume")
# lock again
res = self.rdir.admin_lock("myvolume", "a third functionnal test")
self.assertEqual(res, None)
def test_admin_show(self):
self.rdir.admin_lock("myvolume", "a functionnal test")
self.rdir.admin_set_incident_date("myvolume", 1234)
res = self.rdir.admin_show("myvolume")
self.assertEqual(res, {'incident_date': "1234",
'lock': "a functionnal test"})
def test_admin_clear(self):
# populate the db
self.rdir.admin_set_incident_date("myvolume", 666)
self.rdir.chunk_push("myvolume", "mycontainer1", "mycontent",
"mychunk", mtime=1)
self.rdir.chunk_push("myvolume", "mycontainer2", "mycontent1",
"mychunk", mtime=2, rtime=3)
# clear rebuilt chunk entries
count = self.rdir.admin_clear("myvolume", False)
self.assertEqual(count, 1)
# check db state
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [
("mycontainer1|mycontent|mychunk", {'mtime': 1})
])
self.assertEqual(self.rdir.admin_get_incident_date("myvolume"), None)
# populate again the db
self.rdir.chunk_push("myvolume", "mycontainer3", "mycontent1",
"mychunk", mtime=3, rtime=4)
# clear all entries
count = self.rdir.admin_clear("myvolume", True)
self.assertEqual(count, 2)
# check db state
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [])
def setUp(self):
super(TestRdirBackend, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path}
self.rdir = RdirBackend(self.conf)
class TestRdirBackend(unittest.TestCase):
def setUp(self):
super(TestRdirBackend, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path}
self.rdir = RdirBackend(self.conf)
def tearDown(self):
super(TestRdirBackend, self).tearDown()
del self.rdir
shutil.rmtree(self.db_path)
def test_chunk_push_mtime(self):
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
mtime=1234)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 1234
})])
def test_chunk_push_rtime(self):
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
rtime=5555)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 5555,
'rtime': 5555
})])
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
mtime=4444,
rtime=5555)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 4444,
'rtime': 5555
})])
def test_chunk_push_no_rtime_no_mtime(self):
self.assertRaises(ServerException, self.rdir.chunk_push, "myvolume",
"mycontainer", "mycontent", "mychunk")
def test_push_allowed_tokens(self):
data_put = {
'content_version': 1,
'content_nbchunks': 3,
'content_path': "path",
'content_size': 1234,
'chunk_hash': "1234567890ABCDEF",
'chunk_position': "1",
'chunk_size': 123,
'mtime': 123456,
'rtime': 456
}
self.rdir.chunk_push("myvolume", "mycontainer", "mycontent", "mychunk",
**data_put)
data_fetch = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data_fetch,
[("mycontainer|mycontent|mychunk", data_put)])
def test_chunk_push_update_data(self):
# initial push
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
rtime=5555,
mtime=6666)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 6666,
'rtime': 5555
})])
# update mtime and rtime
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
mtime=1111,
rtime=2222)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 1111,
'rtime': 2222
})])
# update only mtime
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
mtime=9999)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 9999,
'rtime': 2222
})])
# update only rtime
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
rtime=7777)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 9999,
'rtime': 7777
})])
def test_chunk_delete(self):
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
rtime=5555,
mtime=6666)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 6666,
'rtime': 5555
})])
self.rdir.chunk_delete("myvolume", "mycontainer", "mycontent",
"mychunk")
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [])
def test_fetch(self):
# initial push (container name are unordered)
self.rdir.chunk_push("myvolume",
"mycontainer0",
"mycontent2",
"mychunk",
mtime=2)
self.rdir.chunk_push("myvolume",
"mycontainer0",
"mycontent1",
"mychunk",
mtime=1)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent4",
"mychunk",
mtime=10)
self.rdir.chunk_push("myvolume",
"mycontainer1",
"mycontent3",
"mychunk",
mtime=3,
rtime=4)
# fetch all data (leveldb sorted entries by key)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer0|mycontent1|mychunk", {
"mtime": 1
}), ("mycontainer0|mycontent2|mychunk", {
"mtime": 2
}), ("mycontainer1|mycontent3|mychunk", {
"mtime": 3,
"rtime": 4
}), ("mycontainer2|mycontent4|mychunk", {
"mtime": 10
})])
# fetch 0 records max
data = self.rdir.chunk_fetch("myvolume", limit=0)
self.assertEqual(data, [])
# fetch 2 records max
data = self.rdir.chunk_fetch("myvolume", limit=2)
self.assertEqual(data, [("mycontainer0|mycontent1|mychunk", {
"mtime": 1
}), ("mycontainer0|mycontent2|mychunk", {
"mtime": 2
})])
# fetch 5 records max from record number 3
data = self.rdir.chunk_fetch("myvolume",
limit=5,
start_after="mycontainer0|"
"mycontent2|mychunk")
self.assertEqual(data, [("mycontainer1|mycontent3|mychunk", {
"mtime": 3,
"rtime": 4
}), ("mycontainer2|mycontent4|mychunk", {
"mtime": 10
})])
# fetch 5 records max from last record
data = self.rdir.chunk_fetch("myvolume",
limit=2,
start_after="mycontainer2|"
"mycontent4|mychunk")
self.assertEqual(data, [])
# fetch all data from record number 2
data = self.rdir.chunk_fetch("myvolume",
start_after="mycontainer0|"
"mycontent2|"
"mychunk")
self.assertEqual(data, [("mycontainer1|mycontent3|mychunk", {
"mtime": 3,
"rtime": 4
}), ("mycontainer2|mycontent4|mychunk", {
"mtime": 10
})])
# rebuild mode: no incident date so no entry
data = self.rdir.chunk_fetch("myvolume", rebuild=True)
self.assertEqual(data, [])
# rebuild mode: with incident date
self.rdir.admin_set_incident_date("myvolume", 6)
data = self.rdir.chunk_fetch("myvolume", rebuild=True)
self.assertEqual(data, [
("mycontainer0|mycontent1|mychunk", {
"mtime": 1
}),
("mycontainer0|mycontent2|mychunk", {
"mtime": 2
}),
])
def test_rdir_status(self):
# initial pushes
self.rdir.chunk_push("myvolume",
"mycontainer0",
"mycontent1",
"mychunk",
mtime=10)
self.rdir.chunk_push("myvolume",
"mycontainer1",
"mycontent2",
"mychunk",
mtime=10)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent4",
"mychunk",
mtime=10)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent5",
"mychunk",
mtime=20)
data = self.rdir.chunk_status("myvolume")
self.assertEqual(
data, {
'chunk': {
'total': 4
},
'container': {
'mycontainer0': {
'total': 1
},
'mycontainer1': {
'total': 1
},
'mycontainer2': {
'total': 2
},
}
})
def test_rdir_status_rebuild(self):
self.rdir.admin_set_incident_date("myvolume", 30)
# initial pushes
self.rdir.chunk_push("myvolume",
"mycontainer0",
"mycontent1",
"mychunk",
mtime=10)
self.rdir.chunk_push("myvolume",
"mycontainer1",
"mycontent2",
"mychunk",
mtime=10,
rtime=20)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent4",
"mychunk",
mtime=10)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent5",
"mychunk",
mtime=20,
rtime=30)
data = self.rdir.chunk_status("myvolume")
self.assertEqual(
data, {
'rebuild': {
'incident_date': 30
},
'chunk': {
'total': 4,
'rebuilt': 2
},
'container': {
'mycontainer0': {
'total': 1,
'rebuilt': 0
},
'mycontainer1': {
'total': 1,
'rebuilt': 1
},
'mycontainer2': {
'total': 2,
'rebuilt': 1
},
}
})
def test_status(self):
self.assertEqual(self.rdir.status(), {'opened_db_count': 0})
self.rdir.chunk_push("myvolume",
"mycontainer",
"mycontent",
"mychunk",
rtime=5555,
mtime=6666)
self.assertEqual(self.rdir.status(), {'opened_db_count': 1})
def test_multi_volume(self):
self.rdir.chunk_push("myvolume1",
"mycontainer",
"mycontent",
"mychunk",
mtime=1111)
self.rdir.chunk_push("myvolume2",
"mycontainer",
"mycontent",
"mychunk",
mtime=2222)
data = self.rdir.chunk_fetch("myvolume1")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 1111
})])
data = self.rdir.chunk_fetch("myvolume2")
self.assertEqual(data, [("mycontainer|mycontent|mychunk", {
'mtime': 2222
})])
def test_push_mix_container_content_chunk(self):
self.rdir.chunk_push("myvolume",
"mycontainer1",
"mycontent",
"mychunk",
mtime=1)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent1",
"mychunk",
mtime=2)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent2",
"mychunk1",
mtime=3)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent2",
"mychunk2",
mtime=4)
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer1|mycontent|mychunk", {
'mtime': 1
}), ("mycontainer2|mycontent1|mychunk", {
'mtime': 2
}), ("mycontainer2|mycontent2|mychunk1", {
'mtime': 3
}), ("mycontainer2|mycontent2|mychunk2", {
'mtime': 4
})])
def test_admin_incident_date(self):
# no incident date
date = self.rdir.admin_get_incident_date("myvolume")
self.assertEqual(date, None)
# add incident date
self.rdir.admin_set_incident_date("myvolume", 1234)
self.assertEqual(self.rdir.admin_get_incident_date("myvolume"), 1234)
# update incident date
self.rdir.admin_set_incident_date("myvolume", 5555)
self.assertEqual(self.rdir.admin_get_incident_date("myvolume"), 5555)
def test_admin_lock_unlock(self):
# unlock without lock
self.rdir.admin_unlock("myvolume")
# lock
res = self.rdir.admin_lock("myvolume", "a functionnal test")
self.assertEqual(res, None)
# double lock
res = self.rdir.admin_lock("myvolume", "an other functionnal test")
self.assertEqual(res, "a functionnal test")
# unlock
self.rdir.admin_unlock("myvolume")
# lock again
res = self.rdir.admin_lock("myvolume", "a third functionnal test")
self.assertEqual(res, None)
def test_admin_show(self):
self.rdir.admin_lock("myvolume", "a functionnal test")
self.rdir.admin_set_incident_date("myvolume", 1234)
res = self.rdir.admin_show("myvolume")
self.assertEqual(res, {
'incident_date': "1234",
'lock': "a functionnal test"
})
def test_admin_clear(self):
# populate the db
self.rdir.admin_set_incident_date("myvolume", 666)
self.rdir.chunk_push("myvolume",
"mycontainer1",
"mycontent",
"mychunk",
mtime=1)
self.rdir.chunk_push("myvolume",
"mycontainer2",
"mycontent1",
"mychunk",
mtime=2,
rtime=3)
# clear rebuilt chunk entries
count = self.rdir.admin_clear("myvolume", False)
self.assertEqual(count, 1)
# check db state
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [("mycontainer1|mycontent|mychunk", {
'mtime': 1
})])
self.assertEqual(self.rdir.admin_get_incident_date("myvolume"), None)
# populate again the db
self.rdir.chunk_push("myvolume",
"mycontainer3",
"mycontent1",
"mychunk",
mtime=3,
rtime=4)
# clear all entries
count = self.rdir.admin_clear("myvolume", True)
self.assertEqual(count, 2)
# check db state
data = self.rdir.chunk_fetch("myvolume")
self.assertEqual(data, [])
class TestRdirBackend(unittest.TestCase):
def setUp(self):
super(TestRdirBackend, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path}
self.rdir = RdirBackend(self.conf)
self.volume = random_id(32)
self.rdir.create(self.volume)
self.container_0 = '0' + random_id(63)
self.container_1 = '1' + random_id(63)
self.container_2 = '2' + random_id(63)
self.content_0 = '0' + random_id(31)
self.content_1 = '1' + random_id(31)
self.chunk_0 = random_id(64)
def tearDown(self):
super(TestRdirBackend, self).tearDown()
del self.rdir
shutil.rmtree(self.db_path)
def test_explicit_create(self):
newvolume = "mynewvolume"
self.assertRaises(NoSuchDB,
self.rdir.chunk_push,
newvolume, self.container_0, self.content_0,
self.chunk_0, mtime=1234)
self.rdir.create(newvolume)
self.rdir.chunk_push(newvolume, self.container_0, self.content_0,
self.chunk_0, mtime=1234)
def test_chunk_push_mtime(self):
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, mtime=1234)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0), {'mtime': 1234})
])
def test_chunk_push_rtime(self):
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, rtime=5555)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{'mtime': 5555, 'rtime': 5555})
])
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, mtime=4444, rtime=5555)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{'mtime': 4444, 'rtime': 5555})
])
def test_chunk_push_no_rtime_no_mtime(self):
self.assertRaises(ServerException,
self.rdir.chunk_push,
self.volume, self.container_0,
self.content_0, self.chunk_0)
def test_chunk_push_update_data(self):
# initial push
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, rtime=5555, mtime=6666)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{'mtime': 6666, 'rtime': 5555})
])
# update mtime and rtime
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, mtime=1111, rtime=2222)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{'mtime': 1111, 'rtime': 2222})
])
# update only mtime
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, mtime=9999)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{'mtime': 9999, 'rtime': 2222})
])
# update only rtime
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, rtime=7777)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{'mtime': 9999, 'rtime': 7777})
])
def test_chunk_delete(self):
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, rtime=5555, mtime=6666)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{'mtime': 6666, 'rtime': 5555})
])
self.rdir.chunk_delete(self.volume, self.container_0, self.content_0,
self.chunk_0)
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [])
def test_fetch(self):
# initial push (container name are unordered)
self.rdir.chunk_push(self.volume, self.container_0, self.content_1,
self.chunk_0, mtime=2)
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, mtime=1)
self.rdir.chunk_push(self.volume, self.container_2, self.content_0,
self.chunk_0, mtime=10)
self.rdir.chunk_push(self.volume, self.container_1, self.content_0,
self.chunk_0, mtime=3, rtime=4)
# fetch all data
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(len(data), 4)
for c in [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{"mtime": 1}),
("%s|%s|%s" %
(self.container_0, self.content_1, self.chunk_0),
{"mtime": 2}),
("%s|%s|%s" %
(self.container_1, self.content_0, self.chunk_0),
{"mtime": 3, "rtime": 4}),
("%s|%s|%s" %
(self.container_2, self.content_0, self.chunk_0),
{"mtime": 10})]:
self.assertTrue(c in data)
# fetch 0 records max
data = self.rdir.chunk_fetch(self.volume, limit=0)
self.assertEqual(data, [])
# fetch 2 records max
data = self.rdir.chunk_fetch(self.volume, limit=2)
self.assertEqual(len(data), 2)
for c in [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{"mtime": 1}),
("%s|%s|%s" %
(self.container_0, self.content_1, self.chunk_0),
{"mtime": 2})]:
self.assertTrue(c in data)
# fetch 5 records max from record number 3
data = self.rdir.chunk_fetch(
self.volume, limit=5,
start_after="%s|%s|%s" %
(self.container_0, self.content_1, self.chunk_0))
self.assertEqual(len(data), 2)
for c in [
("%s|%s|%s" %
(self.container_1, self.content_0, self.chunk_0),
{"mtime": 3, "rtime": 4}),
("%s|%s|%s" %
(self.container_2, self.content_0, self.chunk_0),
{"mtime": 10})]:
self.assertTrue(c in data)
# fetch 5 records max from last record
data = self.rdir.chunk_fetch(
self.volume, limit=2,
start_after="%s|%s|%s" %
(self.container_2, self.content_0, self.chunk_0))
self.assertEqual(data, [])
# fetch all data from record number 2
data = self.rdir.chunk_fetch(
self.volume,
start_after="%s|%s|%s" %
(self.container_0, self.content_1, self.chunk_0))
self.assertEqual(len(data), 2)
for c in [
("%s|%s|%s" %
(self.container_1, self.content_0, self.chunk_0),
{"mtime": 3, "rtime": 4}),
("%s|%s|%s" %
(self.container_2, self.content_0, self.chunk_0),
{"mtime": 10})]:
self.assertTrue(c in data)
# rebuild mode: no incident date so no entry
data = self.rdir.chunk_fetch(self.volume, rebuild=True)
self.assertEqual(data, [])
# rebuild mode: with incident date
self.rdir.admin_set_incident_date(self.volume, 6)
data = self.rdir.chunk_fetch(self.volume, rebuild=True)
self.assertEqual(len(data), 2)
for c in [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0),
{"mtime": 1}),
("%s|%s|%s" %
(self.container_0, self.content_1, self.chunk_0),
{"mtime": 2})]:
self.assertTrue(c in data)
def test_rdir_status(self):
# initial pushes
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, mtime=10)
self.rdir.chunk_push(self.volume, self.container_1, self.content_0,
self.chunk_0, mtime=10)
self.rdir.chunk_push(self.volume, self.container_2, self.content_0,
self.chunk_0, mtime=10)
self.rdir.chunk_push(self.volume, self.container_2, self.content_1,
self.chunk_0, mtime=20)
data = self.rdir.chunk_status(self.volume)
self.assertEqual(data,
{
'chunk': {
'total': 4
},
'container': {
self.container_0: {
'total': 1
},
self.container_1: {
'total': 1
},
self.container_2: {
'total': 2
},
}
})
def test_rdir_status_rebuild(self):
self.rdir.admin_set_incident_date(self.volume, 30)
# initial pushes
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, mtime=10)
self.rdir.chunk_push(self.volume, self.container_1, self.content_0,
self.chunk_0, mtime=10, rtime=20)
self.rdir.chunk_push(self.volume, self.container_2, self.content_0,
self.chunk_0, mtime=10)
self.rdir.chunk_push(self.volume, self.container_2, self.content_1,
self.chunk_0, mtime=20, rtime=30)
data = self.rdir.chunk_status(self.volume)
self.assertEqual(data,
{
'rebuild': {
'incident_date': 30
},
'chunk': {
'total': 4,
'rebuilt': 2
},
'container': {
self.container_0: {
'total': 1,
'rebuilt': 0
},
self.container_1: {
'total': 1,
'rebuilt': 1
},
self.container_2: {
'total': 2,
'rebuilt': 1
},
}
})
def test_status(self):
self.assertEqual(self.rdir.status(), {'opened_db_count': 0})
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, rtime=5555, mtime=6666)
self.assertEqual(self.rdir.status(), {'opened_db_count': 1})
def test_multi_volume(self):
self.rdir.create("myvolume1")
self.rdir.create("myvolume2")
self.rdir.chunk_push("myvolume1", self.container_0, self.content_0,
self.chunk_0, mtime=1111)
self.rdir.chunk_push("myvolume2", self.container_0, self.content_0,
self.chunk_0, mtime=2222)
data = self.rdir.chunk_fetch("myvolume1")
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0), {'mtime': 1111})
])
data = self.rdir.chunk_fetch("myvolume2")
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0), {'mtime': 2222})
])
def test_admin_incident_date(self):
# no incident date
date = self.rdir.admin_get_incident_date(self.volume)
self.assertEqual(date, None)
# add incident date
self.rdir.admin_set_incident_date(self.volume, 1234)
self.assertEqual(self.rdir.admin_get_incident_date(self.volume), 1234)
# update incident date
self.rdir.admin_set_incident_date(self.volume, 5555)
self.assertEqual(self.rdir.admin_get_incident_date(self.volume), 5555)
def test_admin_lock_unlock(self):
# unlock without lock
self.rdir.admin_unlock(self.volume)
# lock
who = random_id(32)
res = self.rdir.admin_lock(self.volume, who)
self.assertEqual(res, None)
# double lock
res = self.rdir.admin_lock(self.volume, random_id(32))
self.assertEqual(res, who)
# unlock
self.rdir.admin_unlock(self.volume)
# lock again
res = self.rdir.admin_lock(self.volume, random_id(32))
self.assertEqual(res, None)
def test_admin_show(self):
who = random_id(32)
self.rdir.admin_lock(self.volume, who)
self.rdir.admin_set_incident_date(self.volume, 1234)
res = self.rdir.admin_show(self.volume)
self.assertEqual(res, {'incident_date': "1234",
'lock': who})
def test_admin_clear(self):
# populate the db
self.rdir.admin_set_incident_date(self.volume, 666)
self.rdir.chunk_push(self.volume, self.container_0, self.content_0,
self.chunk_0, mtime=1)
self.rdir.chunk_push(self.volume, self.container_1, self.content_0,
self.chunk_0, mtime=2, rtime=3)
# clear rebuilt chunk entries
count = self.rdir.admin_clear(self.volume, False)
self.assertEqual(count, 1)
# check db state
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [
("%s|%s|%s" %
(self.container_0, self.content_0, self.chunk_0), {'mtime': 1})
])
self.assertEqual(self.rdir.admin_get_incident_date(self.volume), None)
# populate again the db
self.rdir.chunk_push(self.volume, self.container_2, self.content_0,
self.chunk_0, mtime=3, rtime=4)
# clear all entries
count = self.rdir.admin_clear(self.volume, True)
self.assertEqual(count, 2)
# check db state
data = self.rdir.chunk_fetch(self.volume)
self.assertEqual(data, [])
