def test_upgradeFrom3(self):
source = Backend.LocalBackend(
os.path.join(THISDIR, "data"))(VOLNAME + "-version3")
self.makeDirs("target")
target = Backend.LocalBackend(self.getDir())("target")
self.assertEqual(B.getVolumeInfo(source)["volume_role"],
ToolCut.WanBackupVolumeRole.Normal)
B.backup(source, target)
def test_Basic(self):
B.testConfiguration()
testbackend.createNamespace("basebackup")
target = testbackend.theBackend()("basebackup")
backupWithMonitoring(self, self.source, target, endSnapshot="A")
backupWithMonitoring(self, self.source, target, endSnapshot="C")
backupWithMonitoring(self, self.source, target, endSnapshot="E")
backupWithMonitoring(self, self.source, target)
def test_Basic(self):
B.testConfiguration()
testbackend.createNamespace("basebackup")
target = testbackend.theBackend()("basebackup")
backupWithMonitoring(self, self.source, target, endSnapshot="A")
backupWithMonitoring(self, self.source, target, endSnapshot="C")
backupWithMonitoring(self, self.source, target, endSnapshot="E")
backupWithMonitoring(self, self.source, target)
def test_upgradeFrom3(self):
source = Backend.LocalBackend(os.path.join(
THISDIR, "data"))(VOLNAME + "-version3")
self.makeDirs("target")
target = Backend.LocalBackend(self.getDir())("target")
self.assertEqual(
B.getVolumeInfo(source)["volume_role"],
ToolCut.WanBackupVolumeRole.Normal)
B.backup(source, target)
def test_Monitoring(self):
B.testConfiguration()
ns = "monitortest"
testbackend.createNamespace(ns)
target = testbackend.theBackend()(ns)
self.assertEqual(B.getProgressInfo(target), None)
backupWithMonitoring(self, self.source, target)
info = B.getProgressInfo(target)
self.assertTrue(type(info), dict)
self.assertEqual(info["total_size"], info["seen"])
self.assertEqual(info["still_to_be_examined"], 0)
self.assertTrue(info["sent_to_backend"] <= info["total_size"])
def test_Monitoring(self):
B.testConfiguration()
ns = "monitortest"
testbackend.createNamespace(ns)
target = testbackend.theBackend()(ns)
self.assertEqual(B.getProgressInfo(target), None)
backupWithMonitoring(self, self.source, target)
info = B.getProgressInfo(target)
self.assertTrue(type(info), dict)
self.assertEqual(info["total_size"], info["seen"])
self.assertEqual(info["still_to_be_examined"], 0)
self.assertTrue(info["sent_to_backend"] <= info["total_size"])
def backupWithMonitoring(self, source, target, *args, **kwargs):
infoList = []
PITReplication.removeProgressInfo(target)
backupThread = ThreadWithExn(target=PITReplication.backup,
args=((source, target) + args),
kwargs=kwargs)
backupThread.start()
progressInfo = None
def check_progress(self, old, new, infoList):
# check monotonicity
self.assertTrue(old == None or new != None)
if old != None:
self.assertEqual(new["total_size"], old["total_size"])
self.assertTrue(new["seen"] >= old["seen"])
self.assertTrue(new["sent_to_backend"] >= old["sent_to_backend"])
if new != None:
self.assertTrue(new["status"] in ["running", "finished"])
if old == None or new["seen"] > old["seen"]:
infoList.append(new)
previoustime = time.time()
delta = 1.0
while backupThread.isAlive():
time.sleep(0.01)
newtime = time.time()
if newtime > previoustime + delta:
previoustime = newtime
newProgressInfo = PITReplication.getProgressInfo(target)
check_progress(self, progressInfo, newProgressInfo, infoList)
progressInfo = newProgressInfo
backupThread.join_with_exception()
newProgressInfo = PITReplication.getProgressInfo(target)
check_progress(self, progressInfo, newProgressInfo, infoList)
self.assertNotEqual(
newProgressInfo, None,
"no progressinfo found after backup has successfully finished")
self.assertEqual(newProgressInfo["status"], "finished")
self.assertTrue(
len(infoList) >= 1, "internal test error in backupWithMonitoring")
return infoList
def backupWithMonitoring(self, source, target, *args, **kwargs):
infoList = []
PITReplication.removeProgressInfo(target)
backupThread = ThreadWithExn(target=PITReplication.backup,
args=((source, target) + args),
kwargs=kwargs)
backupThread.start()
progressInfo = None
def check_progress(self, old, new, infoList):
# check monotonicity
self.assertTrue(old == None or new != None)
if old != None:
self.assertEqual(new["total_size"], old["total_size"])
self.assertTrue(new["seen"] >= old["seen"])
self.assertTrue(new["sent_to_backend"] >= old["sent_to_backend"])
if new != None:
self.assertTrue(new["status"] in ["running", "finished"])
if old == None or new["seen"] > old["seen"]:
infoList.append(new)
previoustime = time.time()
delta = 1.0
while backupThread.isAlive():
time.sleep(0.01)
newtime = time.time()
if newtime > previoustime + delta:
previoustime = newtime
newProgressInfo = PITReplication.getProgressInfo(target)
check_progress(self, progressInfo, newProgressInfo, infoList)
progressInfo = newProgressInfo
backupThread.join_with_exception()
newProgressInfo = PITReplication.getProgressInfo(target)
check_progress(self, progressInfo, newProgressInfo, infoList)
self.assertNotEqual(
newProgressInfo, None, "no progressinfo found after backup has successfully finished")
self.assertEqual(newProgressInfo["status"], "finished")
self.assertTrue(len(infoList) >= 1,
"internal test error in backupWithMonitoring")
return infoList
def test_Config(self):
p2 = Backend.LocalBackend(self.pathTo("backup1"))("somenamespace")
_ = B.prepareEnvironment("jobID",
os.path.join(env.tmpDir, "whatever"),
self.source.getJSONDict(),
p2.getJSONDict(),
endSnapshot="C",
startSnapshot="A")
def test_OutOfOrder(self):
for ns in ["incremental", "final", "usb"]:
testbackend.createNamespace(ns)
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("usb"),
endSnapshot="C")
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("incremental"),
startSnapshot="C")
B.copy(testbackend.theBackend()("usb"),
testbackend.theBackend()("final"))
backupWithMonitoring(self,
testbackend.theBackend()("incremental"),
testbackend.theBackend()("final"))
def test_OutOfOrder(self):
for ns in ["incremental", "final", "usb"]:
testbackend.createNamespace(ns)
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("usb"),
endSnapshot="C")
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("incremental"),
startSnapshot="C")
B.copy(testbackend.theBackend()("usb"),
testbackend.theBackend()("final"))
backupWithMonitoring(self,
testbackend.theBackend()("incremental"),
testbackend.theBackend()("final"))
def test_Config(self):
p2 = Backend.LocalBackend(self.pathTo("backup1"))("somenamespace")
_ = B.prepareEnvironment("jobID",
os.path.join(env.tmpDir, "whatever"),
self.source.getJSONDict(),
p2.getJSONDict(),
endSnapshot="C",
startSnapshot="A")
def failscrub_test(self):
# """ create a namespace
# prepare it with at least 2 snapshots for scrubbing (cfr. backup_test -> testScrubbingAndSnapshotMgmtTarget)
# scrub 1 snapshot -> should succeed
# take a lock on the namespace
# scrub the other snapshot -> should fail
# """
PITReplication.testConfiguration()
ns = "failscrub-test"
testbackend.createNamespace(ns)
target = testbackend.theBackend()(ns)
def check(allss, toscrubss):
self.assertEqual(PITReplication.getVolumeSnapshots(target), allss)
self.assertEqual(
PITReplication.getScrubSnapshots(target), toscrubss)
for ss in PITReplication.getVolumeSnapshots(self.source):
PITReplication.backup(self.source, target, endSnapshot=ss)
check(['A', 'B', 'C', 'D', 'E', 'F'], [])
PITReplication.deleteSnapshots(target, "B")
check(['A', 'C', 'D', 'E', 'F'], ["C"])
pid = lockedexecutable.start(target)
# self.assertRaises(Exception,
# PITReplication.scrubSnapshot,
# target,
# "C",
# 'scrub the other snapshot should fail')
lockedexecutable.stop(pid)
def failscrub_test(self):
# """ create a namespace
# prepare it with at least 2 snapshots for scrubbing (cfr. backup_test -> testScrubbingAndSnapshotMgmtTarget)
# scrub 1 snapshot -> should succeed
# take a lock on the namespace
# scrub the other snapshot -> should fail
# """
PITReplication.testConfiguration()
ns = "failscrub-test"
testbackend.createNamespace(ns)
target = testbackend.theBackend()(ns)
def check(allss, toscrubss):
self.assertEqual(PITReplication.getVolumeSnapshots(target), allss)
self.assertEqual(PITReplication.getScrubSnapshots(target),
toscrubss)
for ss in PITReplication.getVolumeSnapshots(self.source):
PITReplication.backup(self.source, target, endSnapshot=ss)
check(['A', 'B', 'C', 'D', 'E', 'F'], [])
PITReplication.deleteSnapshots(target, "B")
check(['A', 'C', 'D', 'E', 'F'], ["C"])
pid = lockedexecutable.start(target)
# self.assertRaises(Exception,
# PITReplication.scrubSnapshot,
# target,
# "C",
# 'scrub the other snapshot should fail')
lockedexecutable.stop(pid)
def failbackup_test(self):
# """ create a namespace
# backup to certain snapshot -> should succeed
# take a lock on the namespace
# try to backup to the end of the volume -> this should fail
# """
PITReplication.testConfiguration()
ns = "failbackup-test"
testbackend.createNamespace(ns)
target = testbackend.theBackend()(ns)
PITReplication.backup(self.source, target, startSnapshot="A")
pid = lockedexecutable.start(target)
self.assertRaises(Exception,
PITReplication.backup,
self.source,
target,
endSnapshot="C")
lockedexecutable.stop(pid)
def failbackup_test(self):
# """ create a namespace
# backup to certain snapshot -> should succeed
# take a lock on the namespace
# try to backup to the end of the volume -> this should fail
# """
PITReplication.testConfiguration()
ns = "failbackup-test"
testbackend.createNamespace(ns)
target = testbackend.theBackend()(ns)
PITReplication.backup(self.source, target, startSnapshot="A")
pid = lockedexecutable.start(target)
self.assertRaises(Exception,
PITReplication.backup,
self.source,
target,
endSnapshot="C")
lockedexecutable.stop(pid)
def test_ScrubbingAndSnapshotMgmtTarget(self):
testbackend.createNamespace("toscrub")
target = testbackend.theBackend()("toscrub")
# def check(allss, toscrubss):
# self.assertEqual(B.getVolumeSnapshots(target), allss)
# self.assertEqual(B.getScrubSnapshots(target), toscrubss)
for ss in B.getVolumeSnapshots(self.source):
backupWithMonitoring(self, self.source, target, endSnapshot=ss)
def test_ScrubbingAndSnapshotMgmtTarget(self):
testbackend.createNamespace("toscrub")
target = testbackend.theBackend()("toscrub")
# def check(allss, toscrubss):
# self.assertEqual(B.getVolumeSnapshots(target), allss)
# self.assertEqual(B.getScrubSnapshots(target), toscrubss)
for ss in B.getVolumeSnapshots(self.source):
backupWithMonitoring(self, self.source, target, endSnapshot=ss)
def testTool(target):
sp = B.getVolumeSnapshotsTool(target)
count = 0
for s in sp.getSnapshots():
logger.info(s.name())
logger.info(s.stored())
logger.info(s.date())
nose.tools.eq_(s.stored(), sp.snapshotStored(s.name()))
count += s.stored()
count += sp.currentStored()
nose.tools.eq_(sp.stored(), count)
def testTool(target):
sp = B.getVolumeSnapshotsTool(target)
count = 0
for s in sp.getSnapshots():
logger.info(s.name())
logger.info(s.stored())
logger.info(s.date())
nose.tools.eq_(s.stored(),
sp.snapshotStored(s.name()))
count += s.stored()
count += sp.currentStored()
nose.tools.eq_(sp.stored(), count)
def test_restore(self):
ns1 = "xxx"
ns2 = "yyy"
for ns in [ns1, ns2]:
testbackend.createNamespace(ns)
self.makeDirs("DC2", ns2)
DC1 = testbackend.theBackend()
DC2 = Backend.LocalBackend(self.pathTo("DC2"))
B.copy(self.source, DC1(ns1))
B.copy(DC1(ns1), DC1(ns2))
B.copy(DC1(ns2), DC2(ns2))
def test_restore(self):
ns1 = "xxx"
ns2 = "yyy"
for ns in [ns1, ns2]:
testbackend.createNamespace(ns)
self.makeDirs("DC2", ns2)
DC1 = testbackend.theBackend()
DC2 = Backend.LocalBackend(self.pathTo("DC2"))
B.copy(self.source, DC1(ns1))
B.copy(DC1(ns1), DC1(ns2))
B.copy(DC1(ns2), DC2(ns2))
def failrestore_test(self):
# """ create a namespace
# copy a volume to it
# rename the volume
# take a lock on the namespace
# try to rename the volume again -> should fail
# """
PITReplication.testConfiguration()
ns = "failrestore-test"
testbackend.createNamespace(ns)
target = testbackend.theBackend()(ns)
PITReplication.copy(self.source, target)
PITReplication.renameVolume(target, "new_name1")
pid = lockedexecutable.start(target)
self.assertRaises(Exception, PITReplication.renameVolume, target,
"new_name2", 'rename the volume again should fail')
lockedexecutable.stop(pid)
def failrestore_test(self):
# """ create a namespace
# copy a volume to it
# rename the volume
# take a lock on the namespace
# try to rename the volume again -> should fail
# """
PITReplication.testConfiguration()
ns = "failrestore-test"
testbackend.createNamespace(ns)
target = testbackend.theBackend()(ns)
PITReplication.copy(self.source, target)
PITReplication.renameVolume(target, "new_name1")
pid = lockedexecutable.start(target)
self.assertRaises(Exception,
PITReplication.renameVolume,
target,
"new_name2",
'rename the volume again should fail')
lockedexecutable.stop(pid)
def check(allss, toscrubss):
self.assertEqual(PITReplication.getVolumeSnapshots(target), allss)
self.assertEqual(PITReplication.getScrubSnapshots(target),
toscrubss)
def test_VolumeRoles(self):
for ns in ["base", "inc"]:
testbackend.createNamespace(ns)
self.assertEqual(
B.getVolumeInfo(self.source)["volume_role"],
ToolCut.WanBackupVolumeRole.Normal)
for s in ["B", "C", "D"]:
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("base"),
endSnapshot=s)
self.assertEqual(
B.getVolumeInfo(
testbackend.theBackend()("base"))["volume_role"],
ToolCut.WanBackupVolumeRole.BackupBase)
self.assertEqual(
B.getVolumeSnapshots(testbackend.theBackend()("base")),
["B", "C", "D"])
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("inc"),
startSnapshot="B",
endSnapshot="C")
self.assertEqual(
B.getVolumeInfo(testbackend.theBackend()("inc"))["volume_role"],
ToolCut.WanBackupVolumeRole.BackupIncremental)
self.assertEqual(B.getVolumeSnapshots(testbackend.theBackend()("inc")),
["B", "C"])
backupWithMonitoring(self, self.source,
testbackend.theBackend()("inc"))
self.assertEqual(
B.getVolumeInfo(testbackend.theBackend()("inc"))["volume_role"],
ToolCut.WanBackupVolumeRole.BackupIncremental)
self.assertEqual(B.getVolumeSnapshots(testbackend.theBackend()("inc")),
["B", "C", "F"])
self.assertRaises(Exception, B.promote,
testbackend.theBackend()("inc"))
B.promote(testbackend.theBackend()("base"))
self.assertEqual(
B.getVolumeInfo(testbackend.theBackend()("base"))["volume_role"],
ToolCut.WanBackupVolumeRole.Normal)
self.makeDirs("copy")
cpy = Backend.LocalBackend(self.getDir())("copy")
B.copy(self.source, cpy)
self.assertEqual(
B.getVolumeInfo(cpy)["volume_role"],
ToolCut.WanBackupVolumeRole.Normal)
def test_rename(self):
testbackend.createNamespace("xxx")
target = testbackend.theBackend()("xxx")
B.copy(self.source, target)
B.renameVolume(target, "new_name")
def check(allss, toscrubss):
self.assertEqual(PITReplication.getVolumeSnapshots(target), allss)
self.assertEqual(
PITReplication.getScrubSnapshots(target), toscrubss)
def test_rename(self):
testbackend.createNamespace("xxx")
target = testbackend.theBackend()("xxx")
B.copy(self.source, target)
B.renameVolume(target, "new_name")
def test_VolumeRoles(self):
for ns in ["base", "inc"]:
testbackend.createNamespace(ns)
self.assertEqual(B.getVolumeInfo(self.source)["volume_role"],
ToolCut.WanBackupVolumeRole.Normal)
for s in ["B", "C", "D"]:
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("base"),
endSnapshot=s)
self.assertEqual(
B.getVolumeInfo(
testbackend.theBackend()("base"))["volume_role"],
ToolCut.WanBackupVolumeRole.BackupBase)
self.assertEqual(
B.getVolumeSnapshots(testbackend.theBackend()("base")),
["B", "C", "D"])
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("inc"),
startSnapshot="B",
endSnapshot="C")
self.assertEqual(
B.getVolumeInfo(testbackend.theBackend()("inc"))["volume_role"],
ToolCut.WanBackupVolumeRole.BackupIncremental)
self.assertEqual(B.getVolumeSnapshots(testbackend.theBackend()("inc")),
["B", "C"])
backupWithMonitoring(self,
self.source,
testbackend.theBackend()("inc"))
self.assertEqual(
B.getVolumeInfo(testbackend.theBackend()("inc"))["volume_role"],
ToolCut.WanBackupVolumeRole.BackupIncremental)
self.assertEqual(B.getVolumeSnapshots(testbackend.theBackend()("inc")),
["B", "C", "F"])
self.assertRaises(Exception,
B.promote, testbackend.theBackend()("inc"))
B.promote(testbackend.theBackend()("base"))
self.assertEqual(
B.getVolumeInfo(testbackend.theBackend()("base"))["volume_role"],
ToolCut.WanBackupVolumeRole.Normal)
self.makeDirs("copy")
cpy = Backend.LocalBackend(self.getDir())("copy")
B.copy(self.source, cpy)
self.assertEqual(B.getVolumeInfo(cpy)["volume_role"],
ToolCut.WanBackupVolumeRole.Normal)