def setUp(self):
self.config = Config.get_config()
self.db = DB()
self.db.check_upgrade()
self.mark_db_ids()
self.test_folder = tempfile.mkdtemp()
self.files_folder = os.path.join(self.test_folder, "files")
self.store_folder = os.path.join(self.test_folder, "store")
self.restore_folder = os.path.join(self.test_folder, "restore")
utils.makedirs(self.files_folder)
utils.makedirs(self.store_folder)
utils.makedirs(self.restore_folder)
utils.build_file_structure(self.files_folder, 50 * const.Kilobyte, 500 * const.Kilobyte)
# Build a store object (dont save config)
# Note the careful size selection - we want backups to overflow the FolderStore.
self.store = FolderStore("teststore", "2MB", True, self.store_folder)
self.config.storage[self.store.name] = self.store
# Build the backup object (dont save config)
self.backup = Backup("testbackup")
self.backup.include_folders = [self.files_folder]
self.backup.store = self.store.name
self.backup.notify_msg = False
self.include_packages = True
self.config.backups[self.backup.name] = self.backup
# build an options object for use with the backup
self.options = BlankClass()
self.options.dry_run = False
self.options.message = False
self.options.email = False
self.options.shutdown = False
self.options.norecurse = False
self.old_pass = self.config.data_passphrase
self.config.data_passphrase = "banana"
class ServerTestCase(unittest.TestCase):
def setUp(self):
self.config = Config.get_config()
self.db = DB()
self.db.check_upgrade()
self.mark_db_ids()
self.test_folder = tempfile.mkdtemp()
self.files_folder = os.path.join(self.test_folder, "files")
self.store_folder = os.path.join(self.test_folder, "store")
self.restore_folder = os.path.join(self.test_folder, "restore")
utils.makedirs(self.files_folder)
utils.makedirs(self.store_folder)
utils.makedirs(self.restore_folder)
utils.build_file_structure(self.files_folder, 50 * const.Kilobyte, 500 * const.Kilobyte)
# Build a store object (dont save the config)
# Note the careful size selection - we want backups to overflow the FolderStore.
self.store = FolderStore("teststore", "2MB", True, self.store_folder)
self.config.storage[self.store.name] = self.store
# Build the backup object (dont save config)
self.backup = Backup("testbackup")
self.backup.include_folders = [self.files_folder]
self.backup.store = self.store.name
self.backup.notify_msg = False
self.old_pass = self.config.data_passphrase
self.config.data_passphrase = "goofy"
self.backup.encrypt = True
self.config.backups[self.backup.name] = self.backup
# build an options object for use with the backup
self.options = BlankClass()
self.options.dry_run = False
self.options.message = False
self.options.email = False
self.options.shutdown = False
self.options.norecurse = False
def tearDown(self):
self.config.data_passphrase = self.old_pass
# Remove all DB records created during this test
self.clean_db()
shutil.rmtree(self.test_folder)
self.assertFalse(os.path.isdir(self.test_folder))
def testBackupRestore(self):
self.backup_restore_compare()
def testCheckFiles(self):
self.backup.include_packages = True
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Check that all the right files are there.
runs = self.db.runs(self.backup.name)
self.assertEqual(len(runs), 1)
run = runs[0]
folder = run.folder
self.assertTrue(self.store.exists(os.path.join(folder, const.PackageFile + const.EncryptionSuffix)))
self.assertTrue(self.store.exists(os.path.join(folder, const.LOFFile + const.EncryptionSuffix)))
self.assertTrue(self.store.exists(os.path.join(folder, const.ConfigName + const.EncryptionSuffix)))
self.assertTrue(self.store.exists)
def testAutoManagementOfStore1(self):
# Run a set of backups that will overload the store.
# The automanaged store should continue to archive old backups as required.
# Store space reclaimation happens across all backups (i.e. any run).
# We should see older runs from the first backup disappear.
max_size, dummy, dummy = self.store.limit_details()
filesize = utils.du(self.backup.include_folders[0])
# Lets make sure we are going to do enough backups that
# the older ones will be removed.
RunCount = (max_size // filesize) + 2
last_start = None
for cycle in xrange(RunCount):
if last_start:
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - last_start < timedelta(seconds=1):
time.sleep(0.01)
backup = Backup(self.backup.name + str(cycle))
backup.include_folders = self.backup.include_folders
backup.store = self.backup.store
backup.notify_msg = False
self.config.backups[backup.name] = backup
# Run a full backup
b = Run(backup.name, const.FullBackup, self.options)
b.run()
last_start = b.start_time
# Assert that the store is still of an appropriate size
size, used, avail = self.store.current_usage()
self.assertTrue(avail >= 0)
self.assertTrue(used <= max_size)
# Confirm that's true on disk
disksize = utils.du(self.store.root)
self.assertTrue(disksize <= max_size)
# Check that some runs have actually been deleted
runs = self.db.runs(self.backup.name + "0")
self.assertTrue(len(runs) == 0)
runs = self.db.runs(self.backup.name + "1")
self.assertTrue(len(runs) == 0)
def testAutoManagementOfStore2(self):
# Run one backup multiple times to overload a store
max_size, dummy, dummy = self.store.limit_details()
filesize = utils.du(self.backup.include_folders[0])
# Lets make sure we are going to do enough backups that
# the older ones will be removed.
RunCount = (max_size // filesize) + 2
last_start = None
for cycle in xrange(RunCount):
if last_start:
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - last_start < timedelta(seconds=1):
time.sleep(0.01)
# Run a full backup
b = Run(self.backup.name, const.FullBackup, self.options)
b.run()
last_start = b.start_time
# Assert that the store is still of an appropriate size
size, used, avail = self.store.current_usage()
self.assertTrue(avail >= 0)
self.assertTrue(used <= max_size)
# Confirm that's true on disk
disksize = utils.du(self.store.root)
self.assertTrue(disksize <= max_size)
# Check that some runs have actually been deleted
runs = self.db.runs(self.backup.name)
self.assertTrue(len(runs) < RunCount)
def testChanges(self):
pass
# Full Backup
# change a file
# Incremental backup
# Restore most recent. ensure you get latest file
# Restore to just prior to incremental, ensure you get earlier file
# Run a full backup
file = os.path.join(self.files_folder, "changer")
restore_file = os.path.join(self.restore_folder, file[1:])
# t=0 - file does not exist
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - b.start_time < timedelta(seconds=1):
time.sleep(0.01)
# t=1 - file exists
with open(file, "w") as f:
f.write("1")
b = Run("testbackup", const.IncrBackup, self.options)
b.run()
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - b.start_time < timedelta(seconds=1):
time.sleep(0.01)
# t=2 - file changed
with open(file, "w") as f:
f.write("2")
b = Run("testbackup", const.IncrBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
t0 = runs[0].start_time
t1 = runs[1].start_time
t2 = runs[2].start_time
for t, exists, contents in [(t0, False, None), (t1, True, "1"), (t2, True, "2"), (None, True, "2")]:
# Attempt to restore most recent of ALL files
# This tests the default restore.
r = Restore(self.restore_folder, [self.files_folder], t, self.options)
r.run()
if exists:
with open(restore_file, "r") as f:
self.assertEqual(f.read(), contents)
else:
self.assertFalse(os.path.exists(restore_file))
# clean
shutil.rmtree(self.restore_folder)
utils.makedirs(self.restore_folder)
def test7bitFilenames(self):
# Make some 7 bit filenames
strange_folder = os.path.join(self.files_folder, "strange")
utils.makedirs(strange_folder)
for i in xrange(1, 117, 10):
name = "".join([chr(j) for j in xrange(i, i + 10) if chr(j) != "/"])
path = os.path.join(strange_folder, name)
with open(path, "w") as f:
f.write(os.urandom(100))
self.backup_restore_compare()
def testUnicodeFilenames(self):
# Make some unicode bit filenames
# Clean out the ordinary files
shutil.rmtree(self.files_folder)
utils.makedirs(self.files_folder)
unicode_folder = os.path.join(unicode(self.files_folder), u"unicode")
utils.makedirs(unicode_folder)
for i in xrange(1000, 1200, 10):
name = u"".join([unichr(j) for j in xrange(i, i + 10) if unichr(j) != u"/"])
path = os.path.join(unicode_folder, name)
with open(path, "w") as f:
f.write(os.urandom(10))
self.backup_restore_compare()
def backup_restore_compare(self):
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - b.start_time < timedelta(seconds=1):
time.sleep(0.01)
# Attempt to restore every file
r = Restore(self.restore_folder, [self.files_folder],
datetime.now(), self.options)
r.run()
# Check that the restored folder and original folder are identical
left = unicode(self.files_folder)
right = unicode(os.path.join(self.restore_folder, self.files_folder[1:]))
d = utils.dircmp(left, right)
self.assertEqual(d.left_only, set())
self.assertEqual(d.right_only, set())
self.assertEqual(d.diff_files, set())
self.assertTrue(len(d.same_files) > 0)
# Check that all files are in the DB
for folder, _, local_files in os.walk(self.files_folder):
for file in local_files:
path = os.path.join(file, folder)
# This will raise an exception if it does not exist
self.db.select_path(path, build=False)
############################################################################
#
# Utility Routines
#
############################################################################
def mark_db_ids(self):
self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
if self.max_fs_id is None:
self.max_fs_id = 0
self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
if self.max_version_id is None:
self.max_version_id = 0
self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
if self.max_run_id is None:
self.max_run_id = 0
self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
if self.max_message_id is None:
self.max_message_id = 0
def clean_db(self):
self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
class VerifyTestCase(unittest.TestCase):
def setUp(self):
self.config = Config.get_config()
self.db = DB()
self.db.check_upgrade()
self.mark_db_ids()
self.test_folder = tempfile.mkdtemp()
self.files_folder = os.path.join(self.test_folder, "files")
self.store_folder = os.path.join(self.test_folder, "store")
self.restore_folder = os.path.join(self.test_folder, "restore")
utils.makedirs(self.files_folder)
utils.makedirs(self.store_folder)
utils.makedirs(self.restore_folder)
utils.build_file_structure(self.files_folder, 50 * const.Kilobyte, 500 * const.Kilobyte)
# Build a store object (dont save config)
# Note the careful size selection - we want backups to overflow the FolderStore.
self.store = FolderStore("teststore", "2MB", True, self.store_folder)
self.config.storage[self.store.name] = self.store
# Build the backup object (dont save config)
self.backup = Backup("testbackup")
self.backup.include_folders = [self.files_folder]
self.backup.store = self.store.name
self.backup.notify_msg = False
self.include_packages = True
self.config.backups[self.backup.name] = self.backup
# build an options object for use with the backup
self.options = BlankClass()
self.options.dry_run = False
self.options.message = False
self.options.email = False
self.options.shutdown = False
self.options.norecurse = False
self.old_pass = self.config.data_passphrase
self.config.data_passphrase = "banana"
def tearDown(self):
self.config.data_passphrase = self.old_pass
# Remove all DB records created during this test
self.clean_db()
shutil.rmtree(self.test_folder)
self.assertFalse(os.path.isdir(self.test_folder))
def testVerify(self):
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
run = runs[0]
v = Verify("testbackup", run.start_time)
self.assertTrue(v.run())
def testBadVerify(self):
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
run = runs[0]
# Get the location of the data file from the streamer
streamer = StreamOut(None, self.store, b.backup_folder)
datafile = os.path.join(self.store.root, streamer.get_path(0))
size = os.path.getsize(datafile)
# Now corrupt the data file a little
with open(datafile, "r+b") as f:
f.seek(size // 2, 0)
f.write("X")
v = Verify("testbackup", run.start_time)
self.assertRaises(Exception, v.run)
def testBadConfig(self):
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
run = runs[0]
# Delete The Config File
configfile = os.path.join(run.folder, const.ConfigName)
self.store.remove_file(configfile)
v = Verify("testbackup", run.start_time)
self.assertRaises(Exception, v.run)
def testBadVerifyEncrypted(self):
backup = self.config.backups[self.backup.name]
backup.encrypt = True
self.config.backups[backup.name] = backup
# Run a full backup
b = Run("testbackup", const.FullBackup, self.options)
b.run()
# Get the times
runs = self.db.runs("testbackup")
run = runs[0]
# Get the location of the data file from the streamer
streamer = StreamOut(None, self.store, b.backup_folder)
datafile = os.path.join(self.store.root, streamer.get_path(0))
# Now corrupt the data file a little
size = os.path.getsize(datafile)
with open(datafile, "r+b") as f:
f.seek(size // 2, 0)
f.write("X")
v = Verify("testbackup", run.start_time)
self.assertRaises(Exception, v.run)
############################################################################
#
# Utility Routines
#
############################################################################
def mark_db_ids(self):
self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
if self.max_fs_id is None:
self.max_fs_id = 0
self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
if self.max_version_id is None:
self.max_version_id = 0
self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
if self.max_run_id is None:
self.max_run_id = 0
self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
if self.max_message_id is None:
self.max_message_id = 0
def clean_db(self):
self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))
def wiz_execute(wiz):
# Connect
if wiz.fields["storagetype"].value == _("Local Folder"):
store = FolderStore("store", 0, False, wiz.fields["folderpath"].value)
elif wiz.fields["storagetype"].value == _("FTP Server"):
server = wiz.fields["ftpserver"].value
login = wiz.fields["ftplogin"].value
password = wiz.fields["ftppassword"].value
root = wiz.fields["ftproot"].value
sftp = wiz.fields["sftp"].value
store = FTPStore("store", 0, False, server, root, login, password, sftp)
elif wiz.fields["storagetype"].value == _("DropBox"):
login = wiz.fields["dblogin"].value
password = wiz.fields["dbpassword"].value
root = wiz.fields["dbroot"].value
store = DropBoxStore("store", 0, False, root, login, password)
elif wiz.fields["storagetype"].value == _("Server Share"):
mountcmd = wiz.fields["mountcmd"].value
umountcmd = wiz.fields["umountcmd"].value
shareroot = wiz.fields["shareroot"].value
store = ShareStore("store", 0, False, shareroot, mountcmd, umountcmd)
else:
raise Exception("Internal error: bad store type")
log.debug("Store = ", store)
# In case we need to revert our config later
orig_config = Config.get_config()
try:
store.connect()
folders = []
# List all backups
backups = store.list(".")
for backup in backups:
# Make sure its not a system folder
if backup[0] != "_":
try:
runs = store.list(backup + "/")
for run in runs:
folders.append((backup, run))
except:
pass
log.debug("Folders", folders)
if len(folders) == 0:
raise Exception(_("There are no backup runs in this store"))
# Sort them based on the path name
folders.sort(key=lambda item: item[1], reverse=True)
# Now the first item is the one to use (as it is the most recent).
backupname, backuprun = folders[0]
config = os.path.join(backupname, backuprun, const.ConfigName)
configenc = config + const.EncryptionSuffix
# Check if the config exists.
if store.exists(config):
src = config
encrypted = False
elif store.exists(configenc):
src = configenc
encrypted = True
else:
raise Exception(_("The backup runs are missing or corrupt (no config files)"))
if not encrypted:
store.get(src, const.ConfigDir+os.sep)
else:
# Fetch the file.
enc_file = const.ConfigFile + const.EncryptionSuffix
clear_file = const.ConfigFile
store.get(src, const.ConfigDir+os.sep)
# ENCRYPTED
bad = True # keep going until we get a good password
while bad:
password = GetPassword(wiz, backupname, backuprun)
if password == None:
# User is quitting
return
# Lets check the file is good.
try:
log.debug("Decrypting", enc_file, clear_file, password)
ret = cryptor.decrypt_file(password, enc_file, clear_file)
log.debug("Return from decrypt", ret)
if ret != 0:
raise Exception(_("Failed encryption"))
bad = False
if os.path.exists(enc_file):
os.remove(enc_file)
# Looks like this password is a good one.
except:
log.info("Invalid backup password")
dlg.Warn(wiz, _("Invalid password. Please enter the correct backup password."))
os.remove(clear_file)
# Revert the old config
orig_config.save()
dlg.Info(wiz, _("Your configuration has been restored.\nClick OK to restart the UI..."), _("Restore"))
python = sys.executable
log.debug("Starting:", const.UIProgram)
subprocess.Popen(const.UIProgram)
app.quit()
except Exception as e:
dlg.Warn(wiz, str(e), _("Configuration restore failed"))
finally:
store.disconnect()
