def __init__(self, name, limit, auto_manage):
"""
@param name: Name of this store
@param limit: Either a blank string (no limit) OR
[0-9]+(MG|GB|TB)
@auto_manage: should the space on this store be auto-managed?
That means the space will be limited to "limit", and old
runs deleted as required.
"""
# Validate the data self.check_space()
name = name.strip()
if len(name) == 0:
raise Exception("Name cannot be empty or consist of only blanks")
self.name = name
self.limit = limit
self.auto_manage = auto_manage
(total, _, _) = self.limit_details()
# If we are auto-managing, then the store must be big enough to be usable.
# Otherwise we just assume the store is infinite and let the user manually manage.
if self.auto_manage and total < const.MinStoreSize:
raise Exception("Store size must larger than " + utils.readable_form(const.MinStoreSize))
# These fields are all we are saving
self._persistent = ["limit", "name", "auto_manage"]
self._db = None
self.connected = False
self.queue = Queue(maxsize=const.QueueSize)
self.io_worker = None
# For testing... causes a queued xmit to fail.
self.debug_fail = False
def build_config(self):
log.trace("build_config")
#store1 = FTPStore("teststore1", "4MB", True, "localhost", "store1", "ftpuser", "ftpuserX9", False)
#store2 = FTPStore("teststore2", "4MB", True, "localhost", "store2", "ftpuser", "ftpuserX9", False)
if self.options.store:
store1 = self.config.storage[self.options.store].copy()
store2 = store1
else:
# Make the store about 3x the options size
s, dummy, dummy = utils.from_readable_form(self.options.size)
store_size = utils.readable_form(s * 3)
store1 = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1"))
store2 = FolderStore("teststore2", store_size, True, os.path.join(self.store_folder, "teststore2"))
self.config.storage[store1.name] = store1
self.config.storage[store2.name] = store2
backup1 = Backup("testbackup1")
backup1.include_folders = [self.files_folder]
backup1.include_packages = True
backup1.exclude_types = ["Music"]
backup1.exclude_patterns = []
backup1.store = store1.name
backup1.notify_msg = False
self.config.backups[backup1.name] = backup1
backup2 = Backup("testbackup2")
backup2.include_folders = [self.files_folder]
backup2.include_packages = True
backup2.exclude_patterns = []
backup1.exclude_types = ["Videos", "Programs"]
backup2.store = store2.name
backup2.notify_msg = False
self.config.backups[backup2.name] = backup2
def onSavePassword(self, event):
pwd = self.txtMasterPassword.GetValue()
if pwd != self.config.data_passphrase:
# Password has changed. Do we have any stored backups?
# If so, they should be deleted.
runs = self.db.runs()
num_runs = len(runs)
if num_runs > 0:
size = 0
for run in runs:
size += run.size
# Check with the user.
msg = _("You current have {numruns} backup runs stored, " \
"totalling {size} of remote data.\n" \
"Changing the Master Password means old encrypted backups cannot be used.\n" \
"Note that they can be kept for disaster recovery if needed,\n" \
"but we suggest you simply start fresh.").format(\
numruns=num_runs, size=utils.readable_form(size))
mbox = OptionDialog(self, msg, _("Delete Backup Runs"),
_("Also delete all encrypted backup data stored remotely."),
default=True)
if mbox.ShowModal() != wx.ID_OK:
return
delete_offsite_data = mbox.chkOption.GetValue()
# TODO skip if no runs
# We keep track of all errors
errors = ""
with ProgressDialog(self, _("Deleting"), _("Deleting old encrypted backup data.\nPlease wait...")):
for backup in self.config.backups.itervalues():
# If its encrypted
if backup.encrypt:
# If the option set - delete all offline data at the store
if delete_offsite_data:
try:
# Get the list of unique stores used by runs of this backup
runs = self.db.runs(backup.name)
stores = set([r.store for r in runs])
# Get the store and delete all data.
for storename in stores:
store = self.config.storage[storename].copy()
store.delete_backup_data(backup.name)
except Exception as e:
errors += "\nDelete offline data for %s failed: %s" % (backup.name, str(e))
# Now delete the database records of the run abd backup
try:
self.db.delete_backup(backup.name)
except Exception as e:
errors += "\nDelete local backup information for %s failed: " % (backup.name, str(e))
if len(errors) > 0:
dlg.Error(self, errors)
if not pwd:
self.config.data_passphrase = None
app.show_message('Password cleared')
else:
self.config.data_passphrase = pwd
app.show_message('Password set')
self.config.save()
def show_store(self, d):
try:
# General Information
self.txtName.SetValue(d.name)
self.lblName.SetLabel(d.name)
# Storage Information. More specialised first. then more general
if isinstance(d, ShareStore):
self.nbStoreType.SetSelection(2)
self.txtShareRoot.SetValue(d.root)
self.txtShareMount.SetValue(d.mount)
self.txtShareUMount.SetValue(d.umount)
elif isinstance(d, FolderStore):
self.nbStoreType.SetSelection(0)
self.txtFolderPath.SetValue(d.root)
elif isinstance(d, FTPStore):
self.nbStoreType.SetSelection(1)
self.txtFTPAddress.SetValue(d.ip)
self.txtFTPRoot.SetValue(d.root)
self.chkSFTP.SetValue(d.sftp)
self.txtFTPLogin.SetValue(d.login)
self.txtFTPPass.SetValue(d.password)
elif isinstance(d, DropBoxStore):
self.nbStoreType.SetSelection(3)
self.txtDBRoot.SetValue(d.root)
self.txtDBLogin.SetValue(d.login)
self.txtDBPass.SetValue(d.password)
self.txtDBKey.SetValue(d.app_key)
self.txtDBSecretKey.SetValue(d.app_secret_key)
elif isinstance(d, S3Store):
self.nbStoreType.SetSelection(4)
self.txtAmazonBucket.SetValue(d.bucket)
self.txtAmazonKey.SetValue(d.key)
self.txtAmazonSecretKey.SetValue(d.secret_key)
else:
raise Exception("Invalid store type")
(dummy, num, units) = d.limit_details()
self.txtLimitSize.SetValue(str(num))
self.cboLimitUnits.SetStringSelection(units)
self.chkAutoManage.SetValue(d.auto_manage)
self.onAutoManage(None)
# Whats inside the store.
use = self.db.store_usage(d.name)
if not use or use.size == 0:
self.lblContentDetails.SetLabel("Empty")
else:
self.lblContentDetails.SetLabel(
_("Size {size}, Files {files}, Folders {folders}").format(
size=utils.readable_form(use.size), files=utils.comma_int(use.nfiles),
folders=utils.comma_int(use.nfolders)))
self.update_state()
except Exception as e:
log.error("Error showing store:", str(e))
dlg.Error(self, _("Store {store} appears to be corrupt. Unable to show.").format(name=d.name))
def delete(self, name):
use = self.db.store_usage(name)
if use.size > 0:
log.debug(name, utils.readable_form(use.size))
msg = _("Store '{store}' contains {size} of backups.\nAre you sure?").format(
store=name, size=utils.readable_form(use.size))
mbox = OptionDialog(self, msg, _("Delete Store"), _("Also delete all backup data stored on the store."))
if mbox.ShowModal() == wx.ID_OK:
with ProgressDialog(self, _("Deleting"), _("Deleting store %s.\nPlease wait. This can take a while..." % name)):
self.delete_store(name, mbox.chkOption.GetValue())
self.clear()
self.state = ViewState
else:
ret = dlg.OkCancel(self, _("Store '{store}' is not currently used. Delete?").format(store=name))
if ret == wx.ID_OK:
with ProgressDialog(self, _("Deleting"), _("Deleting store %s.\nPlease wait. This can take a while..." % name)):
self.delete_store(name, False)
self.clear()
self.state = ViewState
app.broadcast_update()
def load_run_details(self):
#(run_id=3, name=u'test', type=1, start_time=u'2010-12-28T15:09:39.455313', hash=u'6da92e0e7f25d003bfcc2cc7d845868fe6a433e1c0fe349300659fc86c8f66ba', size=59764, nfiles=0, status=u'Succeeded')
self.lstDetails.InsertColumn(0, _("Name"))
self.lstDetails.InsertColumn(1, _("Value"))
self.lstDetails.Append([_("Backup Name"), self.run.name])
self.lstDetails.Append([_("Run Date"), self.run.start_time_str])
self.lstDetails.Append([_("Run Type"), self.run.type])
self.lstDetails.Append([_("Status"), self.run.status])
self.lstDetails.Append([_("Files Backed Up"), str(self.run.nfiles)])
self.lstDetails.Append([_("Folders Backed Up"), str(self.run.nfolders)])
self.lstDetails.Append([_("Installed Software List"), _("Included") if self.run.packages else _("Not included")])
self.lstDetails.Append([_("Total Size"), utils.readable_form(self.run.size)])
self.lstDetails.Append([_("Hash"), self.run.hash])
self.lstDetails.SetColumnWidth(0, wx.LIST_AUTOSIZE)
self.lstDetails.SetColumnWidth(1, wx.LIST_AUTOSIZE)
def update_stores(self):
'''
Update store detail display.
Note that the size is gleaned from the sum of all runs,
PLUS (because running runs have size 0)
the sum of all files in a running run.
'''
log.trace("update_stores")
if len(self.config.storage) == 0:
self.lstStores.Hide()
else:
self.lblNoStores.Hide()
self.lstStores.DeleteAllColumns()
self.lstStores.DeleteAllItems()
self.lstStores.InsertColumn(0, _("Name"))
self.lstStores.InsertColumn(1, _("Space"))
self.lstStores.InsertColumn(2, _("Used"))
# Includes runs that have completed.
uses = self.db.store_usages()
log.debug("Update Stores: uses=", uses)
for sname, store in self.config.storage.iteritems():
if sname in uses:
used = uses[sname].size
else:
used = 0
self.lstStores.Append([store.name,
_("Unlimited") if not store.auto_manage else store.limit,
utils.readable_form(used)])
self.lstStores.SetColumnWidth(0, wx.LIST_AUTOSIZE)
self.lstStores.SetColumnWidth(1, wx.LIST_AUTOSIZE)
self.lstStores.SetColumnWidth(2, wx.LIST_AUTOSIZE)
log.trace("completed update_stores")
def load_files(self, limit):
with ProgressDialog(self, _("Loading"), _("Loading run files.\nPlease wait...")):
self.lstFiles.DeleteAllColumns()
self.lstFiles.DeleteAllItems()
self.lstFiles.InsertColumn(0, _("Path"))
self.lstFiles.InsertColumn(1, _("Size"))
self.lstFiles.InsertColumn(2, _("Mod Time"))
self.lstFiles.Freeze()
try:
files = self.db.run_contents(self.run.run_id, limit)
for file in files:
wx.Yield()
if file.type == "F":
size = utils.readable_form(file.size)
mod_time = file.mod_time
elif file.type == 'D':
size = _("Folder")
mod_time = file.mod_time
elif file.type == 'X':
size = _("(deleted)")
mod_time = ""
else:
size = "ERROR: Bad type"
path = os.path.join(self.get_path(file.parent_id), file.name)
item = (utils.display_escape(path), size, mod_time)
self.lstFiles.Append(item)
self.lstFiles.SetColumnWidth(0, wx.LIST_AUTOSIZE)
self.lstFiles.SetColumnWidth(1, wx.LIST_AUTOSIZE)
self.lstFiles.SetColumnWidth(2, wx.LIST_AUTOSIZE)
finally:
self.lstFiles.Thaw()
if self.lstFiles.GetItemCount() < limit:
# There probably aren't that many files.
self.pnlAllFiles.Hide()
self.pnlFiles.Fit()
def send_email(self, result, head):
'''
Send a message to the appropriate users.
If result is False (failure) then error message will contain the reason.
@param result:
@param error_message:
'''
log.debug("send_email: ", result, head)
if result:
message_text = head + \
_("\n\nStatistics:\n {files} files backed up.\n "
"{folders} folders backed up.\n {size} copied.\n"
).format(files=self.nfiles, folders=self.nfolders, size=utils.readable_form(self.bytes))
subject = _("Backup {server}/{backup}/{type} completed").format(server=utils.get_hostname(), backup=self.backup.name, type=self.type)
else:
message_text = head
subject = _("Backup {server}/{backup}/{type} failed").format(server=utils.get_hostname(), backup=self.backup.name, type=self.type)
if not self.options.dry_run:
messages = " " + "\n ".join([message.time + " " + message.message for message in self.db.run_messages(self.run_id)])
message_text += _("\nBackup messages:\n") + messages
else:
message_text = "\n"
log.debug("Starting mail send")
try:
sendemail.sendemail(subject, message_text)
except Exception as e:
msg = _("Unable to email results. {error}").format(error=str(e))
if not self.dry_run:
self.db.save_message(msg)
else:
print(msg)
log.trace("send_email completed")
def delete(self, name):
# Lets get some statistics
runs = self.db.runs(backupname=name)
num_runs = len(runs)
size = 0
for run in runs:
size += run.size
if num_runs > 0:
msg = _("Backup '{backup}' has {numruns} runs stored, " \
"totalling {size} of remote data.\n" \
"Are you sure you want to delete the backup definition?\n" \
"(hint - its usually better to just deactivate the backup)").format(\
backup=name, numruns=num_runs, size=utils.readable_form(size))
mbox = OptionDialog(self, msg, _("Delete Backup Definition"),
_("Also delete all backup data stored remotely\nNote that this cannot be undone."))
if mbox.ShowModal() != wx.ID_OK:
return
delete_offsite_data = mbox.chkOption.GetValue()
else:
msg = _("Backup '{backup}' has never run. Are you " \
"sure you want to delete the backup definition?").format(backup=name)
if dlg.OkCancel(self, msg, _("Confirm Delete")) != wx.ID_OK:
return
delete_offsite_data = False
with ProgressDialog(self, _("Deleting"),
_("Deleting backup %s%s.\nPlease wait...") %
(name, " and all offsite data" if delete_offsite_data else "")):
self.delete_backup(name, delete_offsite_data)
import time
time.sleep(3)
self.clear()
self.state = ViewState
app.broadcast_update()
def update_details(self):
'''
Update backup detail display.
'''
if len(self.config.backups) == 0:
self.lstBackups.Hide()
self.lblNoBackups.Show()
else:
self.lblNoBackups.Hide()
self.lstBackups.Show()
self.lstBackups.DeleteAllColumns()
self.lstBackups.DeleteAllItems()
self.lstBackups.InsertColumn(0, _("Name"))
self.lstBackups.InsertColumn(1, _("State"))
self.lstBackups.InsertColumn(2, _("Last Run"))
runs = self.db.run_states()
for bname, backup in self.config.backups.iteritems():
active = _("Active") if backup.active else _("Inactive")
if bname in runs:
run = runs[bname]
if run.status == const.StatusFailed:
msg = ("Ran %s, failed") % run.start_time_str
elif run.status == const.StatusRunning:
msg = _("Running now, started {time}").format(time=run.start_time_str)
else:
msg = _("Ran {time}, saved {files} files, compressed size {size}").format(
time=run.start_time_str, files=run.nfiles, size=utils.readable_form(run.size))
# line = "%s: last run %s, backed up %d files, total size %s" % (bname, run.start_time_str, run.nfiles, utils.readable_form(run.size))
else:
msg = _("Never run")
self.lstBackups.Append([bname, active, msg])
self.lstBackups.SetColumnWidth(0, wx.LIST_AUTOSIZE)
self.lstBackups.SetColumnWidth(1, wx.LIST_AUTOSIZE)
self.lstBackups.SetColumnWidth(2, wx.LIST_AUTOSIZE)
return
def check_space(self, bytes_written):
"""
Check if we have enough space to keep writing this run.
If there isn't - we start deleting runs.
Parameter is the amount of data written to the store, but not
yet logged in the database.
In the future:
a) keep a cache of current usage. It only changes at the completion of a run
b) Have a start-run and end-run call in the store, so we know when to refetch from the DB
c) Then always use the cache.
"""
if not self.auto_manage:
return
# Needs to VERY quickly check that we have enough space in the store
# We target keeping good headroom at all times, so we have plenty of space
# for the next buffer write.
size, used, avail = self.current_usage()
log.debug("CheckSpace size %d used %d avail %d total_bytes %d" % (size, used, avail, bytes_written))
# Check if the amount recorded as used in the DB, plus the amount currently being
# written, is bigger than allowed free space
runs = self.db.store_runs(self.name)
# Remove any that are still running (we dont want to remove their space!)
# Note that we include FAILED runs - therefore they will eventually be removed.
# They dont take any space since their space has already been removed on failure.
runs = [run for run in runs if run.status != const.StatusRunning]
# Note: we need to use a separate store connection for this action
# as our store may be busy writing.
store = None
try:
while len(runs) > 0 and avail - bytes_written < const.MinSpaceAvail:
log.info(
"CheckSpace: Running low! size %d, used %d, avail %d avail-total_bytes %d minspace %d"
% (size, used, avail, avail - bytes_written, const.MinSpaceAvail)
)
# Pick the oldest (the first).
oldest_run = runs[0]
del runs[0]
if not store:
# Create and connect only if required.
store = self.copy()
store.connect()
self.db.delete_run(oldest_run.run_id)
store.delete_run_data(oldest_run)
# Now check again...
size, used, avail = self.current_usage()
finally:
if store:
store.disconnect()
if avail - bytes_written < const.MinSpaceAvail:
# Uh oh.... we have exceeded our usage in this run. CRASH AND BURN
log.error("Out of space on store")
raise StoreFullException(
"Unable to free enough space for backup. Store too small. (size=%s)" % (utils.readable_form(size))
)
def update_runs(self):
if self.cboBackup.GetSelection() == 0:
runs = self.db.runs()
else:
backup_name = self.cboBackup.GetStringSelection()
runs = self.db.runs(backupname=backup_name)
if self.order == const.ASC:
runs.sort(key=lambda x : x.start_time_str, reverse=False)
self.txtOrder.SetLabel("Order: Oldest First")
else:
runs.sort(key=lambda x : x.start_time_str, reverse=True)
self.txtOrder.SetLabel("Order: Newest First")
self.lstRuns.DeleteAllColumns()
self.lstRuns.DeleteAllItems()
self.lstRuns.InsertColumn(0, _("Name"), wx.LIST_FORMAT_LEFT)
self.lstRuns.InsertColumn(1, _("Type"), wx.LIST_FORMAT_CENTER)
self.lstRuns.InsertColumn(2, _("Time"), wx.LIST_FORMAT_CENTER)
self.lstRuns.InsertColumn(3, _("Status"), wx.LIST_FORMAT_CENTER)
self.lstRuns.InsertColumn(4, _("Files"), wx.LIST_FORMAT_CENTER)
self.lstRuns.InsertColumn(5, _("Folders"), wx.LIST_FORMAT_CENTER)
self.lstRuns.InsertColumn(6, _("Size"), wx.LIST_FORMAT_CENTER)
self.itemDataMap = {}
idx = 0
for run in runs:
row = [run.name, run.type, run.start_time_str, run.status, str(run.nfiles), str(run.nfolders), utils.readable_form(run.size)]
self.lstRuns.Append(row)
self.lstRuns.SetItemData(idx, run.run_id)
self.itemDataMap[idx + 1] = row
idx = idx + 1
self.itemIndexMap = self.itemDataMap.keys()
self.lstRuns.SetColumnWidth(0, 100)
self.lstRuns.SetColumnWidth(1, 50)
self.lstRuns.SetColumnWidth(2, wx.LIST_AUTOSIZE)
self.lstRuns.SetColumnWidth(3, 80)
self.lstRuns.SetColumnWidth(4, 120)
self.lstRuns.SetColumnWidth(5, 100)
self.lstRuns.SetColumnWidth(6, wx.LIST_AUTOSIZE)
