def __init__(self, name, limit, auto_manage): """ @param name: Name of this store @param limit: Either a blank string (no limit) OR [0-9]+(MG|GB|TB) @auto_manage: should the space on this store be auto-managed? That means the space will be limited to "limit", and old runs deleted as required. """ # Validate the data self.check_space() name = name.strip() if len(name) == 0: raise Exception("Name cannot be empty or consist of only blanks") self.name = name self.limit = limit self.auto_manage = auto_manage (total, _, _) = self.limit_details() # If we are auto-managing, then the store must be big enough to be usable. # Otherwise we just assume the store is infinite and let the user manually manage. if self.auto_manage and total < const.MinStoreSize: raise Exception("Store size must larger than " + utils.readable_form(const.MinStoreSize)) # These fields are all we are saving self._persistent = ["limit", "name", "auto_manage"] self._db = None self.connected = False self.queue = Queue(maxsize=const.QueueSize) self.io_worker = None # For testing... causes a queued xmit to fail. self.debug_fail = False
def build_config(self): log.trace("build_config") #store1 = FTPStore("teststore1", "4MB", True, "localhost", "store1", "ftpuser", "ftpuserX9", False) #store2 = FTPStore("teststore2", "4MB", True, "localhost", "store2", "ftpuser", "ftpuserX9", False) if self.options.store: store1 = self.config.storage[self.options.store].copy() store2 = store1 else: # Make the store about 3x the options size s, dummy, dummy = utils.from_readable_form(self.options.size) store_size = utils.readable_form(s * 3) store1 = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1")) store2 = FolderStore("teststore2", store_size, True, os.path.join(self.store_folder, "teststore2")) self.config.storage[store1.name] = store1 self.config.storage[store2.name] = store2 backup1 = Backup("testbackup1") backup1.include_folders = [self.files_folder] backup1.include_packages = True backup1.exclude_types = ["Music"] backup1.exclude_patterns = [] backup1.store = store1.name backup1.notify_msg = False self.config.backups[backup1.name] = backup1 backup2 = Backup("testbackup2") backup2.include_folders = [self.files_folder] backup2.include_packages = True backup2.exclude_patterns = [] backup1.exclude_types = ["Videos", "Programs"] backup2.store = store2.name backup2.notify_msg = False self.config.backups[backup2.name] = backup2
def onSavePassword(self, event): pwd = self.txtMasterPassword.GetValue() if pwd != self.config.data_passphrase: # Password has changed. Do we have any stored backups? # If so, they should be deleted. runs = self.db.runs() num_runs = len(runs) if num_runs > 0: size = 0 for run in runs: size += run.size # Check with the user. msg = _("You current have {numruns} backup runs stored, " \ "totalling {size} of remote data.\n" \ "Changing the Master Password means old encrypted backups cannot be used.\n" \ "Note that they can be kept for disaster recovery if needed,\n" \ "but we suggest you simply start fresh.").format(\ numruns=num_runs, size=utils.readable_form(size)) mbox = OptionDialog(self, msg, _("Delete Backup Runs"), _("Also delete all encrypted backup data stored remotely."), default=True) if mbox.ShowModal() != wx.ID_OK: return delete_offsite_data = mbox.chkOption.GetValue() # TODO skip if no runs # We keep track of all errors errors = "" with ProgressDialog(self, _("Deleting"), _("Deleting old encrypted backup data.\nPlease wait...")): for backup in self.config.backups.itervalues(): # If its encrypted if backup.encrypt: # If the option set - delete all offline data at the store if delete_offsite_data: try: # Get the list of unique stores used by runs of this backup runs = self.db.runs(backup.name) stores = set([r.store for r in runs]) # Get the store and delete all data. for storename in stores: store = self.config.storage[storename].copy() store.delete_backup_data(backup.name) except Exception as e: errors += "\nDelete offline data for %s failed: %s" % (backup.name, str(e)) # Now delete the database records of the run abd backup try: self.db.delete_backup(backup.name) except Exception as e: errors += "\nDelete local backup information for %s failed: " % (backup.name, str(e)) if len(errors) > 0: dlg.Error(self, errors) if not pwd: self.config.data_passphrase = None app.show_message('Password cleared') else: self.config.data_passphrase = pwd app.show_message('Password set') self.config.save()
def show_store(self, d): try: # General Information self.txtName.SetValue(d.name) self.lblName.SetLabel(d.name) # Storage Information. More specialised first. then more general if isinstance(d, ShareStore): self.nbStoreType.SetSelection(2) self.txtShareRoot.SetValue(d.root) self.txtShareMount.SetValue(d.mount) self.txtShareUMount.SetValue(d.umount) elif isinstance(d, FolderStore): self.nbStoreType.SetSelection(0) self.txtFolderPath.SetValue(d.root) elif isinstance(d, FTPStore): self.nbStoreType.SetSelection(1) self.txtFTPAddress.SetValue(d.ip) self.txtFTPRoot.SetValue(d.root) self.chkSFTP.SetValue(d.sftp) self.txtFTPLogin.SetValue(d.login) self.txtFTPPass.SetValue(d.password) elif isinstance(d, DropBoxStore): self.nbStoreType.SetSelection(3) self.txtDBRoot.SetValue(d.root) self.txtDBLogin.SetValue(d.login) self.txtDBPass.SetValue(d.password) self.txtDBKey.SetValue(d.app_key) self.txtDBSecretKey.SetValue(d.app_secret_key) elif isinstance(d, S3Store): self.nbStoreType.SetSelection(4) self.txtAmazonBucket.SetValue(d.bucket) self.txtAmazonKey.SetValue(d.key) self.txtAmazonSecretKey.SetValue(d.secret_key) else: raise Exception("Invalid store type") (dummy, num, units) = d.limit_details() self.txtLimitSize.SetValue(str(num)) self.cboLimitUnits.SetStringSelection(units) self.chkAutoManage.SetValue(d.auto_manage) self.onAutoManage(None) # Whats inside the store. use = self.db.store_usage(d.name) if not use or use.size == 0: self.lblContentDetails.SetLabel("Empty") else: self.lblContentDetails.SetLabel( _("Size {size}, Files {files}, Folders {folders}").format( size=utils.readable_form(use.size), files=utils.comma_int(use.nfiles), folders=utils.comma_int(use.nfolders))) self.update_state() except Exception as e: log.error("Error showing store:", str(e)) dlg.Error(self, _("Store {store} appears to be corrupt. Unable to show.").format(name=d.name))
def delete(self, name): use = self.db.store_usage(name) if use.size > 0: log.debug(name, utils.readable_form(use.size)) msg = _("Store '{store}' contains {size} of backups.\nAre you sure?").format( store=name, size=utils.readable_form(use.size)) mbox = OptionDialog(self, msg, _("Delete Store"), _("Also delete all backup data stored on the store.")) if mbox.ShowModal() == wx.ID_OK: with ProgressDialog(self, _("Deleting"), _("Deleting store %s.\nPlease wait. This can take a while..." % name)): self.delete_store(name, mbox.chkOption.GetValue()) self.clear() self.state = ViewState else: ret = dlg.OkCancel(self, _("Store '{store}' is not currently used. Delete?").format(store=name)) if ret == wx.ID_OK: with ProgressDialog(self, _("Deleting"), _("Deleting store %s.\nPlease wait. This can take a while..." % name)): self.delete_store(name, False) self.clear() self.state = ViewState app.broadcast_update()
def load_run_details(self): #(run_id=3, name=u'test', type=1, start_time=u'2010-12-28T15:09:39.455313', hash=u'6da92e0e7f25d003bfcc2cc7d845868fe6a433e1c0fe349300659fc86c8f66ba', size=59764, nfiles=0, status=u'Succeeded') self.lstDetails.InsertColumn(0, _("Name")) self.lstDetails.InsertColumn(1, _("Value")) self.lstDetails.Append([_("Backup Name"), self.run.name]) self.lstDetails.Append([_("Run Date"), self.run.start_time_str]) self.lstDetails.Append([_("Run Type"), self.run.type]) self.lstDetails.Append([_("Status"), self.run.status]) self.lstDetails.Append([_("Files Backed Up"), str(self.run.nfiles)]) self.lstDetails.Append([_("Folders Backed Up"), str(self.run.nfolders)]) self.lstDetails.Append([_("Installed Software List"), _("Included") if self.run.packages else _("Not included")]) self.lstDetails.Append([_("Total Size"), utils.readable_form(self.run.size)]) self.lstDetails.Append([_("Hash"), self.run.hash]) self.lstDetails.SetColumnWidth(0, wx.LIST_AUTOSIZE) self.lstDetails.SetColumnWidth(1, wx.LIST_AUTOSIZE)
def update_stores(self): ''' Update store detail display. Note that the size is gleaned from the sum of all runs, PLUS (because running runs have size 0) the sum of all files in a running run. ''' log.trace("update_stores") if len(self.config.storage) == 0: self.lstStores.Hide() else: self.lblNoStores.Hide() self.lstStores.DeleteAllColumns() self.lstStores.DeleteAllItems() self.lstStores.InsertColumn(0, _("Name")) self.lstStores.InsertColumn(1, _("Space")) self.lstStores.InsertColumn(2, _("Used")) # Includes runs that have completed. uses = self.db.store_usages() log.debug("Update Stores: uses=", uses) for sname, store in self.config.storage.iteritems(): if sname in uses: used = uses[sname].size else: used = 0 self.lstStores.Append([store.name, _("Unlimited") if not store.auto_manage else store.limit, utils.readable_form(used)]) self.lstStores.SetColumnWidth(0, wx.LIST_AUTOSIZE) self.lstStores.SetColumnWidth(1, wx.LIST_AUTOSIZE) self.lstStores.SetColumnWidth(2, wx.LIST_AUTOSIZE) log.trace("completed update_stores")
def load_files(self, limit): with ProgressDialog(self, _("Loading"), _("Loading run files.\nPlease wait...")): self.lstFiles.DeleteAllColumns() self.lstFiles.DeleteAllItems() self.lstFiles.InsertColumn(0, _("Path")) self.lstFiles.InsertColumn(1, _("Size")) self.lstFiles.InsertColumn(2, _("Mod Time")) self.lstFiles.Freeze() try: files = self.db.run_contents(self.run.run_id, limit) for file in files: wx.Yield() if file.type == "F": size = utils.readable_form(file.size) mod_time = file.mod_time elif file.type == 'D': size = _("Folder") mod_time = file.mod_time elif file.type == 'X': size = _("(deleted)") mod_time = "" else: size = "ERROR: Bad type" path = os.path.join(self.get_path(file.parent_id), file.name) item = (utils.display_escape(path), size, mod_time) self.lstFiles.Append(item) self.lstFiles.SetColumnWidth(0, wx.LIST_AUTOSIZE) self.lstFiles.SetColumnWidth(1, wx.LIST_AUTOSIZE) self.lstFiles.SetColumnWidth(2, wx.LIST_AUTOSIZE) finally: self.lstFiles.Thaw() if self.lstFiles.GetItemCount() < limit: # There probably aren't that many files. self.pnlAllFiles.Hide() self.pnlFiles.Fit()
def send_email(self, result, head): ''' Send a message to the appropriate users. If result is False (failure) then error message will contain the reason. @param result: @param error_message: ''' log.debug("send_email: ", result, head) if result: message_text = head + \ _("\n\nStatistics:\n {files} files backed up.\n " "{folders} folders backed up.\n {size} copied.\n" ).format(files=self.nfiles, folders=self.nfolders, size=utils.readable_form(self.bytes)) subject = _("Backup {server}/{backup}/{type} completed").format(server=utils.get_hostname(), backup=self.backup.name, type=self.type) else: message_text = head subject = _("Backup {server}/{backup}/{type} failed").format(server=utils.get_hostname(), backup=self.backup.name, type=self.type) if not self.options.dry_run: messages = " " + "\n ".join([message.time + " " + message.message for message in self.db.run_messages(self.run_id)]) message_text += _("\nBackup messages:\n") + messages else: message_text = "\n" log.debug("Starting mail send") try: sendemail.sendemail(subject, message_text) except Exception as e: msg = _("Unable to email results. {error}").format(error=str(e)) if not self.dry_run: self.db.save_message(msg) else: print(msg) log.trace("send_email completed")
def delete(self, name): # Lets get some statistics runs = self.db.runs(backupname=name) num_runs = len(runs) size = 0 for run in runs: size += run.size if num_runs > 0: msg = _("Backup '{backup}' has {numruns} runs stored, " \ "totalling {size} of remote data.\n" \ "Are you sure you want to delete the backup definition?\n" \ "(hint - its usually better to just deactivate the backup)").format(\ backup=name, numruns=num_runs, size=utils.readable_form(size)) mbox = OptionDialog(self, msg, _("Delete Backup Definition"), _("Also delete all backup data stored remotely\nNote that this cannot be undone.")) if mbox.ShowModal() != wx.ID_OK: return delete_offsite_data = mbox.chkOption.GetValue() else: msg = _("Backup '{backup}' has never run. Are you " \ "sure you want to delete the backup definition?").format(backup=name) if dlg.OkCancel(self, msg, _("Confirm Delete")) != wx.ID_OK: return delete_offsite_data = False with ProgressDialog(self, _("Deleting"), _("Deleting backup %s%s.\nPlease wait...") % (name, " and all offsite data" if delete_offsite_data else "")): self.delete_backup(name, delete_offsite_data) import time time.sleep(3) self.clear() self.state = ViewState app.broadcast_update()
def update_details(self): ''' Update backup detail display. ''' if len(self.config.backups) == 0: self.lstBackups.Hide() self.lblNoBackups.Show() else: self.lblNoBackups.Hide() self.lstBackups.Show() self.lstBackups.DeleteAllColumns() self.lstBackups.DeleteAllItems() self.lstBackups.InsertColumn(0, _("Name")) self.lstBackups.InsertColumn(1, _("State")) self.lstBackups.InsertColumn(2, _("Last Run")) runs = self.db.run_states() for bname, backup in self.config.backups.iteritems(): active = _("Active") if backup.active else _("Inactive") if bname in runs: run = runs[bname] if run.status == const.StatusFailed: msg = ("Ran %s, failed") % run.start_time_str elif run.status == const.StatusRunning: msg = _("Running now, started {time}").format(time=run.start_time_str) else: msg = _("Ran {time}, saved {files} files, compressed size {size}").format( time=run.start_time_str, files=run.nfiles, size=utils.readable_form(run.size)) # line = "%s: last run %s, backed up %d files, total size %s" % (bname, run.start_time_str, run.nfiles, utils.readable_form(run.size)) else: msg = _("Never run") self.lstBackups.Append([bname, active, msg]) self.lstBackups.SetColumnWidth(0, wx.LIST_AUTOSIZE) self.lstBackups.SetColumnWidth(1, wx.LIST_AUTOSIZE) self.lstBackups.SetColumnWidth(2, wx.LIST_AUTOSIZE) return
def check_space(self, bytes_written): """ Check if we have enough space to keep writing this run. If there isn't - we start deleting runs. Parameter is the amount of data written to the store, but not yet logged in the database. In the future: a) keep a cache of current usage. It only changes at the completion of a run b) Have a start-run and end-run call in the store, so we know when to refetch from the DB c) Then always use the cache. """ if not self.auto_manage: return # Needs to VERY quickly check that we have enough space in the store # We target keeping good headroom at all times, so we have plenty of space # for the next buffer write. size, used, avail = self.current_usage() log.debug("CheckSpace size %d used %d avail %d total_bytes %d" % (size, used, avail, bytes_written)) # Check if the amount recorded as used in the DB, plus the amount currently being # written, is bigger than allowed free space runs = self.db.store_runs(self.name) # Remove any that are still running (we dont want to remove their space!) # Note that we include FAILED runs - therefore they will eventually be removed. # They dont take any space since their space has already been removed on failure. runs = [run for run in runs if run.status != const.StatusRunning] # Note: we need to use a separate store connection for this action # as our store may be busy writing. store = None try: while len(runs) > 0 and avail - bytes_written < const.MinSpaceAvail: log.info( "CheckSpace: Running low! size %d, used %d, avail %d avail-total_bytes %d minspace %d" % (size, used, avail, avail - bytes_written, const.MinSpaceAvail) ) # Pick the oldest (the first). oldest_run = runs[0] del runs[0] if not store: # Create and connect only if required. store = self.copy() store.connect() self.db.delete_run(oldest_run.run_id) store.delete_run_data(oldest_run) # Now check again... size, used, avail = self.current_usage() finally: if store: store.disconnect() if avail - bytes_written < const.MinSpaceAvail: # Uh oh.... we have exceeded our usage in this run. CRASH AND BURN log.error("Out of space on store") raise StoreFullException( "Unable to free enough space for backup. Store too small. (size=%s)" % (utils.readable_form(size)) )
def update_runs(self): if self.cboBackup.GetSelection() == 0: runs = self.db.runs() else: backup_name = self.cboBackup.GetStringSelection() runs = self.db.runs(backupname=backup_name) if self.order == const.ASC: runs.sort(key=lambda x : x.start_time_str, reverse=False) self.txtOrder.SetLabel("Order: Oldest First") else: runs.sort(key=lambda x : x.start_time_str, reverse=True) self.txtOrder.SetLabel("Order: Newest First") self.lstRuns.DeleteAllColumns() self.lstRuns.DeleteAllItems() self.lstRuns.InsertColumn(0, _("Name"), wx.LIST_FORMAT_LEFT) self.lstRuns.InsertColumn(1, _("Type"), wx.LIST_FORMAT_CENTER) self.lstRuns.InsertColumn(2, _("Time"), wx.LIST_FORMAT_CENTER) self.lstRuns.InsertColumn(3, _("Status"), wx.LIST_FORMAT_CENTER) self.lstRuns.InsertColumn(4, _("Files"), wx.LIST_FORMAT_CENTER) self.lstRuns.InsertColumn(5, _("Folders"), wx.LIST_FORMAT_CENTER) self.lstRuns.InsertColumn(6, _("Size"), wx.LIST_FORMAT_CENTER) self.itemDataMap = {} idx = 0 for run in runs: row = [run.name, run.type, run.start_time_str, run.status, str(run.nfiles), str(run.nfolders), utils.readable_form(run.size)] self.lstRuns.Append(row) self.lstRuns.SetItemData(idx, run.run_id) self.itemDataMap[idx + 1] = row idx = idx + 1 self.itemIndexMap = self.itemDataMap.keys() self.lstRuns.SetColumnWidth(0, 100) self.lstRuns.SetColumnWidth(1, 50) self.lstRuns.SetColumnWidth(2, wx.LIST_AUTOSIZE) self.lstRuns.SetColumnWidth(3, 80) self.lstRuns.SetColumnWidth(4, 120) self.lstRuns.SetColumnWidth(5, 100) self.lstRuns.SetColumnWidth(6, wx.LIST_AUTOSIZE)