def train(args): source_vocab = Vocab(args.source, args.vocab) target_vocab = Vocab(args.target, args.vocab) att_encdec = ABED(args.vocab, args.hidden_size, args.maxout_hidden_size, args.embed_size) if args.use_gpu: att_encdec.to_gpu() if args.source_validation: if os.path.exists(PLOT_DIR) == False: os.mkdir(PLOT_DIR) fp_loss = open(PLOT_DIR + "loss", "w") fp_loss_val = open(PLOT_DIR + "loss_val", "w") opt = optimizers.AdaDelta(args.rho, args.eps) opt.setup(att_encdec) opt.add_hook(optimizer.WeightDecay(DECAY_COEFF)) opt.add_hook(optimizer.GradientClipping(CLIP_THR)) for epoch in xrange(args.epochs): print "--- epoch: %s/%s ---" % (epoch + 1, args.epochs) source_gen = word_list(args.source) target_gen = word_list(args.target) batch_gen = batch(sort(source_gen, target_gen, 100 * args.minibatch), args.minibatch) n = 0 total_loss = 0.0 for source_batch, target_batch in batch_gen: n += len(source_batch) source_batch = fill_batch_end(source_batch) target_batch = fill_batch_end(target_batch) hyp_batch, loss = forward(source_batch, target_batch, source_vocab, target_vocab, att_encdec, True, 0) total_loss += loss.data * len(source_batch) closed_test(source_batch, target_batch, hyp_batch) loss.backward() opt.update() print "[n=%s]" % (n) print "[total=%s]" % (n) prefix = args.model_path + '%s' % (epoch + 1) serializers.save_hdf5(prefix + '.attencdec', att_encdec) if args.source_validation: total_loss_val, n_val = validation_test(args, att_encdec, source_vocab, target_vocab) fp_loss.write("\t".join([str(epoch), str(total_loss / n) + "\n"])) fp_loss_val.write("\t".join( [str(epoch), str(total_loss_val / n_val) + "\n"])) fp_loss.flush() fp_loss_val.flush() hyp_params = att_encdec.get_hyper_params() Backup.dump(hyp_params, args.model_path + HPARAM_NAME) source_vocab.save(args.model_path + SRC_VOCAB_NAME) target_vocab.save(args.model_path + TAR_VOCAB_NAME) hyp_params = att_encdec.get_hyper_params() Backup.dump(hyp_params, args.model_path + HPARAM_NAME) source_vocab.save(args.model_path + SRC_VOCAB_NAME) target_vocab.save(args.model_path + TAR_VOCAB_NAME) if args.source_validation: fp_loss.close() fp_loss_val.close()
def restore(self, job, revision): """Restore backup based on job and revision.""" job_dict = self._get_job_dict(job) job_dict['name'] = job # start restore process backup = Backup(job_dict, self.db) backup.restore(revision)
def train(args): source_vocab = Vocab(args.source, args.vocab) target_vocab = Vocab(args.target, args.vocab) att_encdec = ABED(args.vocab, args.hidden_size, args.maxout_hidden_size, args.embed_size) if args.use_gpu: att_encdec.to_gpu() if args.source_validation: if os.path.exists(PLOT_DIR)==False: os.mkdir(PLOT_DIR) fp_loss = open(PLOT_DIR+"loss", "w") fp_loss_val = open(PLOT_DIR+"loss_val", "w") opt = optimizers.AdaDelta(args.rho, args.eps) opt.setup(att_encdec) opt.add_hook(optimizer.WeightDecay(DECAY_COEFF)) opt.add_hook(optimizer.GradientClipping(CLIP_THR)) for epoch in xrange(args.epochs): print "--- epoch: %s/%s ---"%(epoch+1, args.epochs) source_gen = word_list(args.source) target_gen = word_list(args.target) batch_gen = batch(sort(source_gen, target_gen, 100*args.minibatch), args.minibatch) n = 0 total_loss = 0.0 for source_batch, target_batch in batch_gen: n += len(source_batch) source_batch = fill_batch_end(source_batch) target_batch = fill_batch_end(target_batch) hyp_batch, loss = forward(source_batch, target_batch, source_vocab, target_vocab, att_encdec, True, 0) total_loss += loss.data*len(source_batch) closed_test(source_batch, target_batch, hyp_batch) loss.backward() opt.update() print "[n=%s]"%(n) print "[total=%s]"%(n) prefix = args.model_path + '%s'%(epoch+1) serializers.save_hdf5(prefix+'.attencdec', att_encdec) if args.source_validation: total_loss_val, n_val = validation_test(args, att_encdec, source_vocab, target_vocab) fp_loss.write("\t".join([str(epoch), str(total_loss/n)+"\n"])) fp_loss_val.write("\t".join([str(epoch), str(total_loss_val/n_val)+"\n"])) fp_loss.flush() fp_loss_val.flush() hyp_params = att_encdec.get_hyper_params() Backup.dump(hyp_params, args.model_path+HPARAM_NAME) source_vocab.save(args.model_path+SRC_VOCAB_NAME) target_vocab.save(args.model_path+TAR_VOCAB_NAME) hyp_params = att_encdec.get_hyper_params() Backup.dump(hyp_params, args.model_path+HPARAM_NAME) source_vocab.save(args.model_path+SRC_VOCAB_NAME) target_vocab.save(args.model_path+TAR_VOCAB_NAME) if args.source_validation: fp_loss.close() fp_loss_val.close()
def test(args): source_vocab = Vocab.load(args.model_path + SRC_VOCAB_NAME) target_vocab = Vocab.load(args.model_path + TAR_VOCAB_NAME) vocab_size, hidden_size, maxout_hidden_size, embed_size = Backup.load( args.model_path + HPARAM_NAME) att_encdec = ABED(vocab_size, hidden_size, maxout_hidden_size, embed_size) if args.use_gpu: att_encdec.to_gpu() serializers.load_hdf5(args.model_path + str(args.epochs) + '.attencdec', att_encdec) with open(args.output + str(args.epochs), 'w') as fp: source_gen = word_list(args.source) target_gen = word_list(args.target) batch_gen = batch(sort(source_gen, target_gen, 100 * args.minibatch), args.minibatch) for source_batch, target_batch in batch_gen: source_batch = fill_batch_end(source_batch) target_batch = fill_batch_end(target_batch) if args.beam_search: hyp_batch = forward_beam(source_batch, None, source_vocab, target_vocab, att_encdec, False, args.limit, args.beam_size) else: hyp_batch = forward(source_batch, None, source_vocab, target_vocab, att_encdec, False, args.limit) for i, hyp in enumerate(hyp_batch): hyp.append(END) hyp = hyp[:hyp.index(END)] show(source_batch[i], target_batch[i], hyp, "TEST") fwrite(source_batch[i], target_batch[i], hyp, fp)
def test(args): source_vocab = Vocab.load(args.model_path+SRC_VOCAB_NAME) target_vocab= Vocab.load(args.model_path+TAR_VOCAB_NAME) vocab_size, hidden_size, maxout_hidden_size, embed_size = Backup.load(args.model_path+HPARAM_NAME) att_encdec = ABED(vocab_size, hidden_size, maxout_hidden_size, embed_size) if args.use_gpu: att_encdec.to_gpu() serializers.load_hdf5(args.model_path+str(args.epochs)+'.attencdec', att_encdec) with open(args.output+str(args.epochs), 'w') as fp: source_gen = word_list(args.source) target_gen = word_list(args.target) batch_gen = batch(sort(source_gen, target_gen, 100*args.minibatch), args.minibatch) for source_batch, target_batch in batch_gen: source_batch = fill_batch_end(source_batch) target_batch = fill_batch_end(target_batch) if args.beam_search: hyp_batch = forward_beam(source_batch, None, source_vocab, target_vocab, att_encdec, False, args.limit, args.beam_size) else: hyp_batch = forward(source_batch, None, source_vocab, target_vocab, att_encdec, False, args.limit) for i, hyp in enumerate(hyp_batch): hyp.append(END) hyp = hyp[:hyp.index(END)] show(source_batch[i], target_batch[i], hyp, "TEST") fwrite(source_batch[i], target_batch[i], hyp, fp)
def testAutoManagementOfStore1(self): # Run a set of backups that will overload the store. # The automanaged store should continue to archive old backups as required. # Store space reclaimation happens across all backups (i.e. any run). # We should see older runs from the first backup disappear. max_size, dummy, dummy = self.store.limit_details() filesize = utils.du(self.backup.include_folders[0]) # Lets make sure we are going to do enough backups that # the older ones will be removed. RunCount = (max_size // filesize) + 2 last_start = None for cycle in xrange(RunCount): if last_start: # Make sure we have ticked to another second since the start of the last backup. while datetime.now() - last_start < timedelta(seconds=1): time.sleep(0.01) backup = Backup(self.backup.name + str(cycle)) backup.include_folders = self.backup.include_folders backup.store = self.backup.store backup.notify_msg = False self.config.backups[backup.name] = backup # Run a full backup b = Run(backup.name, const.FullBackup, self.options) b.run() last_start = b.start_time # Assert that the store is still of an appropriate size size, used, avail = self.store.current_usage() self.assertTrue(avail >= 0) self.assertTrue(used <= max_size) # Confirm that's true on disk disksize = utils.du(self.store.root) self.assertTrue(disksize <= max_size) # Check that some runs have actually been deleted runs = self.db.runs(self.backup.name + "0") self.assertTrue(len(runs) == 0) runs = self.db.runs(self.backup.name + "1") self.assertTrue(len(runs) == 0)
def run(self): logging.info("---------- Redis Backups Starting ----------") configfile = os.path.join(os.environ["STARTPATH"], "config.ini") conf = Config(configfile) serverList = conf.getServers() logging.debug("got server list: %s" % serverList) servers = {} for server in serverList: servers[server] = conf.getSectionDetails(server) awsconf = conf.getSectionDetails("aws") sensuconf = conf.getSectionDetails("sensu") logging.debug(servers) logging.debug(awsconf) logging.debug(sensuconf) while True: logging.debug("Backup process starting...") for server, value in servers.iteritems(): logging.debug(value) prefix = None if value["prefix"] == "none" else value["prefix"] # if value['prefix'] == 'none': # prefix = None # else: # prefix = value['prefix'] job = Backup( sensuconf=sensuconf, server_name=value["hostname"], port=int(value["port"]), save_directory=value["redis_save_dir"], dbFileName=value["redis_db_name"], aws=awsconf, prefix=prefix, ) job.run() time.sleep(3600)
def run(self): logging.info("---------- Redis Backups Starting ----------") configfile = os.path.join(os.environ['STARTPATH'], "config.ini") conf = Config(configfile) serverList = conf.getServers() logging.debug("got server list: %s" % serverList) servers = {} for server in serverList: servers[server] = conf.getSectionDetails(server) awsconf = conf.getSectionDetails("aws") sensuconf = conf.getSectionDetails("sensu") logging.debug(servers) logging.debug(awsconf) logging.debug(sensuconf) while True: logging.debug("Backup process starting...") for server, value in servers.iteritems(): logging.debug(value) prefix = None if value['prefix'] == 'none' else value['prefix'] # if value['prefix'] == 'none': # prefix = None # else: # prefix = value['prefix'] job = Backup(sensuconf=sensuconf, server_name=value['hostname'], port=int(value['port']), save_directory=value['redis_save_dir'], dbFileName=value['redis_db_name'], aws=awsconf, prefix=prefix) job.run() time.sleep(3600)
def _get_backup_list(self, job=None): """Get a list of backup jobs and execute the specific job every given interval.""" # sync list self.sync_file = os.path.join(self.conf_dir, 'sync') self.sync_list = ConfigParser.ConfigParser() self.sync_list.read(self.sync_file) # fallback to one single job if job: jobs = [job, ] else: jobs = self.job_list.sections() for job in jobs: interval = self.job_list.getint(job, 'interval') job_dict = self._get_job_dict(job) job_dict['name'] = job try: ls = parse(self.sync_list.get(job, 'last_sync')) # job is in interval range and is not currently running if (datetime.datetime.now() - ls) > \ datetime.timedelta(0, interval, 0) \ and lbs != 1: logging.info('[%s] Starting backup process' % job) backup = Backup(job_dict, self.db) backup.backup() self._set_sync_option( job, 'last_sync', datetime.datetime.now() ) except: logging.info('[%s] Starting backup process' % job) if not self.sync_list.has_section(job): self.sync_list.add_section(job) backup = Backup(job_dict, self.db) backup.backup() self._set_sync_option(job, 'last_sync', datetime.datetime.now())
def build_config(self): log.trace("build_config") #store1 = FTPStore("teststore1", "4MB", True, "localhost", "store1", "ftpuser", "ftpuserX9", False) #store2 = FTPStore("teststore2", "4MB", True, "localhost", "store2", "ftpuser", "ftpuserX9", False) if self.options.store: store1 = self.config.storage[self.options.store].copy() store2 = store1 else: # Make the store about 3x the options size s, dummy, dummy = utils.from_readable_form(self.options.size) store_size = utils.readable_form(s * 3) store1 = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1")) store2 = FolderStore("teststore2", store_size, True, os.path.join(self.store_folder, "teststore2")) self.config.storage[store1.name] = store1 self.config.storage[store2.name] = store2 backup1 = Backup("testbackup1") backup1.include_folders = [self.files_folder] backup1.include_packages = True backup1.exclude_types = ["Music"] backup1.exclude_patterns = [] backup1.store = store1.name backup1.notify_msg = False self.config.backups[backup1.name] = backup1 backup2 = Backup("testbackup2") backup2.include_folders = [self.files_folder] backup2.include_packages = True backup2.exclude_patterns = [] backup1.exclude_types = ["Videos", "Programs"] backup2.store = store2.name backup2.notify_msg = False self.config.backups[backup2.name] = backup2
def save(self): # BUILD THE BACKUP if len(self.txtName.GetValue()) == 0: raise Exception(_("Backup name cannot be blank")) if self.chkEncrypt.GetValue() and not self.config.data_passphrase: raise Exception(_("You cannot select encryption when the passphrase is blank (see Configuration page).")) if self.txtName.GetValue() == EmptyName: raise Exception(_("You need to provide a proper backup name")) try: # Create the new backup object b = Backup(self.txtName.GetValue()) # General Information b.active = self.chkActive.GetValue() # Folder Information b.include_folders = self.text_to_list(self.txtFolders.GetValue()) b.include_packages = self.chkPackages.GetValue() # Exclusions b.exclude_types = list(self.lstExcludeTypes.GetCheckedStrings()) # returns a tuple, convert to array b.exclude_patterns = self.text_to_list(self.txtExcludePatterns.GetValue()) # Destination b.store = self.cboStore.GetStringSelection() b.encrypt = self.chkEncrypt.GetValue() b.verify = self.chkVerify.GetValue() # Schedule if self.radSchedAdvanced.GetValue(): b.sched_type = "custom" b.sched_times = "%s\n%s" % (self.txtCronIncr.GetValue(), self.txtCronFull.GetValue()) else: if self.radSchedDailyWeekly.GetValue(): b.sched_type = "daily/weekly" time = self.cboTime1.GetStringSelection() day = self.cboDay1.GetStringSelection() elif self.radSchedDailyMonthly.GetValue(): b.sched_type = "daily/monthly" time = self.cboTime2.GetStringSelection() day = self.cboMonthDay2.GetStringSelection() elif self.radSchedHourlyWeekly.GetValue(): b.sched_type = "hourly/weekly" time = self.cboTime3.GetStringSelection() day = self.cboDay3.GetStringSelection() elif self.radSchedNoneDaily.GetValue(): b.sched_type = "none/daily" time = self.cboTime4.GetStringSelection() day = "*" elif self.radSchedNoneWeekly.GetValue(): b.sched_type = "none/weekly" time = self.cboTime5.GetStringSelection() day = self.cboDay5.GetStringSelection() else: raise Exception(_("Corrupt backup")) b.sched_times = time + "/" + day # Notifications b.notify_msg = self.chkNotifyMsg.GetValue() b.notify_email = self.chkNotifyEmail.GetValue() b.shutdown_after = self.chkShutdown.GetValue() b.check() except Exception as e: raise e if self.state == ViewState: # Delete the old name oldname = self.lstItems.GetStringSelection() try: del self.config.backups[oldname] except: pass self.config.backups[b.name] = b self.config.save() self.update_backup_list() # Attempt to save the crontab. If this fails, the backup was corrupt. # But it has been saved. So that is a problem update_crontab(self.config.backups)
if not os.path.isfile('config/global.ini'): raise FileNotFoundError('config/global.ini') config = configparser.ConfigParser() config.read('config/global.ini') servers = config['global']['proxmox_servers'].replace(' ', '').split(',') if is_list_empty(servers): raise RuntimeError('no servers found in config') log.set_loglevel(map_loglevel(config['global']['log_level'])) log.debug(f'CLI args: {vars(args)}') try: if args.action == 'backup': backup = Backup(servers, config) if args.action_backup == 'run': if os.path.isfile('/tmp/proxmox-rbd-backup.lock'): print('There is already an instance running, abort', file=sys.stderr, flush=True) exit(1) backup.init_proxmox() vms_uuid = args.vm_uuid vms_id = args.vm_id vm_name_match = args.vm_name snapshot_name_prefix = args.snapshot_name_prefix allow_using_any_existing_snapshot = args.allow_using_any_existing_snapshot if snapshot_name_prefix: backup.set_snapshot_name_prefix(snapshot_name_prefix) else:
def wiz_execute(wiz): log.debug("Executing backup creation wizard") config = Config.get_config() name = wiz.fields["name"].value path = wiz.fields["folderpath"].value excl = [] for typ in config.file_types.iterkeys(): if wiz.fields['excl-'+typ].value: excl.append(typ) store = wiz.fields["store"].value b = Backup(name) b.active = True b.include_folders = [path] b.include_packages = True b.exclude_types = excl b.store = store b.excrypt = False b.sched_type = "daily/weekly" b.sched_times = "19:00/Sun" b.verify = False b.notify_msg = True b.notify_email = False b.shutdown_after = False config.backups[name] = b config.save() update_crontab(config.backups) dlg.Info(wiz, _("Your backup has been successfully created.")) app.broadcast_update()
def build_config(conf): # This is called to build a default config object # It is used the first time this application is run. log.info("Building new config object") conf.version = 1 conf.file_types = {_("Images"): ["jpg", "jpeg", "raw", "cr2", "png", "gif", "dng", "tiff", "tif", "bmp", "svg", "ppm", "psd"], _("Videos"): ["avi", "mpg", "mpeg", "mp4", "mov", "m4v", "vob", "wmv", "flv"], _("Music"): ["mp3", "aac", "ogg", "flac", "wav", "wma", "mpa"], _("Programs"): ["bin", "dll", "exe", "com", "lib"] } conf.storage = {} store = FolderStore(_("System Backup Folder"), "", False, os.path.join(const.DataDir, "vault-store")) conf.storage[store.name] = store # By default, the data passphrase is blank. conf.data_passphrase = None conf.backups = {} b = Backup(_("Home")) b.include_folders = ["/home"] b.active = False b.include_packages = True b.store = store.name b.notify_msg = True conf.backups[b.name] = b conf.mail_server = "" conf.mail_port = 25 conf.mail_ssl = False conf.mail_auth = False conf.mail_login = "" conf.mail_password = "" conf.mail_from = "" conf.mail_to = "" if const.Debug: # If we are debugging, we create debug store and backup objects store = FolderStore("TestStore", "20MB", True, os.path.join(const.RunDir, "store")) conf.storage[store.name] = store b = Backup("test") b.include_folders = [os.path.join(const.RunDir, "files")] b.include_packages = True b.exclude_types = ["Videos", "Programs"] b.exclude_patterns = ["*/not"] b.store = store.name b.notify_msg = True b.encrypt = True conf.backups[b.name] = b # DEBUG - reset the store and include folders conf.backups["Home"].store = store.name conf.backups["Home"].include_folders = [os.path.join(const.RunDir, "files")]
# (/mnt/linode_backups/mysql.the-jci.org/) and places it at /home/backup_dir/prepare/. # # This is meant to be used when you need to grab data from an incremental backup from lib.config_helper import Config_helper from lib.log_helper import Log_helper from lib.preparer import Preparer from lib.backup import Backup import sys host = "localhost" config_helper = Config_helper(host=host) logger = Log_helper(host, log_name="%s_logger" % host) logger.setup() logger.info_message("######### STARTING PREPARE PROCESS #########") host_backup = Backup(host, logger=logger) backup_dir = host_backup.get_latest_backup_dir_name() logger.info_message("Preparing backup dir %s" % backup_dir) prepare_obj = Preparer(host=host, backup_type=Backup.BACKUP_TYPE_INC, backup_dir=backup_dir, prepare_dir=config_helper.get_prepare_dir(), logger=logger) prepare_obj.setup() prepare_dir = prepare_obj.prepare() logger.info_message("Prepare dir %s" % prepare_dir)