def train(args):
source_vocab = Vocab(args.source, args.vocab)
target_vocab = Vocab(args.target, args.vocab)
att_encdec = ABED(args.vocab, args.hidden_size, args.maxout_hidden_size,
args.embed_size)
if args.use_gpu:
att_encdec.to_gpu()
if args.source_validation:
if os.path.exists(PLOT_DIR) == False: os.mkdir(PLOT_DIR)
fp_loss = open(PLOT_DIR + "loss", "w")
fp_loss_val = open(PLOT_DIR + "loss_val", "w")
opt = optimizers.AdaDelta(args.rho, args.eps)
opt.setup(att_encdec)
opt.add_hook(optimizer.WeightDecay(DECAY_COEFF))
opt.add_hook(optimizer.GradientClipping(CLIP_THR))
for epoch in xrange(args.epochs):
print "--- epoch: %s/%s ---" % (epoch + 1, args.epochs)
source_gen = word_list(args.source)
target_gen = word_list(args.target)
batch_gen = batch(sort(source_gen, target_gen, 100 * args.minibatch),
args.minibatch)
n = 0
total_loss = 0.0
for source_batch, target_batch in batch_gen:
n += len(source_batch)
source_batch = fill_batch_end(source_batch)
target_batch = fill_batch_end(target_batch)
hyp_batch, loss = forward(source_batch, target_batch, source_vocab,
target_vocab, att_encdec, True, 0)
total_loss += loss.data * len(source_batch)
closed_test(source_batch, target_batch, hyp_batch)
loss.backward()
opt.update()
print "[n=%s]" % (n)
print "[total=%s]" % (n)
prefix = args.model_path + '%s' % (epoch + 1)
serializers.save_hdf5(prefix + '.attencdec', att_encdec)
if args.source_validation:
total_loss_val, n_val = validation_test(args, att_encdec,
source_vocab, target_vocab)
fp_loss.write("\t".join([str(epoch), str(total_loss / n) + "\n"]))
fp_loss_val.write("\t".join(
[str(epoch), str(total_loss_val / n_val) + "\n"]))
fp_loss.flush()
fp_loss_val.flush()
hyp_params = att_encdec.get_hyper_params()
Backup.dump(hyp_params, args.model_path + HPARAM_NAME)
source_vocab.save(args.model_path + SRC_VOCAB_NAME)
target_vocab.save(args.model_path + TAR_VOCAB_NAME)
hyp_params = att_encdec.get_hyper_params()
Backup.dump(hyp_params, args.model_path + HPARAM_NAME)
source_vocab.save(args.model_path + SRC_VOCAB_NAME)
target_vocab.save(args.model_path + TAR_VOCAB_NAME)
if args.source_validation:
fp_loss.close()
fp_loss_val.close()
def restore(self, job, revision): """Restore backup based on job and revision.""" job_dict = self._get_job_dict(job) job_dict['name'] = job # start restore process backup = Backup(job_dict, self.db) backup.restore(revision)
def train(args):
source_vocab = Vocab(args.source, args.vocab)
target_vocab = Vocab(args.target, args.vocab)
att_encdec = ABED(args.vocab, args.hidden_size, args.maxout_hidden_size, args.embed_size)
if args.use_gpu:
att_encdec.to_gpu()
if args.source_validation:
if os.path.exists(PLOT_DIR)==False: os.mkdir(PLOT_DIR)
fp_loss = open(PLOT_DIR+"loss", "w")
fp_loss_val = open(PLOT_DIR+"loss_val", "w")
opt = optimizers.AdaDelta(args.rho, args.eps)
opt.setup(att_encdec)
opt.add_hook(optimizer.WeightDecay(DECAY_COEFF))
opt.add_hook(optimizer.GradientClipping(CLIP_THR))
for epoch in xrange(args.epochs):
print "--- epoch: %s/%s ---"%(epoch+1, args.epochs)
source_gen = word_list(args.source)
target_gen = word_list(args.target)
batch_gen = batch(sort(source_gen, target_gen, 100*args.minibatch), args.minibatch)
n = 0
total_loss = 0.0
for source_batch, target_batch in batch_gen:
n += len(source_batch)
source_batch = fill_batch_end(source_batch)
target_batch = fill_batch_end(target_batch)
hyp_batch, loss = forward(source_batch, target_batch, source_vocab, target_vocab, att_encdec, True, 0)
total_loss += loss.data*len(source_batch)
closed_test(source_batch, target_batch, hyp_batch)
loss.backward()
opt.update()
print "[n=%s]"%(n)
print "[total=%s]"%(n)
prefix = args.model_path + '%s'%(epoch+1)
serializers.save_hdf5(prefix+'.attencdec', att_encdec)
if args.source_validation:
total_loss_val, n_val = validation_test(args, att_encdec, source_vocab, target_vocab)
fp_loss.write("\t".join([str(epoch), str(total_loss/n)+"\n"]))
fp_loss_val.write("\t".join([str(epoch), str(total_loss_val/n_val)+"\n"]))
fp_loss.flush()
fp_loss_val.flush()
hyp_params = att_encdec.get_hyper_params()
Backup.dump(hyp_params, args.model_path+HPARAM_NAME)
source_vocab.save(args.model_path+SRC_VOCAB_NAME)
target_vocab.save(args.model_path+TAR_VOCAB_NAME)
hyp_params = att_encdec.get_hyper_params()
Backup.dump(hyp_params, args.model_path+HPARAM_NAME)
source_vocab.save(args.model_path+SRC_VOCAB_NAME)
target_vocab.save(args.model_path+TAR_VOCAB_NAME)
if args.source_validation:
fp_loss.close()
fp_loss_val.close()
def test(args):
source_vocab = Vocab.load(args.model_path + SRC_VOCAB_NAME)
target_vocab = Vocab.load(args.model_path + TAR_VOCAB_NAME)
vocab_size, hidden_size, maxout_hidden_size, embed_size = Backup.load(
args.model_path + HPARAM_NAME)
att_encdec = ABED(vocab_size, hidden_size, maxout_hidden_size, embed_size)
if args.use_gpu:
att_encdec.to_gpu()
serializers.load_hdf5(args.model_path + str(args.epochs) + '.attencdec',
att_encdec)
with open(args.output + str(args.epochs), 'w') as fp:
source_gen = word_list(args.source)
target_gen = word_list(args.target)
batch_gen = batch(sort(source_gen, target_gen, 100 * args.minibatch),
args.minibatch)
for source_batch, target_batch in batch_gen:
source_batch = fill_batch_end(source_batch)
target_batch = fill_batch_end(target_batch)
if args.beam_search:
hyp_batch = forward_beam(source_batch, None, source_vocab,
target_vocab, att_encdec, False,
args.limit, args.beam_size)
else:
hyp_batch = forward(source_batch, None, source_vocab,
target_vocab, att_encdec, False,
args.limit)
for i, hyp in enumerate(hyp_batch):
hyp.append(END)
hyp = hyp[:hyp.index(END)]
show(source_batch[i], target_batch[i], hyp, "TEST")
fwrite(source_batch[i], target_batch[i], hyp, fp)
def test(args):
source_vocab = Vocab.load(args.model_path+SRC_VOCAB_NAME)
target_vocab= Vocab.load(args.model_path+TAR_VOCAB_NAME)
vocab_size, hidden_size, maxout_hidden_size, embed_size = Backup.load(args.model_path+HPARAM_NAME)
att_encdec = ABED(vocab_size, hidden_size, maxout_hidden_size, embed_size)
if args.use_gpu:
att_encdec.to_gpu()
serializers.load_hdf5(args.model_path+str(args.epochs)+'.attencdec', att_encdec)
with open(args.output+str(args.epochs), 'w') as fp:
source_gen = word_list(args.source)
target_gen = word_list(args.target)
batch_gen = batch(sort(source_gen, target_gen, 100*args.minibatch), args.minibatch)
for source_batch, target_batch in batch_gen:
source_batch = fill_batch_end(source_batch)
target_batch = fill_batch_end(target_batch)
if args.beam_search:
hyp_batch = forward_beam(source_batch, None, source_vocab, target_vocab, att_encdec, False, args.limit, args.beam_size)
else:
hyp_batch = forward(source_batch, None, source_vocab, target_vocab, att_encdec, False, args.limit)
for i, hyp in enumerate(hyp_batch):
hyp.append(END)
hyp = hyp[:hyp.index(END)]
show(source_batch[i], target_batch[i], hyp, "TEST")
fwrite(source_batch[i], target_batch[i], hyp, fp)
def testAutoManagementOfStore1(self):
# Run a set of backups that will overload the store.
# The automanaged store should continue to archive old backups as required.
# Store space reclaimation happens across all backups (i.e. any run).
# We should see older runs from the first backup disappear.
max_size, dummy, dummy = self.store.limit_details()
filesize = utils.du(self.backup.include_folders[0])
# Lets make sure we are going to do enough backups that
# the older ones will be removed.
RunCount = (max_size // filesize) + 2
last_start = None
for cycle in xrange(RunCount):
if last_start:
# Make sure we have ticked to another second since the start of the last backup.
while datetime.now() - last_start < timedelta(seconds=1):
time.sleep(0.01)
backup = Backup(self.backup.name + str(cycle))
backup.include_folders = self.backup.include_folders
backup.store = self.backup.store
backup.notify_msg = False
self.config.backups[backup.name] = backup
# Run a full backup
b = Run(backup.name, const.FullBackup, self.options)
b.run()
last_start = b.start_time
# Assert that the store is still of an appropriate size
size, used, avail = self.store.current_usage()
self.assertTrue(avail >= 0)
self.assertTrue(used <= max_size)
# Confirm that's true on disk
disksize = utils.du(self.store.root)
self.assertTrue(disksize <= max_size)
# Check that some runs have actually been deleted
runs = self.db.runs(self.backup.name + "0")
self.assertTrue(len(runs) == 0)
runs = self.db.runs(self.backup.name + "1")
self.assertTrue(len(runs) == 0)
def run(self):
logging.info("---------- Redis Backups Starting ----------")
configfile = os.path.join(os.environ["STARTPATH"], "config.ini")
conf = Config(configfile)
serverList = conf.getServers()
logging.debug("got server list: %s" % serverList)
servers = {}
for server in serverList:
servers[server] = conf.getSectionDetails(server)
awsconf = conf.getSectionDetails("aws")
sensuconf = conf.getSectionDetails("sensu")
logging.debug(servers)
logging.debug(awsconf)
logging.debug(sensuconf)
while True:
logging.debug("Backup process starting...")
for server, value in servers.iteritems():
logging.debug(value)
prefix = None if value["prefix"] == "none" else value["prefix"]
# if value['prefix'] == 'none':
# prefix = None
# else:
# prefix = value['prefix']
job = Backup(
sensuconf=sensuconf,
server_name=value["hostname"],
port=int(value["port"]),
save_directory=value["redis_save_dir"],
dbFileName=value["redis_db_name"],
aws=awsconf,
prefix=prefix,
)
job.run()
time.sleep(3600)
def run(self):
logging.info("---------- Redis Backups Starting ----------")
configfile = os.path.join(os.environ['STARTPATH'], "config.ini")
conf = Config(configfile)
serverList = conf.getServers()
logging.debug("got server list: %s" % serverList)
servers = {}
for server in serverList:
servers[server] = conf.getSectionDetails(server)
awsconf = conf.getSectionDetails("aws")
sensuconf = conf.getSectionDetails("sensu")
logging.debug(servers)
logging.debug(awsconf)
logging.debug(sensuconf)
while True:
logging.debug("Backup process starting...")
for server, value in servers.iteritems():
logging.debug(value)
prefix = None if value['prefix'] == 'none' else value['prefix']
# if value['prefix'] == 'none':
# prefix = None
# else:
# prefix = value['prefix']
job = Backup(sensuconf=sensuconf,
server_name=value['hostname'],
port=int(value['port']),
save_directory=value['redis_save_dir'],
dbFileName=value['redis_db_name'],
aws=awsconf,
prefix=prefix)
job.run()
time.sleep(3600)
def _get_backup_list(self, job=None):
"""Get a list of backup jobs and execute the specific job every
given interval."""
# sync list
self.sync_file = os.path.join(self.conf_dir, 'sync')
self.sync_list = ConfigParser.ConfigParser()
self.sync_list.read(self.sync_file)
# fallback to one single job
if job:
jobs = [job, ]
else:
jobs = self.job_list.sections()
for job in jobs:
interval = self.job_list.getint(job, 'interval')
job_dict = self._get_job_dict(job)
job_dict['name'] = job
try:
ls = parse(self.sync_list.get(job, 'last_sync'))
# job is in interval range and is not currently running
if (datetime.datetime.now() - ls) > \
datetime.timedelta(0, interval, 0) \
and lbs != 1:
logging.info('[%s] Starting backup process' % job)
backup = Backup(job_dict, self.db)
backup.backup()
self._set_sync_option(
job, 'last_sync', datetime.datetime.now()
)
except:
logging.info('[%s] Starting backup process' % job)
if not self.sync_list.has_section(job):
self.sync_list.add_section(job)
backup = Backup(job_dict, self.db)
backup.backup()
self._set_sync_option(job, 'last_sync', datetime.datetime.now())
def build_config(self):
log.trace("build_config")
#store1 = FTPStore("teststore1", "4MB", True, "localhost", "store1", "ftpuser", "ftpuserX9", False)
#store2 = FTPStore("teststore2", "4MB", True, "localhost", "store2", "ftpuser", "ftpuserX9", False)
if self.options.store:
store1 = self.config.storage[self.options.store].copy()
store2 = store1
else:
# Make the store about 3x the options size
s, dummy, dummy = utils.from_readable_form(self.options.size)
store_size = utils.readable_form(s * 3)
store1 = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1"))
store2 = FolderStore("teststore2", store_size, True, os.path.join(self.store_folder, "teststore2"))
self.config.storage[store1.name] = store1
self.config.storage[store2.name] = store2
backup1 = Backup("testbackup1")
backup1.include_folders = [self.files_folder]
backup1.include_packages = True
backup1.exclude_types = ["Music"]
backup1.exclude_patterns = []
backup1.store = store1.name
backup1.notify_msg = False
self.config.backups[backup1.name] = backup1
backup2 = Backup("testbackup2")
backup2.include_folders = [self.files_folder]
backup2.include_packages = True
backup2.exclude_patterns = []
backup1.exclude_types = ["Videos", "Programs"]
backup2.store = store2.name
backup2.notify_msg = False
self.config.backups[backup2.name] = backup2
def save(self):
# BUILD THE BACKUP
if len(self.txtName.GetValue()) == 0:
raise Exception(_("Backup name cannot be blank"))
if self.chkEncrypt.GetValue() and not self.config.data_passphrase:
raise Exception(_("You cannot select encryption when the passphrase is blank (see Configuration page)."))
if self.txtName.GetValue() == EmptyName:
raise Exception(_("You need to provide a proper backup name"))
try:
# Create the new backup object
b = Backup(self.txtName.GetValue())
# General Information
b.active = self.chkActive.GetValue()
# Folder Information
b.include_folders = self.text_to_list(self.txtFolders.GetValue())
b.include_packages = self.chkPackages.GetValue()
# Exclusions
b.exclude_types = list(self.lstExcludeTypes.GetCheckedStrings()) # returns a tuple, convert to array
b.exclude_patterns = self.text_to_list(self.txtExcludePatterns.GetValue())
# Destination
b.store = self.cboStore.GetStringSelection()
b.encrypt = self.chkEncrypt.GetValue()
b.verify = self.chkVerify.GetValue()
# Schedule
if self.radSchedAdvanced.GetValue():
b.sched_type = "custom"
b.sched_times = "%s\n%s" % (self.txtCronIncr.GetValue(), self.txtCronFull.GetValue())
else:
if self.radSchedDailyWeekly.GetValue():
b.sched_type = "daily/weekly"
time = self.cboTime1.GetStringSelection()
day = self.cboDay1.GetStringSelection()
elif self.radSchedDailyMonthly.GetValue():
b.sched_type = "daily/monthly"
time = self.cboTime2.GetStringSelection()
day = self.cboMonthDay2.GetStringSelection()
elif self.radSchedHourlyWeekly.GetValue():
b.sched_type = "hourly/weekly"
time = self.cboTime3.GetStringSelection()
day = self.cboDay3.GetStringSelection()
elif self.radSchedNoneDaily.GetValue():
b.sched_type = "none/daily"
time = self.cboTime4.GetStringSelection()
day = "*"
elif self.radSchedNoneWeekly.GetValue():
b.sched_type = "none/weekly"
time = self.cboTime5.GetStringSelection()
day = self.cboDay5.GetStringSelection()
else:
raise Exception(_("Corrupt backup"))
b.sched_times = time + "/" + day
# Notifications
b.notify_msg = self.chkNotifyMsg.GetValue()
b.notify_email = self.chkNotifyEmail.GetValue()
b.shutdown_after = self.chkShutdown.GetValue()
b.check()
except Exception as e:
raise e
if self.state == ViewState:
# Delete the old name
oldname = self.lstItems.GetStringSelection()
try:
del self.config.backups[oldname]
except:
pass
self.config.backups[b.name] = b
self.config.save()
self.update_backup_list()
# Attempt to save the crontab. If this fails, the backup was corrupt.
# But it has been saved. So that is a problem
update_crontab(self.config.backups)
if not os.path.isfile('config/global.ini'):
raise FileNotFoundError('config/global.ini')
config = configparser.ConfigParser()
config.read('config/global.ini')
servers = config['global']['proxmox_servers'].replace(' ', '').split(',')
if is_list_empty(servers):
raise RuntimeError('no servers found in config')
log.set_loglevel(map_loglevel(config['global']['log_level']))
log.debug(f'CLI args: {vars(args)}')
try:
if args.action == 'backup':
backup = Backup(servers, config)
if args.action_backup == 'run':
if os.path.isfile('/tmp/proxmox-rbd-backup.lock'):
print('There is already an instance running, abort', file=sys.stderr, flush=True)
exit(1)
backup.init_proxmox()
vms_uuid = args.vm_uuid
vms_id = args.vm_id
vm_name_match = args.vm_name
snapshot_name_prefix = args.snapshot_name_prefix
allow_using_any_existing_snapshot = args.allow_using_any_existing_snapshot
if snapshot_name_prefix:
backup.set_snapshot_name_prefix(snapshot_name_prefix)
else:
def wiz_execute(wiz):
log.debug("Executing backup creation wizard")
config = Config.get_config()
name = wiz.fields["name"].value
path = wiz.fields["folderpath"].value
excl = []
for typ in config.file_types.iterkeys():
if wiz.fields['excl-'+typ].value:
excl.append(typ)
store = wiz.fields["store"].value
b = Backup(name)
b.active = True
b.include_folders = [path]
b.include_packages = True
b.exclude_types = excl
b.store = store
b.excrypt = False
b.sched_type = "daily/weekly"
b.sched_times = "19:00/Sun"
b.verify = False
b.notify_msg = True
b.notify_email = False
b.shutdown_after = False
config.backups[name] = b
config.save()
update_crontab(config.backups)
dlg.Info(wiz, _("Your backup has been successfully created."))
app.broadcast_update()
def build_config(conf):
# This is called to build a default config object
# It is used the first time this application is run.
log.info("Building new config object")
conf.version = 1
conf.file_types = {_("Images"): ["jpg", "jpeg", "raw",
"cr2", "png", "gif", "dng", "tiff",
"tif", "bmp", "svg", "ppm", "psd"],
_("Videos"): ["avi", "mpg", "mpeg", "mp4", "mov", "m4v", "vob", "wmv", "flv"],
_("Music"): ["mp3", "aac", "ogg", "flac", "wav", "wma", "mpa"],
_("Programs"): ["bin", "dll", "exe", "com", "lib"]
}
conf.storage = {}
store = FolderStore(_("System Backup Folder"), "", False, os.path.join(const.DataDir, "vault-store"))
conf.storage[store.name] = store
# By default, the data passphrase is blank.
conf.data_passphrase = None
conf.backups = {}
b = Backup(_("Home"))
b.include_folders = ["/home"]
b.active = False
b.include_packages = True
b.store = store.name
b.notify_msg = True
conf.backups[b.name] = b
conf.mail_server = ""
conf.mail_port = 25
conf.mail_ssl = False
conf.mail_auth = False
conf.mail_login = ""
conf.mail_password = ""
conf.mail_from = ""
conf.mail_to = ""
if const.Debug:
# If we are debugging, we create debug store and backup objects
store = FolderStore("TestStore", "20MB", True,
os.path.join(const.RunDir, "store"))
conf.storage[store.name] = store
b = Backup("test")
b.include_folders = [os.path.join(const.RunDir, "files")]
b.include_packages = True
b.exclude_types = ["Videos", "Programs"]
b.exclude_patterns = ["*/not"]
b.store = store.name
b.notify_msg = True
b.encrypt = True
conf.backups[b.name] = b
# DEBUG - reset the store and include folders
conf.backups["Home"].store = store.name
conf.backups["Home"].include_folders = [os.path.join(const.RunDir, "files")]
# (/mnt/linode_backups/mysql.the-jci.org/) and places it at /home/backup_dir/prepare/.
#
# This is meant to be used when you need to grab data from an incremental backup
from lib.config_helper import Config_helper
from lib.log_helper import Log_helper
from lib.preparer import Preparer
from lib.backup import Backup
import sys
host = "localhost"
config_helper = Config_helper(host=host)
logger = Log_helper(host, log_name="%s_logger" % host)
logger.setup()
logger.info_message("######### STARTING PREPARE PROCESS #########")
host_backup = Backup(host, logger=logger)
backup_dir = host_backup.get_latest_backup_dir_name()
logger.info_message("Preparing backup dir %s" % backup_dir)
prepare_obj = Preparer(host=host,
backup_type=Backup.BACKUP_TYPE_INC,
backup_dir=backup_dir,
prepare_dir=config_helper.get_prepare_dir(),
logger=logger)
prepare_obj.setup()
prepare_dir = prepare_obj.prepare()
logger.info_message("Prepare dir %s" % prepare_dir)
