def test_getBackupHosts(test_config, tmp_path): config().jobspooldirectory = str(tmp_path) config().backupmailrecipients = ['*****@*****.**'] se = statusemail() # no jobs ret = se.getBackupHosts([]) assert ret == [] # disabled job path = os.path.join( os.path.dirname(__file__), 'etc/hostname-only.job', ) j = job(path) ret = se.getBackupHosts([j]) assert ret == [] # enabled job path = os.path.join( os.path.dirname(__file__), 'etc/localhost.job', ) j = job(path) ret = se.getBackupHosts([j]) assert ret == ['localhost']
def test_getOverallBackupState_warning(test_config, tmp_path): config().jobspooldirectory = str(tmp_path) config().backupmailrecipients = ['*****@*****.**'] se = statusemail() se.history = [ { 'rsync_backup_status': 1, 'sanity_check': 1, 'integrity_confirmed': True, 'commands': [ { 'returncode': 1, 'continueonerror': True, }, ], }, ] (ret, good, warning, bad) = se.getOverallBackupState(se.history) assert ret == 'warning' assert len(good) == 0 assert len(warning) == 1 assert len(bad) == 0
def test_runBackup_dryrun(test_config, tmp_path, monkeypatch): email_path = os.path.join(str(tmp_path), 'status.eml') def mock_send(self, message): with open(email_path, 'w') as f: f.write(message.as_string()) return True monkeypatch.setattr(mailer.Mailer, 'send', mock_send) config().rsyncpath = os.path.join( os.path.dirname(__file__), 'bin/mock-rsync-ok.sh', ) config().backupmailrecipients = ['*****@*****.**'] path = os.path.join( os.path.dirname(__file__), 'rsync.job', ) runBackup(path, True) assert os.path.exists(email_path) is False assert os.path.exists(config().lockfile) is False
def test_runBackup_no_jobs(test_config, tmp_path, monkeypatch, caplog): email_path = os.path.join(str(tmp_path), 'sudden-death.eml') def mock_send(self, message): with open(email_path, 'w') as f: f.write(message.as_string()) return True monkeypatch.setattr(mailer.Mailer, 'send', mock_send) config().rsyncpath = os.path.join( os.path.dirname(__file__), 'bin/mock-rsync-ok.sh', ) config().backupmailrecipients = ['*****@*****.**'] with pytest.raises(Exception) as e: runBackup(str(tmp_path), False) assert e is not None message = 'Error while reading %s, skipping job' % str(tmp_path) assert message in caplog.text assert os.path.exists(email_path) is False assert os.path.exists(config().lockfile) is False
def test_config(tmp_path): path = os.path.join( str(tmp_path), 'main.yaml', ) options = { 'lockfile': os.path.join(str(tmp_path), 'autorsyncbackup.pid'), 'jobconfigdirectory': os.path.join( str(tmp_path), 'etc', ), 'jobspooldirectory': os.path.join( str(tmp_path), 'spool', ), 'backupdir': os.path.join( str(tmp_path), 'backups', ), 'logfile': os.path.join(str(tmp_path), 'log/autorsyncbackup.log'), 'debuglevel': 4, } with open(path, 'w') as f: for key in options: f.write("%s: %s\n" % (key, options[key])) config(path) return True
def getWorkingDirectory(self): """Check in which folder we place the backup today""" ret = "daily" if(int(datetime.datetime.today().strftime("%w")) == config().weeklybackup): ret = "weekly" if(int(datetime.datetime.today().strftime("%d")) == config().monthlybackup): ret = "monthly" return ret
def _send(self, subject, htmlbody): for to in config().backupmailrecipients: logger().info("INFO: Sent backup report to [%s] via SMTP:%s" % (to, config().smtphost)) message = Message(From=config().backupmailfrom, To=to, charset="utf-8") message.Subject = subject message.Html = htmlbody message.Body = """This is an HTML e-mail with the backup overview, please use a HTML enabled e-mail client.""" sender = Mailer(config().smtphost) sender.send(message)
def _send(self, subject, htmlbody, textbody): for to in config().backupmailrecipients: logger().info("Sent backup report to [%s] via SMTP:%s" % (to, config().smtphost)) message = Message(From=config().backupmailfrom, To=to, charset="utf-8") message.Subject = subject message.Html = htmlbody message.Body = textbody sender = Mailer(config().smtphost) sender.send(message)
def getWorkingDirectory(self): """Check in which folder we place the backup today""" ret = "daily" if (int(datetime.datetime.today().strftime("%w")) == config().weeklybackup): ret = "weekly" if (int(datetime.datetime.today().strftime("%d")) == config().monthlybackup): ret = "monthly" return ret
def test_listJobs(test_config, tmp_path, capsys): config().lockfile = os.path.join( str(tmp_path), 'autorsyncbackup.pid', ) config().jobconfigdirectory = os.path.join( os.path.dirname(__file__), 'etc/list', ) config().jobspooldirectory = str(tmp_path) config().backupdir = os.path.join( str(tmp_path), 'backups', ) for subdir in [ 'daily', 'weekly', 'monthly', ]: path = os.path.join( config().backupdir, 'localhost', subdir, ) os.makedirs(path) listJobs('total') captured = capsys.readouterr() for title in [ 'Hostname', 'Estimated total backup size', 'Average backup size increase', ]: assert title in captured.out assert 'localhost' in captured.out assert '0 Bytes' in captured.out listJobs('average') captured = capsys.readouterr() for title in [ 'Hostname', 'Estimated total backup size', 'Average backup size increase', ]: assert title in captured.out assert 'localhost' in captured.out assert '0 Bytes' in captured.out
def _send(self, subject, htmlbody): for to in config().backupmailrecipients: logger().info("Sent backup report to [%s] via SMTP:%s" % (to, config().smtphost)) message = Message(From=config().backupmailfrom, To=to, charset="utf-8") message.Subject = subject message.Html = htmlbody message.Body = """This is an HTML e-mail with the backup overview, please use a HTML enabled e-mail client.""" sender = Mailer(config().smtphost) sender.send(message)
def test_getMissingHosts(test_config, tmp_path): config().jobspooldirectory = str(tmp_path) config().backupmailrecipients = ['*****@*****.**'] se = statusemail() # no jobs nor history se.history = [] hosts = se.getMissingHosts([]) assert hosts == [] # enabled job, no history path = os.path.join( os.path.dirname(__file__), 'etc/localhost.job', ) j = job(path) se.history = [] hosts = se.getMissingHosts([j]) assert len(hosts) == 1 assert 'localhost' in hosts # enabled job, job in history se.history = [ { 'hostname': 'localhost', }, ] hosts = se.getMissingHosts([j]) assert hosts == [] # disabled job, no history path = os.path.join( os.path.dirname(__file__), 'etc/hostname-only.job', ) j = job(path) se.history = [] hosts = se.getMissingHosts([j]) assert hosts == []
def test_readConfig_exceptions(monkeypatch): def mock_load(stream): return {} monkeypatch.setattr(yaml, 'load', mock_load) config().debugmessages = [] assert len(config().debugmessages) == 0 config().readConfig() assert len(config().debugmessages) > 1
def test_default_config(test_config): path = os.path.join( os.path.dirname(__file__), 'etc/default-config.job', ) j = job(path) assert j.enabled is True assert j.hostname == 'localhost' assert j.ssh is False assert j.ssh_sudo is False assert j.sshusername is None assert j.sshprivatekey is None assert j.rsyncusername == 'autorsyncbackup' assert j.rsyncpassword == 'fee-fi-fo-fum' assert j.rsyncshare == 'backup' assert j.port == 873 assert j.backupdir == config().backupdir assert j.speedlimitkb == config().speedlimitkb assert j.dailyrotation == config().dailyrotation assert j.weeklyrotation == config().weeklyrotation assert j.monthlyrotation == config().monthlyrotation assert j.weeklybackup == config().weeklybackup assert j.monthlybackup == config().monthlybackup assert j.exclude == config().exclude
def test_checkRemoteHost_fail(test_config, tmp_path): config().jobspooldirectory = str(tmp_path) config().rsyncpath = os.path.join( os.path.dirname(__file__), 'bin/mock-rsync-fail.sh', ) path = os.path.join( os.path.dirname(__file__), 'etc/rsync.sh', ) ret = checkRemoteHost(path) assert ret is True
def test_readConfig_load_exception(monkeypatch, capsys): def mock_load(stream): raise IOError('Mock load failure') monkeypatch.setattr(yaml, 'load', mock_load) with pytest.raises(SystemExit) as e: config().readConfig() assert e.type == SystemExit assert e.value.code == 1 captured = capsys.readouterr() assert 'Error while reading main config' in captured.out
def listJobs(sort): with Pidfile(config().lockfile, logger().debug, logger().error): # Run director directorInstance = director() jobs = directorInstance.getJobArray() sizes = {} averages = {} tot_size = 0 tot_avg = 0 for job in jobs: sizes[job.hostname], averages[ job.hostname] = director().getBackupsSize(job) aux = sorted(sizes.items(), key=lambda x: x[1], reverse=True) if sort == 'average': aux = sorted(averages.items(), key=lambda x: x[1], reverse=True) x = PrettyTable([ 'Hostname', 'Estimated total backup size', 'Average backup size increase' ]) for elem in aux: hostname = elem[0] tot_size += sizes[hostname] tot_avg += averages[hostname] size = jinjafilters()._bytesToReadableStr(sizes[hostname]) avg = jinjafilters()._bytesToReadableStr(averages[hostname]) x.add_row([hostname, size, avg]) tot_size = jinjafilters()._bytesToReadableStr(tot_size) tot_avg = jinjafilters()._bytesToReadableStr(tot_avg) x.add_row(['Total', tot_size, tot_avg]) x.align = "l" x.padding_width = 1 print(x)
def executeRsyncViaSshProtocol(self, job, latest): directory = job.backupdir.rstrip('/') + "/" + job.hostname + "/current" sshoptions = "-e 'ssh -p%d -i %s -o \"PasswordAuthentication no\"'" % (job.port, job.sshprivatekey) options = "-aR %s --delete --stats --bwlimit=%d" % (sshoptions, job.speedlimitkb) exclude = self.generateExclude(job) if exclude: options += exclude include = self.generateInclude(job) # Link files to the same inodes as last backup to save disk space and boost backup performance if(latest): latest = "--link-dest=%s" % latest else: latest = "" # Generate rsync CLI command and execute it if(include): command = "%s %s %s %s %s" % (config().rsyncpath, options, latest, include, directory) logger().info("Executing rsync command (%s)" % command) errcode, stdout = self.executeCommand(command) else: stdout = "Include/Fileset is missing, Rsync is never invoked" errcode = 9 job.backupstatus['rsync_stdout'] = stdout job.backupstatus['rsync_return_code'] = errcode return errcode, stdout
def executeRsyncViaSshProtocol(self, job, latest): directory = job.backupdir.rstrip('/') + "/" + job.hostname + "/current" sudo_path = "--rsync-path='sudo rsync'" if job.ssh_sudo else '' sshoptions = ("-e 'ssh -p%d -i %s" " -o \"PasswordAuthentication no\"' %s") % ( job.port, job.sshprivatekey, sudo_path) options = "-aR %s --delete --stats --bwlimit=%d" % (sshoptions, job.speedlimitkb) exclude = self.generateExclude(job) if exclude: options += exclude include = self.generateInclude(job) # Link files to the same inodes as last backup to save disk space # and boost backup performance if (latest): latest = "--link-dest=%s" % latest else: latest = "" # Generate rsync CLI command and execute it if (include): command = "%s %s %s %s %s" % (config().rsyncpath, options, latest, include, directory) logger().info("Executing rsync command (%s)" % command) errcode, stdout = self.executeCommand(command) else: stdout = "Include/Fileset is missing, Rsync is never invoked" errcode = 9 job.backupstatus['rsync_stdout'] = stdout job.backupstatus['rsync_return_code'] = errcode return errcode, stdout
def executeRsyncViaRsyncProtocol(self, job, latest): """Execute rsync command via rsync protocol""" dir = job.backupdir.rstrip('/') + "/" + job.hostname + "/current" options = "--contimeout=5 -aR --delete --stats --bwlimit=%d" % job.speedlimitkb exclude = self.generateExclude(job) if exclude: options += exclude include = self.generateInclude(job) # Link files to the same inodes as last backup to save disk space and boost backup performance if(latest): latest = "--link-dest=%s" % latest else: latest = "" # Generate rsync CLI command and execute it if(include): password = "******"%s\"" % job.rsyncpassword rsyncCommand = "%s %s %s %s %s" % (config().rsyncpath, options, latest, include, dir) command = "%s; %s" % (password, rsyncCommand) logger().info("Executing rsync command (%s)" % rsyncCommand) errcode, stdout = self.executeCommand(command) else: stdout = "Include/Fileset is missing, Rsync is never invoked" errcode = 9 job.backupstatus['rsync_stdout'] = stdout job.backupstatus['rsync_return_code'] = errcode return errcode, stdout
def checkRemoteHostViaRsyncProtocol(self, job): """Check if remote host is up and able to accept connections with our credentials""" password = "******"%s\"" % job.rsyncpassword rsyncCommand = "%s --contimeout=5 rsync://%s@%s:%s/%s" % (config( ).rsyncpath, job.rsyncusername, job.hostname, job.port, job.rsyncshare) command = "%s; %s" % (password, rsyncCommand) logger().info("Executing rsync check (%s)" % rsyncCommand) errcode, stdout = self.executeCommand(command) if errcode != 0: logger().error("Error while connecting to host (%s) - %s" % (job.hostname, stdout)) job.backupstatus['startdatetime'] = int(time.time()) job.backupstatus['enddatetime'] = int(time.time()) job.backupstatus['hostname'] = job.hostname job.backupstatus['rsync_stdout'] = ("Error while connecting" " to host (%s) - %s") % ( job.hostname, stdout) ret = False else: ret = True logger().info(("Successfully connected to host" " via rsync protocol (%s)") % job.hostname) return ret
def executeRsyncViaRsyncProtocol(self, job, latest): """Execute rsync command via rsync protocol""" dir = job.backupdir.rstrip('/') + "/" + job.hostname + "/current" options = "--contimeout=5 -aR --delete --stats --bwlimit=%d" % ( job.speedlimitkb) exclude = self.generateExclude(job) if exclude: options += exclude include = self.generateInclude(job) # Link files to the same inodes as last backup to save disk space # and boost backup performance if (latest): latest = "--link-dest=%s" % latest else: latest = "" # Generate rsync CLI command and execute it if (include): password = "******"%s\"" % job.rsyncpassword rsyncCommand = "%s %s %s %s %s" % (config().rsyncpath, options, latest, include, dir) command = "%s; %s" % (password, rsyncCommand) logger().info("Executing rsync command (%s)" % rsyncCommand) errcode, stdout = self.executeCommand(command) else: stdout = "Include/Fileset is missing, Rsync is never invoked" errcode = 9 job.backupstatus['rsync_stdout'] = stdout job.backupstatus['rsync_return_code'] = errcode return errcode, stdout
def listJobs(sort): with Pidfile(config().lockfile, logger().debug, logger().error): # Run director directorInstance = director() jobs = directorInstance.getJobArray() sizes = {} averages = {} tot_size=0 tot_avg=0 for job in jobs: sizes[job.hostname], averages[job.hostname] = director().getBackupsSize(job) aux = sorted(sizes.items(), key=lambda x: x[1], reverse=True) if sort == 'average': aux = sorted(averages.items(), key=lambda x: x[1], reverse=True) x = PrettyTable(['Hostname', 'Estimated total backup size', 'Average backup size increase']) for elem in aux: hostname = elem[0] tot_size += sizes[hostname] tot_avg += averages[hostname] size = jinjafilters()._bytesToReadableStr(sizes[hostname]) avg = jinjafilters()._bytesToReadableStr(averages[hostname]) x.add_row([hostname, size, avg]) tot_size = jinjafilters()._bytesToReadableStr(tot_size) tot_avg = jinjafilters()._bytesToReadableStr(tot_avg) x.add_row(['Total', tot_size, tot_avg]) x.align = "l" x.padding_width = 1 print(x)
def __init__(self, directorio): super(LogsController, self).__init__() self.directorio = directorio import logging from models.config import config config = config() LOG_FILENAME = config.pathLog + self.directorio + ".log" if "/" in directorio: carpeta = self.directorio.split("/")[0] if not os.path.isdir(config.pathLog + carpeta): os.makedirs(config.pathLog + carpeta) LEVELS = { "debug": logging.DEBUG, "info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR, "critical": logging.CRITICAL, } if len(sys.argv) > 1: level_name = sys.argv[1] level = LEVELS.get(level_name, logging.NOTSET) logging.basicConfig(filename=LOG_FILENAME, level=level)
def test_executeRsyncViaSshProtocol(test_config, tmp_path): config().jobspooldirectory = str(tmp_path) config().rsyncpath = os.path.join( os.path.dirname(__file__), 'bin/mock-rsync-ok.sh', ) path = os.path.join( os.path.dirname(__file__), 'etc/ssh.job', ) j = job(path) r = rsync() (status, stdout) = r.executeRsyncViaSshProtocol(j, None) assert status == 0 assert 'sending incremental file list' in stdout
def test_checkRemoteHost_rsync_fail(test_config, tmp_path): config().jobspooldirectory = str(tmp_path) config().rsyncpath = os.path.join( os.path.dirname(__file__), 'bin/mock-rsync-fail.sh', ) path = os.path.join( os.path.dirname(__file__), 'etc/rsync.job', ) j = job(path) r = rsync() ret = r.checkRemoteHost(j) assert ret is False assert j.backupstatus['rsync_backup_status'] == int(ret)
def test_printOutput_no_history(test_config, tmp_path, capsys): config().jobspooldirectory = str(tmp_path) jobrunhistory(str(tmp_path), check=True) sc = statuscli() sc.printOutput('localhost') captured = capsys.readouterr() assert 'Could not find hostname' in captured.out
def test_runBackup_exception(test_config, tmp_path, monkeypatch): email_path = os.path.join(str(tmp_path), 'sudden-death.eml') def mock_send(self, message): with open(email_path, 'w') as f: f.write(message.as_string()) return True monkeypatch.setattr(mailer.Mailer, 'send', mock_send) config().rsyncpath = os.path.join( os.path.dirname(__file__), 'bin/mock-rsync-ok.sh', ) config().backupmailrecipients = ['*****@*****.**'] with open(config().lockfile, 'w') as f: f.write(str(os.getpid())) path = os.path.join( os.path.dirname(__file__), 'rsync.job', ) runBackup(path, False) assert os.path.exists(email_path) is True assert os.path.exists(config().lockfile) is True if os.path.exists(config().lockfile): os.unlink(config().lockfile)
def test_executeRsync_ssh_latest(test_config, tmp_path): config().jobspooldirectory = str(tmp_path) config().rsyncpath = os.path.join( os.path.dirname(__file__), 'bin/mock-rsync-ok.sh', ) path = os.path.join( os.path.dirname(__file__), 'etc/ssh.job', ) j = job(path) r = rsync() ret = r.executeRsync(j, 'foo') assert ret is True assert j.backupstatus['rsync_backup_status'] == int(ret) assert j.backupstatus['rsync_return_code'] == 0 assert 'sending incremental file list' in j.backupstatus['rsync_stdout']
def test_executeRsync_ssh_no_include(test_config, tmp_path): config().jobspooldirectory = str(tmp_path) config().rsyncpath = os.path.join( os.path.dirname(__file__), 'bin/mock-rsync-ok.sh', ) path = os.path.join( os.path.dirname(__file__), 'etc/ssh-no-include.job', ) j = job(path) r = rsync() ret = r.executeRsync(j, 'foo') assert ret is False assert j.backupstatus['rsync_backup_status'] == 0 assert j.backupstatus['rsync_return_code'] == 9 assert ('Include/Fileset is missing, Rsync is never invoked' ) in j.backupstatus['rsync_stdout']
def getJobArray(self, jobpath): jobArray = [] if jobpath is None: dir = config().jobconfigdirectory.rstrip('/') if(os.path.exists(dir)): os.chdir(dir) for file in glob.glob("*.job"): jobArray.append(job(dir + "/" + file)) else: logger().error("Job directory (%s) doesn't exists, exiting (1)" % dir) else: jobArray.append(job(jobpath)) return jobArray
def test_printOutput(test_config, tmp_path, capsys): config().jobspooldirectory = str(tmp_path) jrh = jobrunhistory(str(tmp_path), check=True) backupstatus = { 'integrity_id': str(uuid.uuid1()), 'hostname': 'localhost', 'startdatetime': time.time(), 'enddatetime': time.time(), 'username': '******', 'ssh': 'False', 'share': 'backup', 'include': '/etc', 'exclude': '*.bak:.cache/*', 'backupdir': '/tmp', 'speedlimitkb': 1600, 'filesrotate': None, 'type': 'daily', 'rsync_backup_status': 1, 'rsync_return_code': 0, 'rsync_pre_stdout': None, 'rsync_stdout': 'foo\nbar\n', 'rsync_number_of_files': 3278, 'rsync_number_of_files_transferred': 1790, 'rsync_total_file_size': 6249777, 'rsync_total_transferred_file_size': 6213437, 'rsync_literal_data': 6213437, 'rsync_matched_data': 0, 'rsync_file_list_size': 80871, 'rsync_file_list_generation_time': 0.001, 'rsync_file_list_transfer_time': 0, 'rsync_total_bytes_sent': 39317, 'rsync_total_bytes_received': 6430608, 'sanity_check': 1, } hooks = [] jrh.insertJob(backupstatus, hooks) sc = statuscli() sc.printOutput(backupstatus['hostname']) captured = capsys.readouterr() assert 'localhost' in captured.out assert 'Ok' in captured.out
def getJobArray(self, jobpath=None): jobArray = [] if jobpath is None: directory = config().jobconfigdirectory.rstrip('/') if (os.path.exists(directory)): os.chdir(directory) for filename in glob.glob("*.job"): jobArray.append(job(directory + "/" + filename)) else: logger().error( "Job directory (%s) doesn't exists, exiting (1)" % directory) else: jobArray.append(job(jobpath)) return jobArray
def test_checkRemoteHost_ssh_fail(test_config, tmp_path, monkeypatch): def mock_connect(self, hostname, username=None, key_filename=None): raise IOError('Mock connection failed') monkeypatch.setattr(paramiko.SSHClient, 'connect', mock_connect) config().jobspooldirectory = str(tmp_path) path = os.path.join( os.path.dirname(__file__), 'etc/ssh.job', ) j = job(path) r = rsync() ret = r.checkRemoteHost(j) assert ret is False assert j.backupstatus['rsync_backup_status'] == int(ret)
def __init__(self, dataset, is_cuda): """ """ super(gan_super, self).__init__() config_vars = config(dataset) self.lamb = config_vars.lamb self.key_dim = config_vars.key_dim self.mem_size = config_vars.mem_size self.z_dim = config_vars.z_dim self.c_dim = config_vars.c_dim self.fc_dim = config_vars.fc_dim self.f_dim = config_vars.f_dim self.lamb = config_vars.lamb self.key_dim = config_vars.key_dim self.mem_size = config_vars.mem_size self.c_dim = config_vars.c_dim self.fc_dim = config_vars.fc_dim self.f_dim = config_vars.f_dim self.choose_k = config_vars.choose_k self.z_dim = config_vars.z_dim self.alpha = config_vars.alpha self.num_steps = config_vars.num_steps self.is_cuda = is_cuda
def test_spam(): assert re.match(r'^\d+$', str(config().spam()))
def test_readConfig(test_config, tmp_path): attributes = { 'mainconfigpath': os.path.join(str(tmp_path), 'main.yaml'), 'rsyncpath': '/usr/bin/rsync', 'lockfile': os.path.join(str(tmp_path), 'autorsyncbackup.pid'), 'jobconfigdirectory': str(tmp_path), 'jobspooldirectory': str(tmp_path), 'backupdir': str(tmp_path), 'logfile': os.path.join(str(tmp_path), 'autorsyncbackup.log'), 'speedlimitkb': 0, 'dailyrotation': 8, 'weeklyrotation': 5, 'monthlyrotation': 13, 'weeklybackup': 6, 'monthlybackup': 1, 'include': [], 'exclude': [], 'backupmailfrom': 'backup@%s' % socket.getfqdn(), 'backupmailrecipients': [], 'jobworkers': 3, 'debuglevel': 0, 'databaseretention': 540, } with open(attributes['mainconfigpath'], 'w') as f: for key in [ 'rsyncpath', 'lockfile', 'jobconfigdirectory', 'jobspooldirectory', 'backupdir', 'logfile', 'debuglevel', ]: f.write('%s: %s\n' % (key, attributes[key])) config().mainconfigpath = attributes['mainconfigpath'] config().backupmailrecipients = [] config().readConfig() assert config().mainconfigpath == attributes['mainconfigpath'] assert config().rsyncpath == attributes['rsyncpath'] assert config().lockfile == attributes['lockfile'] assert config().jobconfigdirectory == attributes['jobconfigdirectory'] assert config().jobspooldirectory == attributes['jobspooldirectory'] assert config().backupdir == attributes['backupdir'] assert config().logfile == attributes['logfile'] assert config().speedlimitkb == attributes['speedlimitkb'] assert config().dailyrotation == attributes['dailyrotation'] assert config().weeklyrotation == attributes['weeklyrotation'] assert config().monthlyrotation == attributes['monthlyrotation'] assert config().weeklybackup == attributes['weeklybackup'] assert config().monthlybackup == attributes['monthlybackup'] assert config().include == attributes['include'] assert config().exclude == attributes['exclude'] assert config().backupmailfrom == attributes['backupmailfrom'] assert config().backupmailrecipients == attributes['backupmailrecipients'] assert config().jobworkers == attributes['jobworkers'] assert config().debuglevel == attributes['debuglevel'] assert config().databaseretention == attributes['databaseretention']
parser.add_option("-c", "--main-config", dest="mainconfig", metavar="path_to_main.yaml", help="set different main config file, default value = /etc/autorsyncbackup/main.yaml", default="/etc/autorsyncbackup/main.yaml") parser.add_option("-d", "--dry-run", action="store_true", dest="dryrun", default=False, help="do not invoke rsync, only perform a login attempt on remote host") parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Write logoutput also to stdout") parser.add_option("-j", "--single-job", metavar="path_to_jobfile.job", dest="job", help="run only the given job file") (options, args) = parser.parse_args() return options if __name__ == "__main__": options = setupCliArguments() config(options.mainconfig) # Welcome message if options.verbose: print "Starting AutoRsyncBackup" # Set logpath logger(config().logfile) logger().setDebuglevel(config().debuglevel) logger().setVerbose(options.verbose) for msg in config().debugmessages: logger().debug(msg) try: with Pidfile(config().lockfile, logger().debug, logger().error): # Run director
def runBackup(jobpath, dryrun): """ Start backup run """ exitFlag = threading.Event() queueLock = threading.Lock() workQueue = queue.Queue(0) try: with Pidfile(config().lockfile, logger().debug, logger().error): # Run director directorInstance = director() jobs = directorInstance.getJobArray(jobpath) # Start threads threads = [] if not dryrun: for i in range(0, config().jobworkers): thread = jobThread(i, exitFlag, queueLock, directorInstance, workQueue) thread.start() threads.append(thread) # Execute jobs queueLock.acquire() durationstats = {} durationstats['backupstartdatetime'] = int(time.time()) for job in jobs: if(job.enabled): if directorInstance.checkRemoteHost(job): if not dryrun: # Add to queue workQueue.put(job) else: jobrunhistory().insertJob(job.backupstatus, None) queueLock.release() # Wait for queue to empty while not workQueue.empty(): time.sleep(0.1) # Notify threads it's time to exit exitFlag.set() # Wait for all threads to complete for t in threads: t.join() durationstats['backupenddatetime'] = int(time.time()) if not dryrun: # Do housekeeping durationstats['housekeepingstartdatetime'] = int(time.time()) for job in jobs: if(job.enabled): if job.backupstatus['rsync_backup_status'] == 1: directorInstance.backupRotate(job) jobrunhistory().deleteHistory() durationstats['housekeepingenddatetime'] = int(time.time()) # Sent status report statusemail().sendStatusEmail(jobs, durationstats) # else: # for job in jobs: # job.showjob() except ProcessRunningException as m: statusemail().sendSuddenDeath(m) logger().error(m)
directorInstance = director() jobs = directorInstance.getJobArray(jobpath) return not directorInstance.checkRemoteHost(jobs[0]) def getLastBackupStatus(hostname): """ Get status of last backup run of given hostname and exit with exitcode 0 (success) or 1 (error) """ return statuscli().printOutput(hostname) if __name__ == "__main__": """ Start application """ # Initialise variables checkSingleHost = False # Get CLI options and Config options = setupCliArguments() config(options.mainconfig) # Welcome message if options.verbose: print("Starting AutoRsyncBackup") # Only check if host is reachable, set appropriate settings if options.job and options.dryrun: checkSingleHost = True options.verbose = True config().debuglevel = 2 # Set logpath logger(config().logfile) logger().setDebuglevel(config().debuglevel) logger().setVerbose(options.verbose)