def checkBackupEnvironment(self, job):
backupdir = job.backupdir.rstrip('/')
try:
if not os.path.exists(backupdir):
os.makedirs(backupdir)
logger().info("Backup path (%s) created" % backupdir)
directory = backupdir + "/" + job.hostname + "/daily"
if not os.path.exists(directory):
os.makedirs(directory)
directory = backupdir + "/" + job.hostname + "/weekly"
if not os.path.exists(directory):
os.makedirs(directory)
directory = backupdir + "/" + job.hostname + "/monthly"
if not os.path.exists(directory):
os.makedirs(directory)
self._moveLastBackupToCurrentBackup(job)
directory = backupdir + "/" + job.hostname + "/current"
if not os.path.exists(directory):
os.makedirs(directory)
except Exception as e:
logger().error(
"Error creating backup directory (%s) for host (%s)" %
(directory, job.hostname))
statusemail().sendSuddenDeath(e)
return False
def checkBackupEnvironment(self, job):
backupdir = job.backupdir.rstrip('/')
try:
if not os.path.exists(backupdir):
os.makedirs(backupdir)
logger().info("Backup path (%s) created" % backupdir)
directory = backupdir + "/" + job.hostname + "/daily"
if not os.path.exists(directory):
os.makedirs(directory)
directory = backupdir + "/" + job.hostname + "/weekly"
if not os.path.exists(directory):
os.makedirs(directory)
directory = backupdir + "/" + job.hostname + "/monthly"
if not os.path.exists(directory):
os.makedirs(directory)
self._moveLastBackupToCurrentBackup(job)
directory = backupdir + "/" + job.hostname + "/current"
if not os.path.exists(directory):
os.makedirs(directory)
except Exception as e:
logger().error("Error creating backup directory (%s) for host (%s)" % (directory, job.hostname))
statusemail().sendSuddenDeath(e)
return False
def test_getBackupHosts(test_config, tmp_path):
config().jobspooldirectory = str(tmp_path)
config().backupmailrecipients = ['*****@*****.**']
se = statusemail()
# no jobs
ret = se.getBackupHosts([])
assert ret == []
# disabled job
path = os.path.join(
os.path.dirname(__file__),
'etc/hostname-only.job',
)
j = job(path)
ret = se.getBackupHosts([j])
assert ret == []
# enabled job
path = os.path.join(
os.path.dirname(__file__),
'etc/localhost.job',
)
j = job(path)
ret = se.getBackupHosts([j])
assert ret == ['localhost']
def test_getOverallBackupState_warning(test_config, tmp_path):
config().jobspooldirectory = str(tmp_path)
config().backupmailrecipients = ['*****@*****.**']
se = statusemail()
se.history = [
{
'rsync_backup_status': 1,
'sanity_check': 1,
'integrity_confirmed': True,
'commands': [
{
'returncode': 1,
'continueonerror': True,
},
],
},
]
(ret, good, warning, bad) = se.getOverallBackupState(se.history)
assert ret == 'warning'
assert len(good) == 0
assert len(warning) == 1
assert len(bad) == 0
def test_getMissingHosts(test_config, tmp_path):
config().jobspooldirectory = str(tmp_path)
config().backupmailrecipients = ['*****@*****.**']
se = statusemail()
# no jobs nor history
se.history = []
hosts = se.getMissingHosts([])
assert hosts == []
# enabled job, no history
path = os.path.join(
os.path.dirname(__file__),
'etc/localhost.job',
)
j = job(path)
se.history = []
hosts = se.getMissingHosts([j])
assert len(hosts) == 1
assert 'localhost' in hosts
# enabled job, job in history
se.history = [
{
'hostname': 'localhost',
},
]
hosts = se.getMissingHosts([j])
assert hosts == []
# disabled job, no history
path = os.path.join(
os.path.dirname(__file__),
'etc/hostname-only.job',
)
j = job(path)
se.history = []
hosts = se.getMissingHosts([j])
assert hosts == []
def test_getStats(test_config, tmp_path):
config().jobspooldirectory = str(tmp_path)
config().backupmailrecipients = ['*****@*****.**']
se = statusemail()
se.history = [
{
'rsync_backup_status': 1,
'commanderror': 'ok',
'startdatetime': time.time() - 10,
'enddatetime': time.time(),
'rsync_number_of_files': 8086,
'rsync_number_of_files_transferred': 42,
'rsync_total_file_size': 6800,
'rsync_total_transferred_file_size': 314,
'rsync_literal_data': 42,
'rsync_matched_data': 42,
'rsync_file_list_size': 386,
'rsync_file_list_generation_time': 0.007,
'rsync_file_list_transfer_time': 0.016,
'rsync_total_bytes_sent': 486,
'rsync_total_bytes_received': 586,
'speedlimitkb': 686,
'sanity_check': 0,
},
]
ret = se.getStats([])
keys = [
'total_host_count',
'total_backups_failed',
'total_rsync_duration',
'total_number_of_files',
'total_number_of_files_transferred',
'total_file_size',
'total_transferred_file_size',
'total_literal_data',
'total_matched_data',
'total_file_list_size',
'total_file_list_generation_time',
'total_file_list_transfer_time',
'total_bytes_sent',
'total_bytes_received',
'total_speed_limit_kb',
'average_backup_duration',
'average_speed_limit_kb',
'total_backups_success',
]
for key in keys:
assert key in ret
assert ret['total_backups_failed'] == 1
assert ret['total_backups_success'] == 0
assert ret['average_backup_duration'] == 0
assert ret['average_speed_limit_kb'] == 0
se.history = [
{
'rsync_backup_status': 0,
'commanderror': 'ok',
'startdatetime': time.time() - 10,
'enddatetime': time.time(),
'rsync_number_of_files': 8086,
'rsync_number_of_files_transferred': 42,
'rsync_total_file_size': 6800,
'rsync_total_transferred_file_size': 314,
'rsync_literal_data': 42,
'rsync_matched_data': 42,
'rsync_file_list_size': 386,
'rsync_file_list_generation_time': 0.007,
'rsync_file_list_transfer_time': 0.016,
'rsync_total_bytes_sent': 486,
'rsync_total_bytes_received': 586,
'speedlimitkb': 686,
'sanity_check': 1,
},
]
ret = se.getStats([])
assert ret['total_backups_failed'] == 1
assert ret['total_backups_success'] == 0
assert ret['average_backup_duration'] == 0
assert ret['average_speed_limit_kb'] == 0
se.history = [
{
'rsync_backup_status': 1,
'commanderror': 'ok',
'startdatetime': time.time() - 10,
'enddatetime': time.time(),
'rsync_number_of_files': 8086,
'rsync_number_of_files_transferred': 42,
'rsync_total_file_size': 6800,
'rsync_total_transferred_file_size': 314,
'rsync_literal_data': 42,
'rsync_matched_data': 42,
'rsync_file_list_size': 386,
'rsync_file_list_generation_time': 0.007,
'rsync_file_list_transfer_time': 0.016,
'rsync_total_bytes_sent': 486,
'rsync_total_bytes_received': 586,
'speedlimitkb': 0,
'sanity_check': 1,
},
]
ret = se.getStats([])
assert ret['total_backups_failed'] == 0
assert ret['total_backups_success'] == 1
assert ret['average_backup_duration'] > 0
assert ret['average_speed_limit_kb'] == 0
se.history = [
{
'rsync_backup_status': 1,
'commanderror': 'ok',
'startdatetime': time.time() - 10,
'enddatetime': time.time(),
'rsync_number_of_files': 8086,
'rsync_number_of_files_transferred': 42,
'rsync_total_file_size': 6800,
'rsync_total_transferred_file_size': 314,
'rsync_literal_data': 42,
'rsync_matched_data': 42,
'rsync_file_list_size': 386,
'rsync_file_list_generation_time': 0.007,
'rsync_file_list_transfer_time': 0.016,
'rsync_total_bytes_sent': 486,
'rsync_total_bytes_received': 586,
'speedlimitkb': 686,
'sanity_check': 1,
},
]
ret = se.getStats([])
assert ret['total_backups_failed'] == 0
assert ret['total_backups_success'] == 1
assert ret['average_backup_duration'] > 0
assert ret['average_speed_limit_kb'] > 0
def test_sendStatusEmail(test_config, tmp_path, monkeypatch):
email_path = os.path.join(str(tmp_path), 'status.eml')
def mock_send(self, message):
with open(email_path, 'w') as f:
f.write(message.as_string())
return True
monkeypatch.setattr(mailer.Mailer, 'send', mock_send)
config().jobspooldirectory = str(tmp_path)
config().backupmailrecipients = ['*****@*****.**']
path = os.path.join(
os.path.dirname(__file__),
'etc/localhost.job',
)
jrh = jobrunhistory(str(tmp_path), check=True)
backupstatus = {
'integrity_id': str(uuid.uuid1()),
'hostname': 'localhost',
'startdatetime': time.time(),
'enddatetime': time.time(),
'username': '******',
'ssh': 'False',
'share': 'backup',
'include': '/etc',
'exclude': '*.bak:.cache/*',
'backupdir': '/tmp',
'speedlimitkb': 1600,
'filesrotate': None,
'type': 'daily',
'rsync_backup_status': 1,
'rsync_return_code': 0,
'rsync_pre_stdout': None,
'rsync_stdout': 'foo\nbar\n',
'rsync_number_of_files': 3278,
'rsync_number_of_files_transferred': 1790,
'rsync_total_file_size': 6249777,
'rsync_total_transferred_file_size': 6213437,
'rsync_literal_data': 6213437,
'rsync_matched_data': 0,
'rsync_file_list_size': 80871,
'rsync_file_list_generation_time': 0.001,
'rsync_file_list_transfer_time': 0,
'rsync_total_bytes_sent': 39317,
'rsync_total_bytes_received': 6430608,
'sanity_check': 1,
}
hooks = [
{
'local': 1,
'runtime': 'before',
'returncode': 0,
'continueonerror': True,
'script': 'uptime',
'stdout': (
' 12:11:51 up 2:40, 13 users, '
' load average: 0.84, 0.71, 0.71\n'
),
'stderr': '',
},
]
jrh.insertJob(backupstatus, hooks)
j = job(path)
jobs = [
j,
]
durationstats = {
'backupstartdatetime': int(time.time()) - 40,
'backupenddatetime': int(time.time()) - 30,
'housekeepingstartdatetime': int(time.time()) - 20,
'housekeepingenddatetime': int(time.time()) - 10,
}
se = statusemail()
se.sendStatusEmail(jobs, durationstats)
assert se.history is not None
assert se.history != []
assert os.path.exists(email_path)
text_body = None
html_body = None
with open(email_path) as f:
msg = email.message_from_file(f)
assert msg is not None
assert msg.is_multipart() is True
for part in msg.walk():
content_type = part.get_content_type()
assert content_type in [
'multipart/alternative',
'text/plain',
'text/html',
]
if content_type == 'multipart/alternative':
continue
body = part.get_payload()
assert body is not None
encoding = part.get('Content-Transfer-Encoding')
if encoding is not None and encoding == 'base64':
body = base64.b64decode(body)
if content_type == 'text/plain':
text_body = body.decode()
elif content_type == 'text/html':
html_body = body.decode()
assert text_body is not None
assert html_body is not None
assert 'Integrity:' in text_body
assert '>Integrity</td>' in html_body
def test_sendSuddenDeath(test_config, tmp_path, monkeypatch):
email_path = os.path.join(str(tmp_path), 'sudden-death.eml')
def mock_send(self, message):
with open(email_path, 'w') as f:
f.write(message.as_string())
return True
monkeypatch.setattr(mailer.Mailer, 'send', mock_send)
config().jobspooldirectory = str(tmp_path)
config().backupmailrecipients = ['*****@*****.**']
exc = 'Mock: There was much rejoicing'
se = statusemail()
se.sendSuddenDeath(exc)
assert os.path.exists(email_path)
text_body = None
html_body = None
with open(email_path) as f:
msg = email.message_from_file(f)
assert msg is not None
assert msg.is_multipart() is True
for part in msg.walk():
content_type = part.get_content_type()
assert content_type in [
'multipart/alternative',
'text/plain',
'text/html',
]
if content_type == 'multipart/alternative':
continue
body = part.get_payload()
assert body is not None
encoding = part.get('Content-Transfer-Encoding')
if encoding is not None and encoding == 'base64':
body = base64.b64decode(body)
if content_type == 'text/plain':
text_body = body.decode()
elif content_type == 'text/html':
html_body = body.decode()
assert text_body is not None
assert html_body is not None
assert exc in text_body
assert '<p>%s</p>' % exc in html_body
def test_init():
se = statusemail()
assert se.jobrunhistory is not None
def runBackup(jobpath, dryrun):
""" Start backup run """
exitFlag = threading.Event()
queueLock = threading.Lock()
workQueue = queue.Queue(0)
try:
with Pidfile(config().lockfile, logger().debug, logger().error):
# Run director
directorInstance = director()
jobs = directorInstance.getJobArray(jobpath)
# Start threads
threads = []
if not dryrun:
for i in range(0, config().jobworkers):
thread = jobThread(i, exitFlag, queueLock,
directorInstance, workQueue)
thread.start()
threads.append(thread)
# Execute jobs
queueLock.acquire()
durationstats = {}
durationstats['backupstartdatetime'] = int(time.time())
for job in jobs:
if (job.enabled):
if directorInstance.checkRemoteHost(job):
if not dryrun:
# Add to queue
workQueue.put(job)
else:
jobrunhistory().insertJob(job.backupstatus, None)
queueLock.release()
# Wait for queue to empty
while not workQueue.empty():
time.sleep(0.1)
# Notify threads it's time to exit
exitFlag.set()
# Wait for all threads to complete
for t in threads:
t.join()
durationstats['backupenddatetime'] = int(time.time())
if not dryrun:
# Do housekeeping
durationstats['housekeepingstartdatetime'] = int(time.time())
for job in jobs:
if (job.enabled):
if job.backupstatus['rsync_backup_status'] == 1:
directorInstance.backupRotate(job)
jobrunhistory().deleteHistory()
durationstats['housekeepingenddatetime'] = int(time.time())
# Sent status report
statusemail().sendStatusEmail(jobs, durationstats)
# else:
# for job in jobs:
# job.showjob()
except ProcessRunningException as m:
statusemail().sendSuddenDeath(m)
logger().error(m)
with Pidfile(config().lockfile, logger().debug, logger().error):
# Run director
director = director()
jobs = director.getJobArray(options.job)
# Execute jobs
durationstats = {}
durationstats['backupstartdatetime'] = int(time.time())
for job in jobs:
if(job.enabled):
if director.checkRemoteHost(job):
if not options.dryrun:
director.checkBackupEnvironment(job)
latest = director.checkForPreviousBackup(job)
director.executeRsync(job, latest)
director.processBackupStatus(job)
durationstats['backupenddatetime'] = int(time.time())
if not options.dryrun:
# Do housekeeping
durationstats['housekeepingstartdatetime'] = int(time.time())
for job in jobs:
if(job.enabled):
if job.backupstatus['rsync_backup_status'] == 1:
director.backupRotate(job)
durationstats['housekeepingenddatetime'] = int(time.time())
# Sent status report
statusemail().sendStatusEmail(jobs, durationstats)
except ProcessRunningException as m:
logger().error(m)
def runBackup(jobpath, dryrun):
""" Start backup run """
exitFlag = threading.Event()
queueLock = threading.Lock()
workQueue = queue.Queue(0)
try:
with Pidfile(config().lockfile, logger().debug, logger().error):
# Run director
directorInstance = director()
jobs = directorInstance.getJobArray(jobpath)
# Start threads
threads = []
if not dryrun:
for i in range(0, config().jobworkers):
thread = jobThread(i, exitFlag, queueLock, directorInstance, workQueue)
thread.start()
threads.append(thread)
# Execute jobs
queueLock.acquire()
durationstats = {}
durationstats['backupstartdatetime'] = int(time.time())
for job in jobs:
if(job.enabled):
if directorInstance.checkRemoteHost(job):
if not dryrun:
# Add to queue
workQueue.put(job)
else:
jobrunhistory().insertJob(job.backupstatus, None)
queueLock.release()
# Wait for queue to empty
while not workQueue.empty():
time.sleep(0.1)
# Notify threads it's time to exit
exitFlag.set()
# Wait for all threads to complete
for t in threads:
t.join()
durationstats['backupenddatetime'] = int(time.time())
if not dryrun:
# Do housekeeping
durationstats['housekeepingstartdatetime'] = int(time.time())
for job in jobs:
if(job.enabled):
if job.backupstatus['rsync_backup_status'] == 1:
directorInstance.backupRotate(job)
jobrunhistory().deleteHistory()
durationstats['housekeepingenddatetime'] = int(time.time())
# Sent status report
statusemail().sendStatusEmail(jobs, durationstats)
# else:
# for job in jobs:
# job.showjob()
except ProcessRunningException as m:
statusemail().sendSuddenDeath(m)
logger().error(m)