def run_monthly(last: bool = False, dry_run: bool = False):
"""
special monthly procedure
"""
if dry_run:
logger.log("robot", "Dry run procedure for testing monthly procedures")
else:
logger.log("exec", "run monthly procedures")
from p05_month import run
e = run(dry_run)
for k, v in e.items():
if v != "Ok":
logger.log_error("robot", k + " -> " + v)
#
# pack monthly
#
cwd = os.getcwd()
os.chdir(log_dir)
month_log = full_logfile.parent / (full_logfile.stem + "_month_" + str(the_date.year) + "_" + str(the_date.month) + ".tgz")
if month_log.exists():
return # already done
year_log = full_logfile.parent / (full_logfile.stem + "_year_" + str(the_date.year) + ".tgz")
if year_log.exists():
return # already done
cmd = "tar czf " + str(month_log) + " " + str(full_logfile)
if dry_run:
logger.log("monthly", "Packing dry")
logger.log("monthly", ">>> " + cmd)
else:
logger.log("monthly", "Packing")
system_exec(cmd)
if not last:
logger.new_log()
os.chdir(cwd)
def remove_tmp():
"""
clean the /tmp folder
:return:
"""
if not os.path.exists("/tmp"):
return
if os.path.islink("/tmp"):
return
logger.log("daily", "Removing scratch and junk files")
cwd = os.getcwd()
os.chdir("/tmp")
list_dir = os.listdir()
now = time.time()
for file in list_dir:
if os.path.isfile(file):
diff = datetime.date.fromtimestamp(now - os.path.getatime(file))
if diff.day > 7:
os.remove(file)
continue
if os.path.isdir(file):
if file in [".", ".X11-unix", ".ICE-unix", "vi.recover"]:
continue
diff = datetime.date.fromtimestamp(now - os.path.getmtime(file))
if diff.day > 1:
shutil.rmtree(file)
continue
os.chdir(cwd)
def send_html():
"""
sed a mail if exist
"""
if not os.path.exists(html_mail_file):
return
logger.log("mailing", "Mail need to be sent")
m_list = get_mailing_list()
mailing_string = " ".join(m_list)
header_lines = "\n".join([
"From: [email protected]", "To: " + mailing_string,
"Subject: Activity repport from argawaen.net server",
"Mime-Version: 1.0", "Content-Type: text/html"
])
fd = open(html_mail_file)
html_lines_mail = fd.readlines()
fd.close()
mail_txt = open(mail_to_send, "w")
mail_txt.write(header_lines + "\n")
mail_txt.write("\n")
for line in html_lines_mail:
mail_txt.write(line + "\n")
mail_txt.close()
if not sendmail(mail_to_send, mailing_string):
# do save+flush ONLY if mail is sent successfully
return
save_dir = os.path.join(data_dir, "save")
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for f in [md_mail_file, html_mail_file, mail_to_send]:
shutil.copy2(f, os.path.join(save_dir, os.path.basename(f)))
def login_account():
"""
get login statistics (not really working)
:return:
"""
# ac -p | sort -nr -k 2
ret, lines = system_exec("ac -p | sort -nr -k 2")
logger.log("weekly", "login time statistics\n" + "\n".join(lines))
add_paragraph_with_lines("Login statistic", 3, lines=lines)
def run_30sec(dry_run: bool = False):
"""
procedure run every 30 seconds
"""
if dry_run:
logger.log("robot", "Dry run procedure for testing 30 seconds procedures")
from p00_30seconds import run
e = run(dry_run)
for k, v in e.items():
if v != "Ok":
logger.log_error("robot", k + " -> " + v)
def end_mail_procedure(cleanup: bool = True):
"""
gather all stuf, send mail and clean
:return:
"""
if not os.path.exists(md_mail_file):
return
logger.log("mailing", "Mail need to be sent")
generate_htmlfile()
send_html()
clean_mail()
def run_special(dry_run: bool = False):
"""
run the content of the special folder
"""
if dry_run:
logger.log("robot", "Dry run procedure for testing special procedures")
else:
logger.log("exec", "run special procedures")
from p07_special import run
e = run(dry_run)
for k, v in e.items():
if v != "Ok":
logger.log_error("robot", k + " -> " + v)
def run_10min(dry_run: bool = False):
"""
procedure run every 10 minutes
"""
if dry_run:
logger.log("robot", "Dry run procedure for testing 10 minutes procedures")
else:
logger.log("exec", "run 10 minutes procedures")
from p01_tenmin import run
e = run(dry_run)
for k, v in e.items():
if v != "Ok":
logger.log_error("robot", k + " -> " + v)
def services():
"""
check for service that are not runin
:return:
"""
ret, lines = system_exec("rcctl ls failed")
if len(lines) == 0:
# everything is OK!
return
logger.log(
"daily",
"Services that should be running but aren't\n" + "\n".join(lines))
add_paragraph_with_items("Services that should be running but aren't",
lines=lines)
def run_daily(dry_run: bool = False):
"""
daily procedure
"""
run_hourly(dry_run, True)
if dry_run:
logger.log("robot", "Dry run procedure for testing daily procedures")
else:
logger.log("exec", "run daily procedures")
from p03_day import run
e = run(dry_run)
for k, v in e.items():
if v != "Ok":
logger.log_error("robot", k + " -> " + v)
def main(dry_run: bool = False):
"""
main script execution
:param dry_run: if the script should be run without system modification
:return:
"""
logger.log("newversion_check",
"Daily OpenBSD version Check" + ["", " dry"][dry_run])
old_version = get_actual_version()
new_version = increment_version(old_version).strip()
url_new = base_repository + new_version + "/"
if exist_http_page(url_new):
logger.log("newversion_check",
"WARNING: new OpenBSD version available!", 1)
add_paragraph(
"OpenBSD Version Check",
message="**WARNING**: new OpenBSD version available [here](" +
url_new + ")")
else:
logger.log("newversion_check", "no new OpenBSD version available!")
# logger.log("newversion_check"," check at: '"+urlnew+"'")
url_new = base_repository + old_version + "/"
if not exist_http_page(url_new):
logger.log_error("newversion_check",
"Actual OpenBSD version no more supported")
add_paragraph(
"OpenBSD Version Check",
message="**WARNING**: actual OpenBSD version no more supported")
else:
logger.log("newversion_check",
"Actual OpenBSD version still available!")
def run_weekly(dry_run: bool = False):
"""
weekly procedure
"""
run_daily(dry_run)
if dry_run:
logger.log("robot", "Dry run procedure for testing weekly procedures")
else:
logger.log("exec", "run weekly procedures")
from p04_week import run
e = run(dry_run)
for k, v in e.items():
if v != "Ok":
logger.log_error("robot", k + " -> " + v)
def run_hourly(dry_run: bool = False, from_daily: bool = False):
"""
hourly procedures
"""
run_10min(dry_run)
if dry_run:
logger.log("robot", "Dry run procedure for testing hourly procedures")
else:
logger.log("exec", "run hourly procedures")
from p02_hour import run
e = run(dry_run)
for k, v in e.items():
if v != "Ok":
logger.log_error("robot", k + " -> " + v)
def purge_account():
"""
purge the account data
:return:
"""
if not os.path.exists("/var/account/acct"):
return
logger.log("daily", "Purging accounting records")
if os.path.exists("/var/account/acct.2"):
os.rename("/var/account/acct.2", "/var/account/acct.3")
if os.path.exists("/var/account/acct.1"):
os.rename("/var/account/acct.1", "/var/account/acct.2")
if os.path.exists("/var/account/acct.0"):
os.rename("/var/account/acct.0", "/var/account/acct.1")
shutil.copy("/var/account/acct", "/var/account/acct.1")
def newsyslog_forced():
"""
monthly log rotate has to be forced
:return:
"""
logger.log("monthly", "newsyslog forced")
ret, lines = system_exec("/usr/bin/newsyslog -F")
if len(lines) != 0:
# il y a un probleme
logger.log_error("monthly",
"problem in newsyslog execution\n" + "\n".join(lines))
add_paragraph_with_lines(
"newsyslog",
pre_message=["problem in newsyslog execution"],
lines=lines)
def run_unit_tests():
"""
run all unittests
"""
import sys
logger.log("TestingSystem", "*-------------------------*")
logger.log("TestingSystem", "| testing procedure |")
logger.log("TestingSystem", "*-------------------------*")
test_exec = Path(__file__).parent / "test.py"
cmd = "python " + str(test_exec)
logger.log("TestingSystem", cmd)
ret, lines = system_exec(cmd)
for line in lines:
logger.log("TestingSystem", line)
if ret != 0:
sys.exit(1)
def locate_database():
"""
update the locate database
:return:
"""
# /usr/libexec/locate.updatedb
logger.log("weekly", "Updating locate database")
ret, lines = system_exec("/usr/libexec/locate.updatedb")
if len(lines) != 0:
# houston, we got a problem
logger.log_error(
"weekly", "problem in locate database update\n" + "\n".join(lines))
add_paragraph_with_lines(
"locate_database",
3,
pre_message=["Problems in locate database reconstruction"],
lines=lines)
def disk():
"""
check disk space
:return:
"""
ret, lines = system_exec("df -hl")
if len(lines) == 0:
return
logger.log("daily", "Disks:\n" + "\n".join(lines))
ct = []
r = []
for line in lines:
if len(ct) == 0:
ct = line.split(maxsplit=5)
continue
r.append(line.split(maxsplit=5))
add_paragraph_with_array("Disks", col_titles=ct, rows=r)
def whatis_database():
"""
update the whatis database
:return:
"""
# /usr/sbin/makewhatis
logger.log("weekly", "whatis database update")
ret, lines = system_exec("/usr/sbin/makewhatis")
if len(lines) != 0:
# il y a un probleme
logger.log_error(
"weekly", "problem in whatis database update\n" + "\n".join(lines))
add_paragraph_with_lines(
"whatis_database",
3,
pre_message=["problem in whatis database update"],
lines=lines)
def main(dry_run: bool = False):
"""
main script execution
:param dry_run: if the script should be run without system modification
:return:
"""
logger.log("weekly", "running weekly procedure")
#
add_mail_line("##WEEKLY procedure##")
#
if not dry_run:
locate_database()
#
check_packages()
#
if not dry_run:
whatis_database()
#
login_account()
def main(dry_run: bool = False):
"""
main script execution
:param dry_run: if the script should be run without system modification
:return:
"""
logger.log("daily", "runing daily procedure")
#
hlines = setheaderlines()
add_paragraph_with_lines("DAILY procedure", 3, lines=hlines)
#
if not dry_run:
remove_tmp()
#
if not dry_run:
purge_account()
#
services()
#
check_daily_errors()
#
disk()
def sendmail(local_mail_file, local_mailing_list):
"""
:param local_mail_file:
:param local_mailing_list:
:return:
"""
ret, lines = system_exec("cat " + local_mail_file + " | sendmail " +
local_mailing_list)
if len(lines) == 0:
return True
# there is a problem
temp_pb = False
for line in lines:
if "451" in line:
temp_pb = True
break
if not temp_pb:
# this is a true problem
logger.log_error("mailing", "Mail sending problem:")
for line in lines:
logger.log("mailing", line)
return False
# attempt to restart smtpd:
if not restart_smtpd():
# error during restart
logger.log_error("mailing", "unable to restart smtpd")
return False
# resend message
ret, lines = system_exec("cat " + local_mail_file + " | sendmail " +
local_mailing_list)
if len(lines) == 0:
return True
logger.log_error("mailing", "Mail sending problem:")
for line in lines:
logger.log("mailing", line)
return False
def main(dry_run: bool = False):
"""
main script execution
:param dry_run: if the script should be run without system modification
:return:
"""
#
if check_certificates():
logger.log("autoSSLRenew", "Certificates Still valid")
add_paragraph("SSL renewal",
message="SSL certificates are still valid")
return
logger.log("autoSSLRenew", "Certificates due to renewal")
ret, lines = system_exec("/usr/local/bin/certbot renew" +
["", " --dry-run"][dry_run])
if ret != 0:
logger.log_error("autoSSLRenew",
"certbot return code (" + str(ret) + ")")
for line in lines:
logger.log_error("autoSSLRenew", line)
return
if check_certificates() or dry_run:
logger.log("autoSSLRenew",
"SSL Certificates have been successfully renewed")
add_paragraph(
"SSL renewal",
message="SSL Certificates have been successfully renewed")
if dry_run:
return
ret, lines = system_exec("rcctl restart apache2")
if ret == 0:
return
logger.log_error("autoSSLRenew",
"Unable to restart web server after renewal")
for line in lines:
logger.log_error("autoSSLRenew", line)
ret, lines = system_exec("rcctl restart smtpd")
if ret == 0:
return
logger.log_error("autoSSLRenew",
"Unable to restart mail server after renewal")
for line in lines:
logger.log_error("autoSSLRenew", line)
else:
logger.log_error(
"autoSSLRenew",
"SSL Certificates are still invalid after renew\n" +
"\n".join(lines))
add_paragraph_with_lines(
"SSL renewal",
pre_message=["SSL Certificates are still invalid after renew"],
lines=lines)
def run_yearly(dry_run: bool = False):
"""
special yearly procedures
"""
run_monthly(True, dry_run)
if dry_run:
logger.log("robot", "Dry run procedure for testing yearly procedures")
else:
logger.log("exec", "run yearly procedures")
from p06_year import run
e = run(dry_run)
for k, v in e.items():
if v != "Ok":
logger.log_error("robot", k + " -> " + v)
#
# pack yearly
#
cwd = os.getcwd()
os.chdir(log_dir)
year_log = full_logfile.parent / (full_logfile.stem + "_year_" + str(the_date.year) + ".tgz")
if year_log.exists():
return # already done
cmd = "tar czf " + str(year_log) + " " + \
str(full_logfile.parent / full_logfile.stem) + "_month_" + str(the_date.year) + "_*.tgz "
if dry_run:
logger.log("yearly", "Packing dry:")
logger.log("yearly", ">>> " + cmd)
logger.log("yearly", ">>> logger.new_log")
else:
logger.log("yearly", "Packing")
system_exec(cmd)
logger.new_log()
os.chdir(cwd)
logger.log("yearly", "Happy new year!")
def main():
"""
main section
"""
parser = argparse.ArgumentParser()
parser.add_argument("-f", "--fast", action="store_true",
help="Run the script in FAST mode: only 30sec actions no mail")
parser.add_argument("-s", "--special", action="store_true", help="Run the script in FAST mode: no mail")
parser.add_argument("-m", "--mailing", action="store_true", help="Run the script in mail-only mode")
parser.add_argument("-d", "--dry_run", action="store_true",
help="Run the script in dry-run mode for testing purpose")
parser.add_argument("-v", "--verbose", action="count", help="Level of verbosity, default= 1", default=1)
parser.add_argument("-t", "--test_mode", action="store_true",
help="Run the script in test mode to run the unit-tests")
args = parser.parse_args()
# initialise the default logging system
if args.test_mode:
init_loger("console", 5)
run_unit_tests()
return
if args.special:
init_loger("console", args.verbose)
run_special(args.dry_run)
return
if args.fast:
init_loger(full_logfile, args.verbose)
run_30sec()
return
if args.dry_run:
init_loger(os.path.join(log_dir, "maintenance_dry_run.log"), args.verbose)
logger.log("robot", "\n\n --- Dry run procedure for testing --- \n")
run_30sec(True)
run_weekly(True)
run_yearly(True)
end_mail_procedure()
logger.log("robot", "\n\n --- Done Dry run procedure for testing --- \n")
return
init_loger(full_logfile, args.verbose)
if args.mailing:
end_mail_procedure(False)
return
#
# run maintenance procedures
#
if the_date.minute >= 50:
# hourly or only 10min
if the_date.hour >= 23:
# daily?
if the_date.weekday() == 6:
# weekly?
run_weekly()
else:
run_daily()
else:
run_hourly()
else:
run_10min()
#
# run monthly/yearly procedure
#
if tomorrow_date.day == 1:
# so today is the last month day
if tomorrow_date.year != the_date.year:
# so we are at the last day of the year
run_yearly()
else:
run_monthly()
#
# mailing procedure
#
end_mail_procedure()
logger.log("exec", "--- DONE ---")
