def create_log_directory(host_or_ip, id):
host = host_or_ip.strip().replace('.', '_').replace(' ','_')
date_string = datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S")
directory = get_autlogs_directory() + host + '-' + date_string + '-' + str(id)
if not path.exists(directory):
makedirs(directory)
return host + '-' + date_string + '-' + str(id)
def create_log_directory(host_or_ip, id):
host = host_or_ip.strip().replace('.', '_').replace(' ', '_')
date_string = datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S")
directory = get_autlogs_directory() + host + '-' + date_string + '-' + str(
id)
if not path.exists(directory):
makedirs(directory)
return host + '-' + date_string + '-' + str(id)
def perform_housekeeping_tasks(self, db_session, system_option):
inventory_history_per_host = system_option.inventory_history_per_host
install_history_per_host = system_option.install_history_per_host
download_history_per_user = system_option.download_history_per_user
total_system_logs = system_option.total_system_logs
current_system_logs_count = db_session.query(Log).count()
system_logs_threshold = int(total_system_logs * 1.1)
# If the current system logs count > the threshold (10% more than total_system_logs),
# trim the log table back to the total_system_logs
if current_system_logs_count > system_logs_threshold:
num_records_to_purge = current_system_logs_count - total_system_logs
# Select the logs by created_time in ascending order (older logs)
logs = db_session.query(Log).order_by(
Log.created_time.asc()).limit(num_records_to_purge)
for log in logs:
db_session.delete(log)
db_session.commit()
# Scanning the InventoryJobHistory table for records that should be deleted.
skip_count = 0
current_host_id = -1
inventory_jobs = db_session.query(InventoryJobHistory)\
.order_by(InventoryJobHistory.host_id, InventoryJobHistory.created_time.desc())
for inventory_job in inventory_jobs:
if inventory_job.host_id != current_host_id:
current_host_id = inventory_job.host_id
skip_count = 0
if skip_count >= inventory_history_per_host:
# Delete the session log directory
try:
if inventory_job.session_log is not None:
shutil.rmtree(get_autlogs_directory() +
inventory_job.session_log)
except:
logger.exception(
'InventoryManagerScheduler hit exception- inventory job = %s',
inventory_job.id)
db_session.delete(inventory_job)
skip_count += 1
db_session.commit()
# Scanning the InstallJobHistory table for records that should be deleted.
skip_count = 0
current_host_id = -1
install_jobs = db_session.query(InstallJobHistory)\
.order_by(InstallJobHistory.host_id, InstallJobHistory.created_time.desc())
for install_job in install_jobs:
if install_job.host_id != current_host_id:
current_host_id = install_job.host_id
skip_count = 0
if skip_count >= install_history_per_host:
# Delete the session log directory
try:
if install_job.session_log is not None:
shutil.rmtree(get_autlogs_directory() +
install_job.session_log)
except:
logger.exception(
'InventoryManagerScheduler hit exception - install job = %s',
install_job.id)
db_session.delete(install_job)
skip_count += 1
db_session.commit()
# Scanning the DownloadJobHistory table for records that should be deleted.
skip_count = 0
current_user_id = -1
download_jobs = db_session.query(DownloadJobHistory)\
.order_by(DownloadJobHistory.user_id, DownloadJobHistory.created_time.desc())
for download_job in download_jobs:
if download_job.user_id != current_user_id:
current_user_id = download_job.user_id
skip_count = 0
if skip_count >= download_history_per_user:
db_session.delete(download_job)
skip_count += 1
db_session.commit()
def perform_housekeeping_tasks(self, db_session, system_option):
inventory_history_per_host = system_option.inventory_history_per_host
install_history_per_host = system_option.install_history_per_host
download_history_per_user = system_option.download_history_per_user
total_system_logs = system_option.total_system_logs
current_system_logs_count = db_session.query(Log).count()
system_logs_threshold = int(total_system_logs * 1.1)
# If the current system logs count > the threshold (10% more than total_system_logs),
# trim the log table back to the total_system_logs
if current_system_logs_count > system_logs_threshold:
num_records_to_purge = current_system_logs_count - total_system_logs
# Select the logs by created_time in ascending order (older logs)
logs = db_session.query(Log).order_by(Log.created_time.asc()).limit(num_records_to_purge)
for log in logs:
db_session.delete(log)
db_session.commit()
# Scanning the InventoryJobHistory table for records that should be deleted.
skip_count = 0
current_host_id = -1
inventory_jobs = db_session.query(InventoryJobHistory)\
.order_by(InventoryJobHistory.host_id, InventoryJobHistory.created_time.desc())
for inventory_job in inventory_jobs:
if inventory_job.host_id != current_host_id:
current_host_id = inventory_job.host_id
skip_count = 0
if skip_count >= inventory_history_per_host:
# Delete the session log directory
try:
if inventory_job.session_log is not None:
shutil.rmtree(get_autlogs_directory() + inventory_job.session_log)
except:
logger.exception('InventoryManagerScheduler hit exception- inventory job = %s', inventory_job.id)
db_session.delete(inventory_job)
skip_count += 1
db_session.commit()
# Scanning the InstallJobHistory table for records that should be deleted.
skip_count = 0
current_host_id = -1
install_jobs = db_session.query(InstallJobHistory)\
.order_by(InstallJobHistory.host_id, InstallJobHistory.created_time.desc())
for install_job in install_jobs:
if install_job.host_id != current_host_id:
current_host_id = install_job.host_id
skip_count = 0
if skip_count >= install_history_per_host:
# Delete the session log directory
try:
if install_job.session_log is not None:
shutil.rmtree(get_autlogs_directory() + install_job.session_log)
except:
logger.exception('InventoryManagerScheduler hit exception - install job = %s', install_job.id)
db_session.delete(install_job)
skip_count += 1
db_session.commit()
# Scanning the DownloadJobHistory table for records that should be deleted.
skip_count = 0
current_user_id = -1
download_jobs = db_session.query(DownloadJobHistory)\
.order_by(DownloadJobHistory.user_id, DownloadJobHistory.created_time.desc())
for download_job in download_jobs:
if download_job.user_id != current_user_id:
current_user_id = download_job.user_id
skip_count = 0
if skip_count >= download_history_per_user:
db_session.delete(download_job)
skip_count += 1
db_session.commit()
# By importing models here, it forces creation of tables in the database for a new installation. # This will prevent gunicorn workers from trying to create the database tables all at the same time. # See csmserver launch script import models from utils import create_directory from constants import get_autlogs_directory, get_repository_directory, get_temp_directory # Create the necessary supporting directories create_directory(get_autlogs_directory()) create_directory(get_repository_directory()) create_directory(get_temp_directory())
def log_directory(self):
return get_autlogs_directory() + self.install_job.session_log
def log_directory(self):
return get_autlogs_directory() + self.inventory_job.session_log