def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Task processor', scripts_log, level=logging.DEBUG)
lck, err = lock.get_lock('task_processor')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
logger.log_or_print(
'Task processor execution initiated.', lg, level='info')
db_path, err = config.get_db_path()
if err:
raise Exception(err)
ret, err = tasks_utils.process_tasks()
if err:
raise Exception(err)
except Exception, e:
str = 'Error running the task processor : %s' % e
lock.release_lock('task_processor')
logger.log_or_print(str, lg, level='critical')
return -1
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'IntegralSTOR backup report generation', scripts_log, level=logging.DEBUG)
urb_reports_dir, err = config.get_urbackup_reports_dir_path()
if err:
raise Exception(err)
lck, err = lock.get_lock('generate_backup_report')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print(
'IntegralSTOR backup report generation initiated.', lg, level='info')
ret, err = urbackup.generate_todays_pdf_report()
if err:
raise Exception(err)
except Exception, e:
# print str(e)
lock.release_lock('generate_backup_report')
logger.log_or_print('Error generating IntegralSTOR backup report: %s' %
e, lg, level='critical')
return -1, 'Error generating IntegralSTOR backup report : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Task processor',
scripts_log,
level=logging.DEBUG)
lck, err = lock.get_lock('task_processor')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
logger.log_or_print('Task processor execution initiated.',
lg,
level='info')
db_path, err = config.get_db_path()
if err:
raise Exception(err)
ret, err = tasks_utils.process_tasks()
if err:
raise Exception(err)
except Exception, e:
str = 'Error running the task processor : %s' % e
lock.release_lock('task_processor')
logger.log_or_print(str, lg, level='critical')
return -1
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Export old audit messages', scripts_log, level=logging.DEBUG)
logger.log_or_print(
'Processing export of old audits initiated.', lg, level='info')
if len(sys.argv) != 3:
raise Exception(
'Usage : python export_old_audits.py <min_to_export(default 1000)> <export_count(default 500)>')
min_to_export = int(sys.argv[1])
export_count = int(sys.argv[2])
ret, err = audit.export_old_audits(min_to_export, export_count)
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error exporting old audits: %s' %
e, lg, level='critical')
return -1, 'Error exporting old audits : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Generate manifest',
scripts_log,
level=logging.DEBUG)
logger.log_or_print('Generate manifest initiated.', lg, level='info')
num_args = len(sys.argv)
if num_args > 1:
path = sys.argv[1]
else:
path, err = config.get_system_status_path()
if err:
raise Exception(err)
if not path:
path = '/tmp'
logger.log_or_print("Generating the manifest in %s" % path,
lg,
level='info')
rc, err = gen_manifest(path)
if err:
raise Exception(err)
# print rc
except Exception, e:
str = "Error generating manifest file : %s" % e
logger.log_or_print(str, lg, level='critical')
return -1
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Export old alert', scripts_log, level=logging.DEBUG)
logger.log_or_print(
'Processing export of old alerts initiated.', lg, level='info')
if len(sys.argv) != 2:
raise Exception(
'Usage : python export_old_alerts.py <older_than_x_days>')
else:
days = sys.argv[1]
ret, err = alerts.export_old_alerts(int(days))
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error exporting old alerts: %s' %
e, lg, level='critical')
return -1, 'Error exporting old alerts : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Pool usage stats record', scripts_log, level=logging.DEBUG)
lck, err = lock.get_lock('record_pool_usage_stats')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print(
'Pool usage stats collection initiated.', lg, level='info')
ret, err = _record_pool_usage_stats()
if err:
raise Exception(err)
except Exception, e:
# print str(e)
lock.release_lock('record_pool_usage_stats')
logger.log_or_print('Error collecting pool usage stats : %s' %
e, lg, level='critical')
return -1, 'Error collecting pool usage stats : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Process email queue', scripts_log, level=logging.DEBUG)
logger.log_or_print(
'Processing email queue initiated.', lg, level='info')
lck, err = lock.get_lock('process_email_queue')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
ret, err = mail.process_email_queue()
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error processing email queue: %s' %
e, lg, level='critical')
lock.release_lock('process_email_queue')
return -1, 'Error processing email queue: %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Export old audit messages',
scripts_log,
level=logging.DEBUG)
logger.log_or_print('Processing export of old audits initiated.',
lg,
level='info')
if len(sys.argv) != 3:
raise Exception(
'Usage : python export_old_audits.py <min_to_export(default 1000)> <export_count(default 500)>'
)
min_to_export = int(sys.argv[1])
export_count = int(sys.argv[2])
ret, err = audit.export_old_audits(min_to_export, export_count)
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error exporting old audits: %s' % e,
lg,
level='critical')
return -1, 'Error exporting old audits : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'IntegralSTOR backup report generation',
scripts_log,
level=logging.DEBUG)
urb_reports_dir, err = config.get_urbackup_reports_dir_path()
if err:
raise Exception(err)
lck, err = lock.get_lock('generate_backup_report')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print('IntegralSTOR backup report generation initiated.',
lg,
level='info')
ret, err = urbackup.generate_todays_pdf_report()
if err:
raise Exception(err)
except Exception, e:
# print str(e)
lock.release_lock('generate_backup_report')
logger.log_or_print('Error generating IntegralSTOR backup report: %s' %
e,
lg,
level='critical')
return -1, 'Error generating IntegralSTOR backup report : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Process email queue',
scripts_log,
level=logging.DEBUG)
logger.log_or_print('Processing email queue initiated.',
lg,
level='info')
lck, err = lock.get_lock('process_email_queue')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
ret, err = mail.process_email_queue()
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error processing email queue: %s' % e,
lg,
level='critical')
lock.release_lock('process_email_queue')
return -1, 'Error processing email queue: %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Generate manifest', scripts_log, level=logging.DEBUG)
logger.log_or_print('Generate manifest initiated.', lg, level='info')
num_args = len(sys.argv)
if num_args > 1:
path = sys.argv[1]
else:
path, err = config.get_system_status_path()
if err:
raise Exception(err)
if not path:
path = '/tmp'
logger.log_or_print("Generating the manifest in %s" %
path, lg, level='info')
rc, err = gen_manifest(path)
if err:
raise Exception(err)
# print rc
except Exception, e:
str = "Error generating manifest file : %s" % e
logger.log_or_print(str, lg, level='critical')
return -1
def main():
lg = None
try:
stop_services = False
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'ZFS pool usage check', scripts_log, level=logging.DEBUG)
lck, err = lock.get_lock('check_zfs_pools_usage')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print(
'ZFS pool usage check initiated.', lg, level='info')
if len(sys.argv) != 3:
raise Exception(
'Usage : python check_zfs_pools_usage.py <warning_percentage> <critical_percentage>')
warning_percentage = int(sys.argv[1])
critical_percentage = int(sys.argv[2])
pool_list, err = zfs.get_pools()
if err:
raise Exception(err)
alerts_list = []
for pool_info in pool_list:
percentage = float(pool_info['usage']['used_percent'])
alert = False
if percentage > critical_percentage:
severity_str = 'CRITICAL'
severity_type = 3
alert = True
print_percentage = critical_percentage
logger.log_or_print('ZFS pool %s is %d%% full.' % (
pool_info['pool_name'], int(percentage)), lg, level='critical')
elif percentage > warning_percentage:
severity_type = 2
severity_str = 'warning'
print_percentage = warning_percentage
alert = True
if alert:
alert_str = 'ZFS pool %s has exceeded the %s threshold capacity of %d%% and is now %d%% full.' % (
pool_info['pool_name'], severity_str, print_percentage, percentage)
alerts_list.append({'subsystem_type_id': 6, 'severity_type_id': severity_type,
'component': pool_info['pool_name'], 'alert_str': alert_str})
if alerts_list:
retval, err = alerts.record_alerts(alerts_list)
if err:
raise Exception(err)
except Exception, e:
# print str(e)
lock.release_lock('check_zfs_pools_usage')
logger.log_or_print('Error checking ZFS pool usage: %s' %
e, lg, level='critical')
return -1, 'Error checking ZFS pool usage : %s' % e
def zero_logs():
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
with open(scripts_log, 'w') as f:
pass
except Exception, e:
return False, "Error zeroing logs : %s" % str(e)
def zero_logs():
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
with open(scripts_log, 'w') as f:
pass
except Exception, e:
return False, "Error zeroing logs : %s" % str(e)
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Current logs archive generation', scripts_log, level=logging.DEBUG)
logs_archives_dir, err = config.get_logs_archives_dir_path()
if err:
raise Exception(err)
lck, err = lock.get_lock('generate_current_logs_archive')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print(
'Current logs archive generation initiated.', lg, level='info')
now_local_epoch, err = datetime_utils.get_epoch(when='now')
if err:
raise Exception(err)
now_local_str, err = datetime_utils.convert_from_epoch(
now_local_epoch, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local')
if err:
raise Exception(err)
zf_name = 'IntegralSTOR_system_logs_%s.zip' % now_local_str
try:
os.makedirs(logs_archives_dir)
except:
pass
zf = zipfile.ZipFile('%s/%s' % (logs_archives_dir, zf_name), 'w')
for root, dirs, files in os.walk('/var/log/integralstor'):
if root.startswith('/var/log/integralstor/archives'):
continue
for file in files:
# print '%s/%s'%(root[len('/var/log/integralstor/'):], file)
zf.write(os.path.join(root, file), '%s/%s' %
(root[len('/var/log/integralstor/'):], file))
zf.close()
except Exception, e:
# print str(e)
lock.release_lock('generate_current_logs_archive')
logger.log_or_print('Error generating current logs archive : %s' %
e, lg, level='critical')
return -1, 'Error generating current logs archive: %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Purge database',
scripts_log,
level=logging.DEBUG)
logger.log_or_print('Database purge initiated.', lg, level='info')
if len(sys.argv) != 4:
raise Exception(
'Usage : python purge_db.py <alerts_older_than_x_days> <min_audits_to_export> <audit_export_count'
)
lck, err = lock.get_lock('purge_db')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
alerts_days = sys.argv[1]
audit_min_to_export = int(sys.argv[2])
audit_export_count = int(sys.argv[3])
ret, err = alerts.export_old_alerts(int(alerts_days))
if err:
raise Exception(err)
ret, err = audit.export_old_audits(audit_min_to_export,
audit_export_count)
if err:
raise Exception(err)
ret, err = mail.purge_email_queue(7)
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error purging database: %s' % e,
lg,
level='critical')
lock.release_lock('purge_db')
return -1, 'Error purging database : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Generate status',
scripts_log,
level=logging.DEBUG)
logger.log_or_print('Generate status initiated.', lg, level='info')
platform, err = config.get_platform()
if err:
raise Exception(err)
default_path = False
num_args = len(sys.argv)
if num_args > 1:
path = sys.argv[1]
else:
default_path = True
path, err = config.get_system_status_path()
if err:
raise Exception(err)
if not path:
path = '/tmp'
# print platform, path
logger.log_or_print("Generating the status in %s" % path,
lg,
level='info')
rc, err = gen_status(path, lg)
if err:
raise Exception(err)
# print rc
except Exception, e:
str = "Error generating status file : %s" % e
logger.log_or_print(str, lg, level='critical')
sys.exit(-1)
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Purge database', scripts_log, level=logging.DEBUG)
logger.log_or_print('Database purge initiated.', lg, level='info')
if len(sys.argv) != 4:
raise Exception(
'Usage : python purge_db.py <alerts_older_than_x_days> <min_audits_to_export> <audit_export_count')
lck, err = lock.get_lock('purge_db')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
alerts_days = sys.argv[1]
audit_min_to_export = int(sys.argv[2])
audit_export_count = int(sys.argv[3])
ret, err = alerts.export_old_alerts(int(alerts_days))
if err:
raise Exception(err)
ret, err = audit.export_old_audits(
audit_min_to_export, audit_export_count)
if err:
raise Exception(err)
ret, err = mail.purge_email_queue(7)
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error purging database: %s' %
e, lg, level='critical')
lock.release_lock('purge_db')
return -1, 'Error purging database : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Generate status', scripts_log, level=logging.DEBUG)
logger.log_or_print('Generate status initiated.', lg, level='info')
platform, err = config.get_platform()
if err:
raise Exception(err)
default_path = False
num_args = len(sys.argv)
if num_args > 1:
path = sys.argv[1]
else:
default_path = True
path, err = config.get_system_status_path()
if err:
raise Exception(err)
if not path:
path = '/tmp'
# print platform, path
logger.log_or_print("Generating the status in %s" %
path, lg, level='info')
rc, err = gen_status(path, lg)
if err:
raise Exception(err)
# print rc
except Exception, e:
str = "Error generating status file : %s" % e
logger.log_or_print(str, lg, level='critical')
sys.exit(-1)
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Remove old files', scripts_log, level=logging.DEBUG)
logger.log_or_print('Old file removal initiated.', lg, level='info')
if len(sys.argv) != 2:
raise Exception(
'Usage : python remove_old_files.py <older_than_days>')
older_than_days = int(sys.argv[1])
ret, err = remove_old_files(lg, older_than_days)
if err:
raise Exception(err)
except Exception, e:
str = "Error removing old files: %s" % e
print str
logger.log_or_print(str, lg, level='critical')
sys.exit(-1)
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Remove old files',
scripts_log,
level=logging.DEBUG)
logger.log_or_print('Old file removal initiated.', lg, level='info')
if len(sys.argv) != 2:
raise Exception(
'Usage : python remove_old_files.py <older_than_days>')
older_than_days = int(sys.argv[1])
ret, err = remove_old_files(lg, older_than_days)
if err:
raise Exception(err)
except Exception, e:
str = "Error removing old files: %s" % e
print str
logger.log_or_print(str, lg, level='critical')
sys.exit(-1)
def download_log(request):
""" Download the system log of the type specified in log_type POST param
This calls the /sys_log via an http request on that node to get the info"""
return_dict = {}
try:
hw_platform, err = config.get_hardware_platform()
if err:
raise Exception(err)
if hw_platform and hw_platform != 'dell':
raise Exception('Unknown hardware platform')
return_dict['hw_platform'] = hw_platform
form = log_management_forms.DownloadLogsForm(request.POST or None)
if request.method == 'POST':
if form.is_valid():
cd = form.cleaned_data
log_type = cd['log_type']
if log_type in ['alerts', 'audit', 'hardware']:
response = django.http.HttpResponse()
if log_type == 'alerts':
response['Content-disposition'] = 'attachment; filename=alerts_log.txt'
all_alerts, err = alerts.get_alerts()
if err:
raise Exception(err)
for alert in all_alerts:
if int(alert['repeat_count']) > 1:
response.write('Last alert time %s\nAlert message: %s\nRepeated count: %d\n\n' %
(alert['last_update_time'], alert['alert_str'], int(alert['repeat_count'])))
else:
response.write('Last alert time %s\nAlert message: %s\n\n' %
(alert['last_update_time'], alert['alert_str']))
response.flush()
elif log_type == 'audit':
response['Content-disposition'] = 'attachment; filename=audit_log.txt'
all_audits, err = audit.get_entries()
if err:
raise Exception(err)
for audit_info in all_audits:
response.write('Time : %s \n' % audit_info['time'])
response.write('Source IP : %s \n' %
audit_info['ip'])
response.write('Action : %s \n' %
audit_info['action_str'])
response.write('\n')
response.flush()
elif log_type == 'hardware':
response['Content-disposition'] = 'attachment; filename=hardware_logs.txt'
hw_platform, err = config.get_hardware_platform()
if not hw_platform or hw_platform != 'dell':
raise Exception('Unknown hardware platform')
if hw_platform == 'dell':
from integralstor.platforms import dell
logs_dict, err = dell.get_alert_logs()
if err:
raise Exception(err)
if not logs_dict:
raise Exception('No logs detected!')
for timestamp, log_list in logs_dict.items():
for log in log_list:
response.write('Time : %s\n' %
log['date_time'])
response.write(
'Severity : %s\n' % log['Severity'])
response.write(
'Description : %s\n' % log['description'])
response.write('\n')
response.flush()
else:
raise Exception('Unknown platform')
else:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
system_logs = [('/var/log/boot.log', 'boot.log'), ('/var/log/dmesg', 'dmesg'), ('/var/log/messages', 'messages'),
('/var/log/smblog.vfs', 'samba'), ('/var/log/samba/log.winbindd', 'winbind'), (scripts_log, 'scripts')]
now_local_epoch, err = datetime_utils.get_epoch(when='now')
if err:
raise Exception(err)
now_local_str, err = datetime_utils.convert_from_epoch(
now_local_epoch, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local')
if err:
raise Exception(err)
zf_name = 'IntegralSTOR_system_logs_%s.zip' % now_local_str
try:
out = io.BytesIO()
zf = zipfile.ZipFile(out, 'w')
for entry in system_logs:
zf.write(entry[0], arcname=entry[1])
#zf.write(file_name, arcname=display_name)
zf.close()
except Exception as e:
raise Exception(
"Error compressing log file : %s" % str(e))
response = django.http.HttpResponse(
out.getvalue(), content_type='application/x-compressed')
response['Content-disposition'] = 'attachment; filename=%s' % (
zf_name)
return response
# either a get or an invalid form so send back form
return_dict['form'] = form
return django.shortcuts.render_to_response('download_log_form.html', return_dict, context_instance=django.template.context.RequestContext(request))
except Exception, e:
return_dict['base_template'] = "system_base.html"
return_dict["page_title"] = 'Download system logs'
return_dict['tab'] = 'logs_tab'
return_dict["error"] = 'Error downloading system logs'
return_dict["error_details"] = str(e)
return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Process audit events', scripts_log, level=logging.DEBUG)
num_args = len(sys.argv)
if num_args != 2:
raise Exception(
'Usage : python process_audit_notifications <event_notification_trigger_id>')
else:
ent_id = sys.argv[1]
logger.log_or_print(
'Processing audit notifications initiated.', lg, level='info')
enh_list, err = event_notifications.get_event_notification_holdings(
ent_id, 'by_event_notification_trigger_id')
if err:
raise Exception(err)
ent, err = event_notifications.get_event_notification_trigger(ent_id)
# print ent, err
if err:
raise Exception(err)
if not ent:
raise Exception(
'Could not find the specified event notification trigger')
if enh_list:
processed_successfully = False
if ent['notification_type_id'] == 1:
msg_list = []
for enh in enh_list:
# print enh
# Need to generate an email into the email_queue
msg, err = audit.generate_audit_email_body(enh['event_id'])
# print msg, err
if err:
raise Exception(err)
msg_list.append(msg)
if msg_list:
# Now generate ONE email for all the messages corresponding
# to that trigger..
final_msg = '\n\n------------------------------------------------------\n\n'.join(
msg_list)
# print 'final msg - ', final_msg
enc, err = mail.get_event_notification_configuration(
ent['enc_id'])
# print enc, err
if err:
raise Exception(err)
processed_successfully, err = mail.enqueue(
enc['recipient_list'], "Audit message from IntegralSTOR storage system", final_msg)
# print 'enqueue', processed_successfully, err
if err:
raise Exception(err)
else:
raise Exception('Unknown event notification type.')
if processed_successfully:
# Successfully enqueued so now remove them all from the holding
# table
for enh in enh_list:
r, err = event_notifications.delete_event_notification_holding(
enh['enh_id'])
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error processing audit notifications : %s' %
e, lg, level='critical')
return -1, 'Error processing audit notifications : %s' % e
def main():
lg = None
try:
stop_services = False
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('ZFS pool usage check',
scripts_log,
level=logging.DEBUG)
lck, err = lock.get_lock('check_zfs_pools_usage')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print('ZFS pool usage check initiated.',
lg,
level='info')
if len(sys.argv) != 3:
raise Exception(
'Usage : python check_zfs_pools_usage.py <warning_percentage> <critical_percentage>'
)
warning_percentage = int(sys.argv[1])
critical_percentage = int(sys.argv[2])
pool_list, err = zfs.get_pools()
if err:
raise Exception(err)
alerts_list = []
for pool_info in pool_list:
percentage = float(pool_info['usage']['used_percent'])
alert = False
if percentage > critical_percentage:
severity_str = 'CRITICAL'
severity_type = 3
alert = True
print_percentage = critical_percentage
logger.log_or_print('ZFS pool %s is %d%% full.' %
(pool_info['pool_name'], int(percentage)),
lg,
level='critical')
elif percentage > warning_percentage:
severity_type = 2
severity_str = 'warning'
print_percentage = warning_percentage
alert = True
if alert:
alert_str = 'ZFS pool %s has exceeded the %s threshold capacity of %d%% and is now %d%% full.' % (
pool_info['pool_name'], severity_str, print_percentage,
percentage)
alerts_list.append({
'subsystem_type_id': 6,
'severity_type_id': severity_type,
'component': pool_info['pool_name'],
'alert_str': alert_str
})
if alerts_list:
retval, err = alerts.record_alerts(alerts_list)
if err:
raise Exception(err)
except Exception, e:
# print str(e)
lock.release_lock('check_zfs_pools_usage')
logger.log_or_print('Error checking ZFS pool usage: %s' % e,
lg,
level='critical')
return -1, 'Error checking ZFS pool usage : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('System status report generation',
scripts_log,
level=logging.DEBUG)
status_reports_dir, err = config.get_staus_reports_dir_path()
if err:
raise Exception(err)
lck, err = lock.get_lock('generate_system_status_report')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print('System status report generation initiated.',
lg,
level='info')
if len(sys.argv) != 2:
raise Exception(
'Usage : python generate_system_status_report.py <past_x_days>'
)
past_x_days = int(sys.argv[1])
start_time, err = datetime_utils.get_epoch(
when='midnight', num_previous_days=past_x_days)
if err:
raise Exception(err)
now, err = datetime_utils.get_epoch(when='now')
if err:
raise Exception(err)
now_local_str, err = datetime_utils.convert_from_epoch(
now, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local')
if err:
raise Exception(err)
tmp_file_name = 'integralstor_status_%s' % now_local_str
tmp_file_name_with_path = '/tmp/%s' % tmp_file_name
with open(tmp_file_name_with_path, 'w') as f:
ret, err = generate_global_header(f)
# print ret, err
f.write('\n')
ret, err = generate_dmidecode_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_cpu_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_memory_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(
f, 'nmcli con', 'Networking connections')
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(f, 'ip addr',
'IP addresses')
# print ret, err
f.write('\n\n')
hw_platform, err = config.get_hardware_platform()
# print ret, err
if hw_platform:
if hw_platform == 'dell':
ret, err = generate_dell_hw_status(f)
# print ret, err
f.write('\n\n')
ret, err = generate_disks_status_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(
f,
'df -HT --exclude-type=devtmpfs --exclude-type=tmpfs --exclude-type=zfs',
'OS disk space usage')
# print ret, err
f.write('\n\n')
ret, err = generate_zfs_info_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(f, 'zpool list',
'ZFS pool space usage')
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(
f,
'zfs list -t filesystem -o name,used,avail,refer,mountpoint,dedup,compression,quota,xattr,recordsize,acltype',
'ZFS datasets')
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(
f,
'zfs list -t volume -o name,used,avail,refer,mountpoint,dedup,compression,volsize,volblocksize',
'ZFS zvols')
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(f, 'zpool status -v',
'ZFS pool status')
# print ret, err
f.write('\n\n')
ret, err = generate_audits_section(f, start_time, past_x_days)
# print ret, err
f.write('\n\n')
ret, err = generate_alerts_section(f, start_time, past_x_days)
# print ret, err
f.write('\n\n')
try:
os.makedirs(status_reports_dir)
except:
pass
final_file_name_with_path = '%s/%s' % (status_reports_dir,
tmp_file_name)
shutil.move(tmp_file_name_with_path, final_file_name_with_path)
d, err = mail.load_email_settings()
if not err and d and 'support_email_addresses' in d and d[
'support_email_addresses']:
# Email settings present so send it out to the support email
# address
email_header = '%s - IntegralSTOR system status report' % socket.getfqdn(
)
email_body = 'Please find the latest IntegralSTOR system status report'
processed_successfully, err = mail.enqueue(
d['support_email_addresses'],
email_header,
email_body,
attachment_file_location=final_file_name_with_path,
delete_attachment_file=False)
if err:
raise Exception(err)
except Exception, e:
# print str(e)
lock.release_lock('generate_system_status_report')
logger.log_or_print('Error generating system status report : %s' % e,
lg,
level='critical')
return -1, 'Error generating system status report : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Current configuration archive generation',
scripts_log,
level=logging.DEBUG)
config_archives_dir, err = config.get_config_archives_dir_path()
if err:
raise Exception(err)
lck, err = lock.get_lock('generate_current_config_archive')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print('Current config archive generation initiated.',
lg,
level='info')
db_path, err = config.get_db_path()
if err:
raise Exception(err)
pki_dir, err = config.get_pki_dir()
if err:
raise Exception(err)
config_file_list = [('/etc/samba/smb.conf', 'smb.conf'),
('/etc/krb5.conf', 'krb5.conf'),
(db_path, 'integral_view_config.db'),
('/etc/exports', 'exports'),
('/etc/vsftpd/vsftpd.conf', 'vsftpd.conf'),
('/etc/tgt/targets.conf', 'targets.conf'),
('/etc/resolv.conf', 'resolv.conf'),
('/etc/hosts', 'hosts'), ('/etc/passwd', 'passwd'),
('/etc/group', 'group')]
config_dir_list = [(pki_dir, 'pki')]
now_local_epoch, err = datetime_utils.get_epoch(when='now')
if err:
raise Exception(err)
now_local_str, err = datetime_utils.convert_from_epoch(
now_local_epoch,
return_format='str',
str_format='%Y_%m_%d_%H_%M',
to='local')
if err:
raise Exception(err)
zf_name = 'IntegralSTOR_system_configuration_%s.zip' % now_local_str
try:
os.makedirs(config_archives_dir)
except:
pass
try:
zf = zipfile.ZipFile('%s/%s' % (config_archives_dir, zf_name), 'w')
for entry in config_file_list:
if os.path.exists(entry[0]):
zf.write(entry[0], arcname=entry[1])
for entry in config_dir_list:
if os.path.exists(entry[0]):
if entry[0][-1] == '/':
path = entry[0][:-1]
else:
path = entry[0]
for root, dirs, files in os.walk(path):
base = root[len(path) + 1:]
for file in files:
if base:
zf.write(os.path.join(root, file),
'%s/%s/%s' % (entry[1], base, file))
else:
zf.write(os.path.join(root, file),
'%s/%s' % (entry[1], file))
zf.close()
except Exception as e:
raise Exception("Error compressing log file : %s" % str(e))
except Exception, e:
# print str(e)
lock.release_lock('generate_current_config_archive')
logger.log_or_print('Error generating current config archive : %s' % e,
lg,
level='critical')
return -1, 'Error generating current config archive: %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Process report notifications', scripts_log, level=logging.DEBUG)
num_args = len(sys.argv)
if num_args != 2:
raise Exception(
'Usage : python process_report_notifications <event_notification_trigger_id>')
else:
ent_id = sys.argv[1]
logger.log_or_print(
'Processing report notifications initiated.', lg, level='info')
status_reports_dir, err = config.get_staus_reports_dir_path()
if err:
raise Exception(err)
urb_reports_dir, err = config.get_urbackup_reports_dir_path()
if err:
raise Exception(err)
rr_reports_dir, err = config.get_remote_replication_reports_dir_path()
if err:
raise Exception(err)
ent, err = event_notifications.get_event_notification_trigger(ent_id)
# print ent, err
if err:
raise Exception(err)
if not ent:
raise Exception(
'Could not find the specified event notification trigger')
if ent['notification_type_id'] == 1:
enc, err = mail.get_event_notification_configuration(ent['enc_id'])
# print enc, err
if err:
raise Exception(err)
if ent['event_type_id'] == 3:
attachment_location = None
if ent['event_subtype_id'] == 1:
# System status repor
# Find the latest system status report and mail it out
all_files = glob.glob('%s/*' % status_reports_dir)
latest_file = max(all_files, key=os.path.getctime)
attachment_location = latest_file
email_header = 'IntegralSTOR system status report'
email_body = 'Please find the latest IntegralSTOR system status report'
elif ent['event_subtype_id'] == 2:
# urbackup report processing here
email_header = 'IntegralSTOR backup status report'
email_body = 'Please find the latest IntegralSTOR backup status report'
all_files = glob.glob('%s/*' % urb_reports_dir)
latest_file = max(all_files, key=os.path.getctime)
attachment_location = latest_file
elif ent['event_subtype_id'] == 3:
# remote replication report processing here
ret, err = remote_replication.generate_pdf_report()
if err:
raise exception(err)
email_header = 'IntegralSTOR remote replication status report'
email_body = 'Please find the latest IntegralSTOR remote replication status report'
all_files = glob.glob('%s/*' % rr_reports_dir)
latest_file = max(all_files, key=os.path.getctime)
attachment_location = latest_file
processed_successfully, err = mail.enqueue(
enc['recipient_list'], email_header, email_body, attachment_file_location=attachment_location, delete_attachment_file=False)
# print 'enqueue', processed_successfully, err
if err:
raise Exception(err)
except Exception, e:
# print str(e)
logger.log_or_print('Error processing report notifications : %s' %
e, lg, level='critical')
return -1, 'Error processing report notifications : %s' % e
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Poll for alerts', scripts_log, level=logging.DEBUG)
lck, err = lock.get_lock('integralstor_poll_for_alerts')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
logger.log_or_print('Poll for alerts initiated.', lg, level='info')
now = int(time.time())
db_path, err = config.get_db_path()
if err:
raise Exception(err)
tasks_query = "select * from tasks where last_run_time > '%d' and (status = 'error-retrying' or status = 'failed');" % (
now - 110)
# print "\ntasks_query: ", tasks_query
rows, err = db.get_multiple_rows(db_path, tasks_query)
# print "\nrows: ", rows
if err:
raise Exception(err)
alert_list = None
if rows:
for row in rows:
if row['status'] == 'error-retrying':
alert_list.append({'subsystem_type_id': 7, 'severity_type_id': 2,
'component': row['description'], 'alert_str': "Task: %s failed but will be retried." % row['description']})
elif row['status'] == 'failed':
alert_list.append({'subsystem_type_id': 7, 'severity_type_id': 3,
'component': row['description'], 'alert_str': "Task: %s failed." % row['description']})
# print "\nalert_list: ", alert_list
hw_platform, err = config.get_hardware_platform()
if hw_platform:
if hw_platform == 'dell':
from integralstor.platforms import dell
alerts_dict, err = dell.get_alert_logs()
if alerts_dict:
current_time = int(time.time())
for time_stamp, alerts_list in alerts_dict.items():
for alert_dict in alerts_list:
if alert_dict['Severity'] == 'Critical':
if (current_time - time_stamp) < (60 * 60):
alert_list.append({'subsystem_type_id': 5, 'severity_type_id': 3,
'component': 'Dell Hardware component', 'alert_str': alert_dict['description']})
# print time_stamp, alert_dict
if alert_list:
alerts.record_alerts(alert_list)
lock.release_lock('integralstor_poll_for_alerts')
except Exception, e:
print "Error generating alerts : %s ! Exiting." % str(e)
logger.log_or_print('Error polling for alerts : %s' %
e, lg, level='critical')
return -1
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Poll for alerts',
scripts_log,
level=logging.DEBUG)
lck, err = lock.get_lock('integralstor_poll_for_alerts')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
logger.log_or_print('Poll for alerts initiated.', lg, level='info')
now = int(time.time())
db_path, err = config.get_db_path()
if err:
raise Exception(err)
tasks_query = "select * from tasks where last_run_time > '%d' and (status = 'error-retrying' or status = 'failed');" % (
now - 110)
# print "\ntasks_query: ", tasks_query
rows, err = db.get_multiple_rows(db_path, tasks_query)
# print "\nrows: ", rows
if err:
raise Exception(err)
alert_list = None
if rows:
for row in rows:
if row['status'] == 'error-retrying':
alert_list.append({
'subsystem_type_id':
7,
'severity_type_id':
2,
'component':
row['description'],
'alert_str':
"Task: %s failed but will be retried." %
row['description']
})
elif row['status'] == 'failed':
alert_list.append({
'subsystem_type_id':
7,
'severity_type_id':
3,
'component':
row['description'],
'alert_str':
"Task: %s failed." % row['description']
})
# print "\nalert_list: ", alert_list
hw_platform, err = config.get_hardware_platform()
if hw_platform:
if hw_platform == 'dell':
from integralstor.platforms import dell
alerts_dict, err = dell.get_alert_logs()
if alerts_dict:
current_time = int(time.time())
for time_stamp, alerts_list in alerts_dict.items():
for alert_dict in alerts_list:
if alert_dict['Severity'] == 'Critical':
if (current_time - time_stamp) < (60 * 60):
alert_list.append({
'subsystem_type_id':
5,
'severity_type_id':
3,
'component':
'Dell Hardware component',
'alert_str':
alert_dict['description']
})
# print time_stamp, alert_dict
if alert_list:
alerts.record_alerts(alert_list)
lock.release_lock('integralstor_poll_for_alerts')
except Exception, e:
print "Error generating alerts : %s ! Exiting." % str(e)
logger.log_or_print('Error polling for alerts : %s' % e,
lg,
level='critical')
return -1
def main():
lg = None
try:
stop_services = False
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'OS file system check', scripts_log, level=logging.DEBUG)
lck, err = lock.get_lock('check_os_filesystems')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print('OS filesystem check initiated.', lg, level='info')
if len(sys.argv) != 3:
raise Exception(
'Usage : python check_os_filesystems.py <warning_percentage> <critical_percentage>')
warning_percentage = int(sys.argv[1])
critical_percentage = int(sys.argv[2])
os_disk_stats, err = disks.get_os_partition_stats()
if err:
raise Exception(err)
alerts_list = []
for partition in os_disk_stats:
fs_name = partition['fs_name']
percentage_used = 100 - partition['percentage_free']
alert = False
if percentage_used > critical_percentage:
severity_str = 'CRITICAL'
severity_type = 3
alert = True
print_percentage = critical_percentage
if '/var' in fs_name:
# print 'stopping services'
stop_services = True
logger.log_or_print(
'OS filesystem %s full. Stopping all data services.' % fs_name, lg, level='critical')
elif percentage_used > warning_percentage:
severity_type = 2
severity_str = 'warning'
print_percentage = warning_percentage
alert = True
if alert:
alert_str = 'Partition %s has exceeded the %s threshold capacity of %d%% and is now %d%% full.' % (
fs_name, severity_str, print_percentage, percentage_used)
if severity_type == 3:
alert_str += ' Stopping all data services now. Please clear up space before resuming these services.'
alerts_list.append({'subsystem_type_id': 8, 'severity_type_id': severity_type,
'component': fs_name, 'alert_str': alert_str})
if alerts_list:
retval, err = alerts.record_alerts(alerts_list)
if err:
raise Exception(err)
if stop_services:
services = ['smb', 'winbind', 'nfs', 'vsftpd', 'urbackup-server']
for service_name in services:
services_management.update_service_status(service_name, 'stop')
except Exception, e:
# print str(e)
lock.release_lock('check_os_filesystems')
logger.log_or_print('Error checking OS filesystems : %s' %
e, lg, level='critical')
return -1, 'Error checking OS filesystems : %s' % e
def main():
lg = None
try:
stop_services = False
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('OS file system check',
scripts_log,
level=logging.DEBUG)
lck, err = lock.get_lock('check_os_filesystems')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print('OS filesystem check initiated.', lg, level='info')
if len(sys.argv) != 3:
raise Exception(
'Usage : python check_os_filesystems.py <warning_percentage> <critical_percentage>'
)
warning_percentage = int(sys.argv[1])
critical_percentage = int(sys.argv[2])
os_disk_stats, err = disks.get_os_partition_stats()
if err:
raise Exception(err)
alerts_list = []
for partition in os_disk_stats:
fs_name = partition['fs_name']
percentage_used = 100 - partition['percentage_free']
alert = False
if percentage_used > critical_percentage:
severity_str = 'CRITICAL'
severity_type = 3
alert = True
print_percentage = critical_percentage
if '/var' in fs_name:
# print 'stopping services'
stop_services = True
logger.log_or_print(
'OS filesystem %s full. Stopping all data services.' %
fs_name,
lg,
level='critical')
elif percentage_used > warning_percentage:
severity_type = 2
severity_str = 'warning'
print_percentage = warning_percentage
alert = True
if alert:
alert_str = 'Partition %s has exceeded the %s threshold capacity of %d%% and is now %d%% full.' % (
fs_name, severity_str, print_percentage, percentage_used)
if severity_type == 3:
alert_str += ' Stopping all data services now. Please clear up space before resuming these services.'
alerts_list.append({
'subsystem_type_id': 8,
'severity_type_id': severity_type,
'component': fs_name,
'alert_str': alert_str
})
if alerts_list:
retval, err = alerts.record_alerts(alerts_list)
if err:
raise Exception(err)
if stop_services:
services = ['smb', 'winbind', 'nfs', 'vsftpd', 'urbackup-server']
for service_name in services:
services_management.update_service_status(service_name, 'stop')
except Exception, e:
# print str(e)
lock.release_lock('check_os_filesystems')
logger.log_or_print('Error checking OS filesystems : %s' % e,
lg,
level='critical')
return -1, 'Error checking OS filesystems : %s' % e