def generate_dell_hw_status(f):
try:
hw_platform, err = config.get_hardware_platform()
if hw_platform:
if hw_platform == 'dell':
f.write(
'--------------------- Dell hardware status BEGIN ------------------------\n\n'
)
from integralstor.platforms import dell
psu_status, err = dell.get_psu_status()
if err:
raise Exception(err)
f.write('Number of PSUs detected : %d\n' %
psu_status['psu_count'])
f.write('PSU redundancy : ' +
('Redundancy present' if psu_status['redundancy'] else
'No redundancy') + '\n')
for index, psu in enumerate(psu_status['psu_list']):
f.write('PSU %d prescence' % index + (
'Present' if psu_psu['prescence'] else 'Not present') +
'\n')
f.write('PSU %d switch on status' % index +
('On' if psu_psu['switch_on'] else 'Off') + '\n')
f.write('PSU %d status' % index +
('Failed' if psu_psu['failure'] else 'OK') + '\n')
f.write('\n')
f.write(
'--------------------- Dell hardware status END ------------------------\n\n'
)
except Exception, e:
return False, 'Error generating Dell hardware statussection: %s' % str(
e)
class ViewLogsForm(forms.Form):
""" Form to get the info about which log to view"""
ch = [('alerts', 'Alerts log'), ('audit', 'Audit log')]
hw_platform, err = config.get_hardware_platform()
if hw_platform and hw_platform == 'dell':
ch.append(('hardware', 'Hardware log'))
log_type = forms.ChoiceField(choices=ch)
def get_ipmi_status():
"""Hardware status using IPMI."""
ipmi_status = []
try:
fil = os.popen("ipmitool sdr")
str4 = fil.read()
lines = re.split("\r?\n", str4)
ipmi_status = []
for line in lines:
l = line.rstrip()
if not l:
continue
# print l
comp_list = l.split('|')
comp = comp_list[0].strip()
status = comp_list[2].strip()
hw_platform, err = config.get_hardware_platform()
if status != "ns":
if hw_platform and hw_platform == 'dell':
if ('fan' not in comp.lower()) and ('PS' not in comp) and ('voltage' not in comp.lower()) and ('current' not in comp.lower()) and (comp not in ['Inlet Temp', 'CMOS Battery', 'Pwr Consumption', 'Temp']):
continue
td = {}
td["parameter_name"] = comp_list[0].strip()
td["reading"] = comp_list[1].strip()
td["status"] = comp_list[2].strip()
'''
if comp in ["CPU Temp", "CPU1 Temperature"]:
td["parameter_name"] = "CPU Temperature"
td["component_name"] = "CPU"
elif comp in ["System Temp", "MB1 Temperature"]:
td["parameter_name"] = "System Temperature"
td["component_name"] = "System"
elif comp == "DIMMA1 Temp":
td["parameter_name"] = "Memory card 1 temperature"
td["component_name"] = "Memory card 1"
elif comp == "DIMMA2 Temp":
td["parameter_name"] = "Memory card 2 temperature"
td["component_name"] = "Memory card 2"
elif comp == "DIMMA3 Temp":
td["parameter_name"] = "Memory card 3 temperature"
td["component_name"] = "Memory card 3"
elif comp == "FAN1":
td["parameter_name"] = "Fan 1 speed"
td["component_name"] = "Fan 1"
elif comp == "FAN2":
td["parameter_name"] = "Fan 2 speed"
td["component_name"] = "Fan 2"
elif comp == "FAN3":
td["parameter_name"] = "Fan 3 speed"
td["component_name"] = "Fan 3"
elif comp == "CPU_FAN1":
td["parameter_name"] = "CPU Fan"
td["component_name"] = "CPU Fan"
'''
ipmi_status.append(td)
except Exception, e:
return None, 'Error retrieving IPMI information: %s' % str(e)
class DownloadLogsForm(forms.Form):
""" Form to get the info about which log to download"""
# ch = [('boot', 'Boot log'), ('dmesg', 'Dmesg log'), ('message', 'Message log'), ('smb', 'Samba logs'),
# ('winbind', 'Samba Winbind logs'), ('alerts', 'Alerts log'), ('audit', 'Audit log')]
ch = [('system', 'System logs'), ('alerts', 'Alerts log'),
('audit', 'Audit log')]
hw_platform, err = config.get_hardware_platform()
if hw_platform and hw_platform == 'dell':
ch.append(('hardware', 'Hardware log'))
log_type = forms.ChoiceField(choices=ch)
def view_disks(request):
return_dict = {}
type = 'data'
try:
if "ack" in request.GET:
if request.GET["ack"] == "blink":
return_dict['ack_message'] = "Disk identification LED successfully activated"
elif request.GET["ack"] == "unblink":
return_dict['ack_message'] = "Disk identification LED successfully de-activated"
ret, err = django_utils.get_request_parameter_values(
request, ['type'])
if err:
raise Exception(err)
if ('type' not in ret) or ret['type'] not in ['data', 'os']:
type = 'data'
else:
type = ret['type']
si, err = system_info.load_system_config()
if err:
raise Exception(err)
if not si:
raise Exception('Error loading system configuration')
hw_platform, err = config.get_hardware_platform()
if hw_platform:
return_dict['hw_platform'] = hw_platform
if hw_platform == 'dell':
from integralstor.platforms import dell
idrac_url, err = dell.get_idrac_addr()
if idrac_url:
return_dict['idrac_url'] = idrac_url
if type == 'os':
os_disk_stats, err = disks.get_os_partition_stats()
if err:
raise Exception(err)
return_dict['os_disk_stats'] = os_disk_stats
return_dict['node'] = si
return_dict['system_info'] = si
return_dict["disk_status"] = si['disks']
# print si['disks']
return_dict['node_name'] = si['fqdn']
if type == 'os':
return django.shortcuts.render_to_response('view_os_disks.html', return_dict, context_instance=django.template.context.RequestContext(request))
else:
return django.shortcuts.render_to_response('view_data_disks.html', return_dict, context_instance=django.template.context.RequestContext(request))
except Exception, e:
return_dict['base_template'] = "storage_base.html"
return_dict["page_title"] = 'Disks'
return_dict['tab'] = 'view_%s_disks_tab' % type
return_dict["error"] = 'Error loading disk information'
return_dict["error_details"] = str(e)
return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def get_hardware_specific_status():
return_dict = {}
try:
hw_platform, err = config.get_hardware_platform()
if hw_platform:
return_dict['hw_platform'] = hw_platform
if hw_platform == 'dell':
return_dict['hw_platform'] = 'dell'
from integralstor.platforms import dell
idrac_url, err = dell.get_idrac_addr()
if idrac_url:
return_dict['idrac_url'] = idrac_url
except Exception, e:
return None, 'Error retrieving hardware specific status: %s' % str(e)
def get_hardware_specific_status():
return_dict = {}
try:
hw_platform, err = config.get_hardware_platform()
if hw_platform:
return_dict['hw_platform'] = hw_platform
if hw_platform == 'dell':
return_dict['hw_platform'] = 'dell'
from integralstor.platforms import dell
idrac_url, err = dell.get_idrac_addr()
if idrac_url:
return_dict['idrac_url'] = idrac_url
except Exception, e:
return None, 'Error retrieving hardware specific status: %s' % str(e)
def view_hardware_logs(request):
return_dict = {}
try:
hw_platform, err = config.get_hardware_platform()
if err:
raise Exception(err)
if hw_platform and hw_platform != 'dell':
raise Exception('Unknown hardware platform')
return_dict['hw_platform'] = hw_platform
if hw_platform == 'dell':
from integralstor.platforms import dell
logs_dict, err = dell.get_alert_logs()
if logs_dict:
return_dict['logs_dict'] = logs_dict
return django.shortcuts.render_to_response('view_hardware_logs.html', return_dict, context_instance=django.template.context.RequestContext(request))
except Exception, e:
return_dict['base_template'] = "system_base.html"
return_dict["page_title"] = 'View and download logs'
return_dict['tab'] = 'logs_tab'
return_dict["error"] = 'Error loading hardware logs'
return_dict["error_details"] = str(e)
return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('System status report generation',
scripts_log,
level=logging.DEBUG)
status_reports_dir, err = config.get_staus_reports_dir_path()
if err:
raise Exception(err)
lck, err = lock.get_lock('generate_system_status_report')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock.')
logger.log_or_print('System status report generation initiated.',
lg,
level='info')
if len(sys.argv) != 2:
raise Exception(
'Usage : python generate_system_status_report.py <past_x_days>'
)
past_x_days = int(sys.argv[1])
start_time, err = datetime_utils.get_epoch(
when='midnight', num_previous_days=past_x_days)
if err:
raise Exception(err)
now, err = datetime_utils.get_epoch(when='now')
if err:
raise Exception(err)
now_local_str, err = datetime_utils.convert_from_epoch(
now, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local')
if err:
raise Exception(err)
tmp_file_name = 'integralstor_status_%s' % now_local_str
tmp_file_name_with_path = '/tmp/%s' % tmp_file_name
with open(tmp_file_name_with_path, 'w') as f:
ret, err = generate_global_header(f)
# print ret, err
f.write('\n')
ret, err = generate_dmidecode_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_cpu_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_memory_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(
f, 'nmcli con', 'Networking connections')
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(f, 'ip addr',
'IP addresses')
# print ret, err
f.write('\n\n')
hw_platform, err = config.get_hardware_platform()
# print ret, err
if hw_platform:
if hw_platform == 'dell':
ret, err = generate_dell_hw_status(f)
# print ret, err
f.write('\n\n')
ret, err = generate_disks_status_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(
f,
'df -HT --exclude-type=devtmpfs --exclude-type=tmpfs --exclude-type=zfs',
'OS disk space usage')
# print ret, err
f.write('\n\n')
ret, err = generate_zfs_info_section(f)
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(f, 'zpool list',
'ZFS pool space usage')
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(
f,
'zfs list -t filesystem -o name,used,avail,refer,mountpoint,dedup,compression,quota,xattr,recordsize,acltype',
'ZFS datasets')
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(
f,
'zfs list -t volume -o name,used,avail,refer,mountpoint,dedup,compression,volsize,volblocksize',
'ZFS zvols')
# print ret, err
f.write('\n\n')
ret, err = generate_command_based_section(f, 'zpool status -v',
'ZFS pool status')
# print ret, err
f.write('\n\n')
ret, err = generate_audits_section(f, start_time, past_x_days)
# print ret, err
f.write('\n\n')
ret, err = generate_alerts_section(f, start_time, past_x_days)
# print ret, err
f.write('\n\n')
try:
os.makedirs(status_reports_dir)
except:
pass
final_file_name_with_path = '%s/%s' % (status_reports_dir,
tmp_file_name)
shutil.move(tmp_file_name_with_path, final_file_name_with_path)
d, err = mail.load_email_settings()
if not err and d and 'support_email_addresses' in d and d[
'support_email_addresses']:
# Email settings present so send it out to the support email
# address
email_header = '%s - IntegralSTOR system status report' % socket.getfqdn(
)
email_body = 'Please find the latest IntegralSTOR system status report'
processed_successfully, err = mail.enqueue(
d['support_email_addresses'],
email_header,
email_body,
attachment_file_location=final_file_name_with_path,
delete_attachment_file=False)
if err:
raise Exception(err)
except Exception, e:
# print str(e)
lock.release_lock('generate_system_status_report')
logger.log_or_print('Error generating system status report : %s' % e,
lg,
level='critical')
return -1, 'Error generating system status report : %s' % e
def view_dashboard(request, page = None):
return_dict = {}
try:
return_dict["page_title"] = 'Overall system health'
return_dict['tab'] = 'system_health_tab'
return_dict["error"] = 'Error loading system health data'
if request.method != 'GET':
raise Exception('Invalid access method. Please use the menus')
si, err = system_info.load_system_config()
if err:
raise Exception(err)
if not si:
raise Exception('Error loading system configuration')
#node_name = si.keys()[0]
#node = si[node_name]
return_dict['node'] = si
# print node.keys()
# By default show error page
template = "logged_in_error.html"
# Chart specific declarations
# will return 02, instead of 2.
end_epoch, err = datetime_utils.get_epoch(when='now')
if err:
raise Exception(err)
start_epoch = end_epoch - 3 * 60 * 60
start, err = datetime_utils.convert_from_epoch(
start_epoch, return_format='str', str_format='%H:%M:%S', to='local')
if err:
raise Exception(err)
end, err = datetime_utils.convert_from_epoch(
end_epoch, return_format='str', str_format='%H:%M:%S', to='local')
if err:
raise Exception(err)
todays_date = (datetime.date.today()).strftime('%02d')
value_list = []
time_list = []
num_bad_disks = 0
num_hw_raid_bad_disks = 0
num_hw_raid_ctrl_disks = 0
num_smart_ctrl_disks = 0
num_disks = len(si['disks'])
disks_ok = True
for sn, disk in si['disks'].items():
if 'status' in disk:
if 'hw_raid' in disk:
if not disk['hw_raid']:
num_smart_ctrl_disks += 1
if (disk['status'] is not None and disk['status'].upper() not in ['PASSED', 'OK']):
num_bad_disks += 1
disks_ok = False
else:
num_hw_raid_ctrl_disks += 1
if (disk['status'] is not None and disk['status'].upper() != 'OK'):
num_hw_raid_bad_disks += 1
disks_ok = False
else:
# Assume its a non raid disk
num_smart_ctrl_disks += 1
if (disk['status'] is not None and disk['status'].upper() not in ['PASSED', 'OK']):
num_bad_disks += 1
disks_ok = False
return_dict['num_disks'] = num_disks
return_dict['num_bad_disks'] = num_bad_disks
return_dict['disks_ok'] = disks_ok
return_dict['num_hw_raid_bad_disks'] = num_hw_raid_bad_disks
return_dict['num_hw_raid_ctrl_disks'] = num_hw_raid_ctrl_disks
return_dict['num_smart_ctrl_disks'] = num_smart_ctrl_disks
if 'ipmi_status' in si:
num_sensors = len(si['ipmi_status'])
num_bad_sensors = 0
ipmi_ok = True
for sensor in si['ipmi_status']:
if sensor['status'] in ['ok', 'nr', 'na']:
continue
else:
num_bad_sensors += 1
ipmi_ok = False
return_dict['num_sensors'] = num_sensors
return_dict['num_bad_sensors'] = num_bad_sensors
return_dict['ipmi_ok'] = ipmi_ok
services_dict, err = services_management.get_sysd_services_status()
if err:
raise Exception(err)
num_services = len(services_dict)
num_failed_services = 0
num_active_services = 0
num_inactive_services = 0
services_ok = True
if services_dict:
for service, service_d in services_dict.items():
if service_d["info"]["status"]["status_str"] == "Active":
num_active_services += 1
elif service_d["info"]["status"]["status_str"] == "Inactive":
num_inactive_services += 1
elif service_d["info"]["status"]["status_str"] == "Failed":
num_failed_services += 1
services_ok = False
elif service_d["info"]["status"]["status_str"] == "Unknown State":
num_failed_services += 1
services_ok = False
return_dict['num_services'] = num_services
return_dict['num_active_services'] = num_active_services
return_dict['num_inactive_services'] = num_inactive_services
return_dict['num_failed_services'] = num_failed_services
return_dict['services_ok'] = services_ok
else:
raise Exception('Error retrieving services status')
pools, err = zfs.get_pools()
if err:
raise Exception(err)
num_pools = len(pools)
num_bad_pools = 0
num_degraded_pools = 0
num_high_usage_pools = 0
for pool in pools:
if pool['usage']['used_percent'] > 75:
num_high_usage_pools += 1
if pool['config']['pool']['root']['status']['state'] == 'ONLINE':
pass
elif pool['config']['pool']['root']['status']['state'] == 'DEGRADED':
num_degraded_pools += 1
else:
num_bad_pools += 1
return_dict['num_pools'] = num_pools
return_dict['num_bad_pools'] = num_bad_pools
return_dict['num_degraded_pools'] = num_degraded_pools
return_dict['num_high_usage_pools'] = num_high_usage_pools
load_avg_ok = True
if (si["load_avg"]["5_min"] > si["load_avg"]["cpu_cores"]) or (si["load_avg"]["15_min"] > si["load_avg"]["cpu_cores"]):
load_avg_ok = False
return_dict['load_avg_ok'] = load_avg_ok
shares_list, err = cifs.get_shares_list()
if err:
raise Exception(err)
return_dict['num_cifs_shares'] = len(shares_list)
exports_list, err = nfs.load_exports_list()
if err:
raise Exception(err)
return_dict['num_nfs_exports'] = len(exports_list)
target_list, err = iscsi_stgt.get_targets()
if err:
raise Exception(err)
return_dict['num_iscsi_targets'] = len(target_list)
with open('/proc/uptime', 'r') as f:
uptime_seconds = float(f.readline().split()[0])
uptime_str = '%s hours' % (
':'.join(str(datetime.timedelta(seconds=uptime_seconds)).split(':')[:2]))
return_dict['uptime_str'] = uptime_str
# CPU status
if not page:
page = "sys_health"
if page == "cpu":
return_dict["page_title"] = 'CPU statistics'
return_dict['tab'] = 'cpu_tab'
return_dict["error"] = 'Error loading CPU statistics'
cpu, err = stats.get_system_stats(todays_date, start, end, "cpu")
if err:
raise Exception(err)
value_dict = {}
if cpu:
for key in cpu.keys():
value_list = []
time_list = []
if key == "date":
pass
else:
if cpu[key]:
for a in cpu[key]:
time_list.append(a[0])
value_list.append(a[1])
value_dict[key] = value_list
return_dict["data_dict"] = value_dict
queue, err = stats.get_system_stats(
todays_date, start, end, "queue")
if err:
raise Exception(err)
value_dict = {}
if queue:
for key in queue.keys():
value_list = []
time_list = []
if key == "date":
pass
else:
for a in queue[key]:
time_list.append(a[0])
value_list.append(a[1])
value_dict[key] = value_list
return_dict["data_dict_queue"] = value_dict
return_dict['node'] = si
d = {}
template = "view_cpu_stats.html"
elif page == "sys_health":
return_dict["page_title"] = 'Overall system health'
return_dict['tab'] = 'system_health_tab'
return_dict["error"] = 'Error loading system health data'
template = "view_dashboard.html"
hw_platform, err = config.get_hardware_platform()
if hw_platform:
return_dict['hw_platform'] = hw_platform
if hw_platform == 'dell':
from integralstor.platforms import dell
idrac_url, err = dell.get_idrac_addr()
if idrac_url:
return_dict['idrac_url'] = idrac_url
# Memory
elif page == "memory":
return_dict["page_title"] = 'Memory statistics'
return_dict['tab'] = 'memory_tab'
return_dict["error"] = 'Error loading memory statistics'
mem, err = stats.get_system_stats(
todays_date, start, end, "memory")
if err:
raise Exception(err)
if mem:
for a in mem["memused"]:
time_list.append(a[0])
value_list.append((a[1] / (1024 * 1024)))
return_dict['memory_status'] = si['memory']
template = "view_memory_stats.html"
# Network
elif page == "network":
return_dict["page_title"] = 'Network statistics'
return_dict['tab'] = 'network_tab'
return_dict["error"] = 'Error loading Network statistics'
network, err = stats.get_system_stats(
todays_date, start, end, "network")
if err:
raise Exception(err)
value_dict = {}
if network:
for key in network.keys():
value_list = []
time_list = []
if key == "date" or key == "lo":
pass
else:
for a in network[key]["ifutil-percent"]:
time_list.append(a[0])
value_list.append(a[1])
value_dict[key] = value_list
return_dict["data_dict"] = value_dict
return_dict["network_status"] = si['interfaces']
template = "view_network_stats.html"
return_dict["labels"] = time_list
return_dict["data"] = value_list
return django.shortcuts.render_to_response(template, return_dict, context_instance=django.template.context.RequestContext(request))
except Exception, e:
return_dict['base_template'] = "monitoring_base.html"
return_dict["error_details"] = str(e)
return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def download_log(request):
""" Download the system log of the type specified in log_type POST param
This calls the /sys_log via an http request on that node to get the info"""
return_dict = {}
try:
hw_platform, err = config.get_hardware_platform()
if err:
raise Exception(err)
if hw_platform and hw_platform != 'dell':
raise Exception('Unknown hardware platform')
return_dict['hw_platform'] = hw_platform
form = log_management_forms.DownloadLogsForm(request.POST or None)
if request.method == 'POST':
if form.is_valid():
cd = form.cleaned_data
log_type = cd['log_type']
if log_type in ['alerts', 'audit', 'hardware']:
response = django.http.HttpResponse()
if log_type == 'alerts':
response['Content-disposition'] = 'attachment; filename=alerts_log.txt'
all_alerts, err = alerts.get_alerts()
if err:
raise Exception(err)
for alert in all_alerts:
if int(alert['repeat_count']) > 1:
response.write('Last alert time %s\nAlert message: %s\nRepeated count: %d\n\n' %
(alert['last_update_time'], alert['alert_str'], int(alert['repeat_count'])))
else:
response.write('Last alert time %s\nAlert message: %s\n\n' %
(alert['last_update_time'], alert['alert_str']))
response.flush()
elif log_type == 'audit':
response['Content-disposition'] = 'attachment; filename=audit_log.txt'
all_audits, err = audit.get_entries()
if err:
raise Exception(err)
for audit_info in all_audits:
response.write('Time : %s \n' % audit_info['time'])
response.write('Source IP : %s \n' %
audit_info['ip'])
response.write('Action : %s \n' %
audit_info['action_str'])
response.write('\n')
response.flush()
elif log_type == 'hardware':
response['Content-disposition'] = 'attachment; filename=hardware_logs.txt'
hw_platform, err = config.get_hardware_platform()
if not hw_platform or hw_platform != 'dell':
raise Exception('Unknown hardware platform')
if hw_platform == 'dell':
from integralstor.platforms import dell
logs_dict, err = dell.get_alert_logs()
if err:
raise Exception(err)
if not logs_dict:
raise Exception('No logs detected!')
for timestamp, log_list in logs_dict.items():
for log in log_list:
response.write('Time : %s\n' %
log['date_time'])
response.write(
'Severity : %s\n' % log['Severity'])
response.write(
'Description : %s\n' % log['description'])
response.write('\n')
response.flush()
else:
raise Exception('Unknown platform')
else:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
system_logs = [('/var/log/boot.log', 'boot.log'), ('/var/log/dmesg', 'dmesg'), ('/var/log/messages', 'messages'),
('/var/log/smblog.vfs', 'samba'), ('/var/log/samba/log.winbindd', 'winbind'), (scripts_log, 'scripts')]
now_local_epoch, err = datetime_utils.get_epoch(when='now')
if err:
raise Exception(err)
now_local_str, err = datetime_utils.convert_from_epoch(
now_local_epoch, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local')
if err:
raise Exception(err)
zf_name = 'IntegralSTOR_system_logs_%s.zip' % now_local_str
try:
out = io.BytesIO()
zf = zipfile.ZipFile(out, 'w')
for entry in system_logs:
zf.write(entry[0], arcname=entry[1])
#zf.write(file_name, arcname=display_name)
zf.close()
except Exception as e:
raise Exception(
"Error compressing log file : %s" % str(e))
response = django.http.HttpResponse(
out.getvalue(), content_type='application/x-compressed')
response['Content-disposition'] = 'attachment; filename=%s' % (
zf_name)
return response
# either a get or an invalid form so send back form
return_dict['form'] = form
return django.shortcuts.render_to_response('download_log_form.html', return_dict, context_instance=django.template.context.RequestContext(request))
except Exception, e:
return_dict['base_template'] = "system_base.html"
return_dict["page_title"] = 'Download system logs'
return_dict['tab'] = 'logs_tab'
return_dict["error"] = 'Error downloading system logs'
return_dict["error_details"] = str(e)
return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger(
'Poll for alerts', scripts_log, level=logging.DEBUG)
lck, err = lock.get_lock('integralstor_poll_for_alerts')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
logger.log_or_print('Poll for alerts initiated.', lg, level='info')
now = int(time.time())
db_path, err = config.get_db_path()
if err:
raise Exception(err)
tasks_query = "select * from tasks where last_run_time > '%d' and (status = 'error-retrying' or status = 'failed');" % (
now - 110)
# print "\ntasks_query: ", tasks_query
rows, err = db.get_multiple_rows(db_path, tasks_query)
# print "\nrows: ", rows
if err:
raise Exception(err)
alert_list = None
if rows:
for row in rows:
if row['status'] == 'error-retrying':
alert_list.append({'subsystem_type_id': 7, 'severity_type_id': 2,
'component': row['description'], 'alert_str': "Task: %s failed but will be retried." % row['description']})
elif row['status'] == 'failed':
alert_list.append({'subsystem_type_id': 7, 'severity_type_id': 3,
'component': row['description'], 'alert_str': "Task: %s failed." % row['description']})
# print "\nalert_list: ", alert_list
hw_platform, err = config.get_hardware_platform()
if hw_platform:
if hw_platform == 'dell':
from integralstor.platforms import dell
alerts_dict, err = dell.get_alert_logs()
if alerts_dict:
current_time = int(time.time())
for time_stamp, alerts_list in alerts_dict.items():
for alert_dict in alerts_list:
if alert_dict['Severity'] == 'Critical':
if (current_time - time_stamp) < (60 * 60):
alert_list.append({'subsystem_type_id': 5, 'severity_type_id': 3,
'component': 'Dell Hardware component', 'alert_str': alert_dict['description']})
# print time_stamp, alert_dict
if alert_list:
alerts.record_alerts(alert_list)
lock.release_lock('integralstor_poll_for_alerts')
except Exception, e:
print "Error generating alerts : %s ! Exiting." % str(e)
logger.log_or_print('Error polling for alerts : %s' %
e, lg, level='critical')
return -1
def get_disk_info(disk_name):
"""Get all details about a disk given its name """
# Given a disk name like sda, get the disk info
return_list = []
try:
disk_id, err = get_disk_id(disk_name)
if err:
raise Exception(err)
capacity, err = get_capacity(disk_name)
if err:
raise Exception(err)
rotational, err = is_rotational(disk_name)
if err:
raise Exception(err)
os_partitions, err = get_os_partitions()
if err:
raise Exception(err)
partitions, err = get_partitions(disk_name)
if err:
raise Exception(err)
raid = False
hw_platform, err = config.get_hardware_platform()
if hw_platform and hw_platform == 'dell':
from platforms import dell
raid_to_unix_mapping, err = dell.get_hardware_raid_to_unix_mapping()
if raid_to_unix_mapping and '/dev/%s' % disk_name in raid_to_unix_mapping.keys():
# Its a hardware RAID so pull the underlying info
raid = True
controller_id = raid_to_unix_mapping['/dev/%s' %
disk_name]['controller_number']
device_id = raid_to_unix_mapping['/dev/%s' %
disk_name]['device_id']
disk_list, err = dell.get_hardware_raid_hard_drives(
controller_id, device_id)
if disk_list:
for disk in disk_list:
disk_dict = {}
disk_dict['serial_number'] = disk['serial_number']
#disk_dict['name'] = disk_name
#disk_dict['path'] = '/dev/%s' % disk_name
disk_dict['id'] = disk_id
disk_dict['capacity'] = capacity
disk_dict['rotational'] = rotational
disk_dict['hw_raid'] = True
return_list.append(disk_dict)
if not raid:
disk_dict = {}
disk_dict['hw_raid'] = False
disk_dict['id'] = disk_id
disk_dict['capacity'] = capacity
serial_number, err = get_serial_number(disk_name)
if err:
raise Exception(err)
disk_dict['serial_number'] = serial_number
disk_dict['rotational'] = rotational
return_list.append(disk_dict)
for disk_dict in return_list:
disk_dict['os_device'] = False
if partitions:
for partition in partitions:
if partition['name'] in os_partitions:
disk_dict['os_device'] = True
except Exception, e:
return None, "Error getting disk information : %s" % str(e)
def get_disk_status(disk_name):
"""Get the smart status of dell hw raid status if applicable of a disk given its name """
return_list = []
try:
hw_platform, err = config.get_hardware_platform()
hw_raid = False
disk_hw_info_all = {}
if hw_platform and hw_platform == 'dell':
from platforms import dell
disk_hw_info_tmp, err = dell.get_all_disks(0)
iter = 1
while disk_hw_info_tmp:
disk_hw_info_all.update(disk_hw_info_tmp)
disk_hw_info_tmp, err = dell.get_all_disks(iter)
iter += 1
raid_to_unix_mapping, err = dell.get_hardware_raid_to_unix_mapping()
if raid_to_unix_mapping and '/dev/%s' % disk_name in raid_to_unix_mapping.keys():
# Its a hardware RAID so pull the underlying info
hw_raid = True
controller_id = raid_to_unix_mapping['/dev/%s' %
disk_name]['controller_number']
device_id = raid_to_unix_mapping['/dev/%s' %
disk_name]['device_id']
disk_list, err = dell.get_hardware_raid_hard_drives(
controller_id, device_id)
if disk_list:
for disk in disk_list:
disk_dict = {}
disk_dict['serial_number'] = disk['serial_number']
disk_dict['hw_raid'] = True
disk_dict['status'] = disk['status']
return_list.append(disk_dict)
if not hw_raid:
disk_dict = {}
serial_number, err = get_serial_number(disk_name)
if err:
raise Exception(err)
disk_dict['serial_number'] = serial_number
status, err = get_smart_status(disk_name)
if err:
raise Exception(err)
disk_dict['status'] = status
return_list.append(disk_dict)
partitions, err = get_partitions(disk_name)
if err:
raise Exception(err)
scsi_info, err = get_disk_scsi_info(disk_name)
if err:
raise Exception(err)
for disk_dict in return_list:
if partitions:
disk_dict['partitions'] = partitions
if scsi_info:
disk_dict['scsi_info'] = scsi_info
disk_dict['name'] = disk_name
disk_dict['path'] = '/dev/%s' % disk_name
if hw_platform and hw_platform == 'dell':
if disk_hw_info_all and disk_dict['serial_number'] in disk_hw_info_all.keys():
serial_number = disk_dict['serial_number']
disk_dict['target_id'] = disk_hw_info_all[serial_number]['target_id']
disk_dict['controller_number'] = disk_hw_info_all[serial_number]['controller_number']
disk_dict['enclosure_id'] = disk_hw_info_all[serial_number]['enclosure_id']
disk_dict['channel'] = disk_hw_info_all[serial_number]['channel']
except Exception, e:
return None, "Error getting disk status : %s" % str(e)
def get_ipmi_status():
"""Hardware status using IPMI."""
ipmi_status = []
try:
fil = os.popen("ipmitool sdr")
str4 = fil.read()
lines = re.split("\r?\n", str4)
ipmi_status = []
for line in lines:
l = line.rstrip()
if not l:
continue
# print l
comp_list = l.split('|')
comp = comp_list[0].strip()
status = comp_list[2].strip()
hw_platform, err = config.get_hardware_platform()
if status != "ns":
if hw_platform and hw_platform == 'dell':
if ('fan' not in comp.lower()) and ('PS' not in comp) and (
'voltage' not in comp.lower()
) and ('current' not in comp.lower()) and (comp not in [
'Inlet Temp', 'CMOS Battery', 'Pwr Consumption',
'Temp'
]):
continue
td = {}
td["parameter_name"] = comp_list[0].strip()
td["reading"] = comp_list[1].strip()
td["status"] = comp_list[2].strip()
'''
if comp in ["CPU Temp", "CPU1 Temperature"]:
td["parameter_name"] = "CPU Temperature"
td["component_name"] = "CPU"
elif comp in ["System Temp", "MB1 Temperature"]:
td["parameter_name"] = "System Temperature"
td["component_name"] = "System"
elif comp == "DIMMA1 Temp":
td["parameter_name"] = "Memory card 1 temperature"
td["component_name"] = "Memory card 1"
elif comp == "DIMMA2 Temp":
td["parameter_name"] = "Memory card 2 temperature"
td["component_name"] = "Memory card 2"
elif comp == "DIMMA3 Temp":
td["parameter_name"] = "Memory card 3 temperature"
td["component_name"] = "Memory card 3"
elif comp == "FAN1":
td["parameter_name"] = "Fan 1 speed"
td["component_name"] = "Fan 1"
elif comp == "FAN2":
td["parameter_name"] = "Fan 2 speed"
td["component_name"] = "Fan 2"
elif comp == "FAN3":
td["parameter_name"] = "Fan 3 speed"
td["component_name"] = "Fan 3"
elif comp == "CPU_FAN1":
td["parameter_name"] = "CPU Fan"
td["component_name"] = "CPU Fan"
'''
ipmi_status.append(td)
except Exception, e:
return None, 'Error retrieving IPMI information: %s' % str(e)
def view_disks(request):
return_dict = {}
type = 'data'
try:
if "ack" in request.GET:
if request.GET["ack"] == "blink":
return_dict[
'ack_message'] = "Disk identification LED successfully activated"
elif request.GET["ack"] == "unblink":
return_dict[
'ack_message'] = "Disk identification LED successfully de-activated"
ret, err = django_utils.get_request_parameter_values(request, ['type'])
if err:
raise Exception(err)
if ('type' not in ret) or ret['type'] not in ['data', 'os']:
type = 'data'
else:
type = ret['type']
si, err = system_info.load_system_config()
if err:
raise Exception(err)
if not si:
raise Exception('Error loading system configuration')
hw_platform, err = config.get_hardware_platform()
if hw_platform:
return_dict['hw_platform'] = hw_platform
if hw_platform == 'dell':
from integralstor.platforms import dell
idrac_url, err = dell.get_idrac_addr()
if idrac_url:
return_dict['idrac_url'] = idrac_url
if type == 'os':
os_disk_stats, err = disks.get_os_partition_stats()
if err:
raise Exception(err)
return_dict['os_disk_stats'] = os_disk_stats
return_dict['node'] = si
return_dict['system_info'] = si
return_dict["disk_status"] = si['disks']
# print si['disks']
return_dict['node_name'] = si['fqdn']
if type == 'os':
return django.shortcuts.render_to_response(
'view_os_disks.html',
return_dict,
context_instance=django.template.context.RequestContext(
request))
else:
return django.shortcuts.render_to_response(
'view_data_disks.html',
return_dict,
context_instance=django.template.context.RequestContext(
request))
except Exception, e:
return_dict['base_template'] = "storage_base.html"
return_dict["page_title"] = 'Disks'
return_dict['tab'] = 'view_%s_disks_tab' % type
return_dict["error"] = 'Error loading disk information'
return_dict["error_details"] = str(e)
return django.shortcuts.render_to_response(
"logged_in_error.html",
return_dict,
context_instance=django.template.context.RequestContext(request))
def main():
lg = None
try:
scripts_log, err = config.get_scripts_log_path()
if err:
raise Exception(err)
lg, err = logger.get_script_logger('Poll for alerts',
scripts_log,
level=logging.DEBUG)
lck, err = lock.get_lock('integralstor_poll_for_alerts')
if err:
raise Exception(err)
if not lck:
raise Exception('Could not acquire lock. Exiting.')
logger.log_or_print('Poll for alerts initiated.', lg, level='info')
now = int(time.time())
db_path, err = config.get_db_path()
if err:
raise Exception(err)
tasks_query = "select * from tasks where last_run_time > '%d' and (status = 'error-retrying' or status = 'failed');" % (
now - 110)
# print "\ntasks_query: ", tasks_query
rows, err = db.get_multiple_rows(db_path, tasks_query)
# print "\nrows: ", rows
if err:
raise Exception(err)
alert_list = None
if rows:
for row in rows:
if row['status'] == 'error-retrying':
alert_list.append({
'subsystem_type_id':
7,
'severity_type_id':
2,
'component':
row['description'],
'alert_str':
"Task: %s failed but will be retried." %
row['description']
})
elif row['status'] == 'failed':
alert_list.append({
'subsystem_type_id':
7,
'severity_type_id':
3,
'component':
row['description'],
'alert_str':
"Task: %s failed." % row['description']
})
# print "\nalert_list: ", alert_list
hw_platform, err = config.get_hardware_platform()
if hw_platform:
if hw_platform == 'dell':
from integralstor.platforms import dell
alerts_dict, err = dell.get_alert_logs()
if alerts_dict:
current_time = int(time.time())
for time_stamp, alerts_list in alerts_dict.items():
for alert_dict in alerts_list:
if alert_dict['Severity'] == 'Critical':
if (current_time - time_stamp) < (60 * 60):
alert_list.append({
'subsystem_type_id':
5,
'severity_type_id':
3,
'component':
'Dell Hardware component',
'alert_str':
alert_dict['description']
})
# print time_stamp, alert_dict
if alert_list:
alerts.record_alerts(alert_list)
lock.release_lock('integralstor_poll_for_alerts')
except Exception, e:
print "Error generating alerts : %s ! Exiting." % str(e)
logger.log_or_print('Error polling for alerts : %s' % e,
lg,
level='critical')
return -1