def generate_dell_hw_status(f): try: hw_platform, err = config.get_hardware_platform() if hw_platform: if hw_platform == 'dell': f.write( '--------------------- Dell hardware status BEGIN ------------------------\n\n' ) from integralstor.platforms import dell psu_status, err = dell.get_psu_status() if err: raise Exception(err) f.write('Number of PSUs detected : %d\n' % psu_status['psu_count']) f.write('PSU redundancy : ' + ('Redundancy present' if psu_status['redundancy'] else 'No redundancy') + '\n') for index, psu in enumerate(psu_status['psu_list']): f.write('PSU %d prescence' % index + ( 'Present' if psu_psu['prescence'] else 'Not present') + '\n') f.write('PSU %d switch on status' % index + ('On' if psu_psu['switch_on'] else 'Off') + '\n') f.write('PSU %d status' % index + ('Failed' if psu_psu['failure'] else 'OK') + '\n') f.write('\n') f.write( '--------------------- Dell hardware status END ------------------------\n\n' ) except Exception, e: return False, 'Error generating Dell hardware statussection: %s' % str( e)
class ViewLogsForm(forms.Form): """ Form to get the info about which log to view""" ch = [('alerts', 'Alerts log'), ('audit', 'Audit log')] hw_platform, err = config.get_hardware_platform() if hw_platform and hw_platform == 'dell': ch.append(('hardware', 'Hardware log')) log_type = forms.ChoiceField(choices=ch)
def get_ipmi_status(): """Hardware status using IPMI.""" ipmi_status = [] try: fil = os.popen("ipmitool sdr") str4 = fil.read() lines = re.split("\r?\n", str4) ipmi_status = [] for line in lines: l = line.rstrip() if not l: continue # print l comp_list = l.split('|') comp = comp_list[0].strip() status = comp_list[2].strip() hw_platform, err = config.get_hardware_platform() if status != "ns": if hw_platform and hw_platform == 'dell': if ('fan' not in comp.lower()) and ('PS' not in comp) and ('voltage' not in comp.lower()) and ('current' not in comp.lower()) and (comp not in ['Inlet Temp', 'CMOS Battery', 'Pwr Consumption', 'Temp']): continue td = {} td["parameter_name"] = comp_list[0].strip() td["reading"] = comp_list[1].strip() td["status"] = comp_list[2].strip() ''' if comp in ["CPU Temp", "CPU1 Temperature"]: td["parameter_name"] = "CPU Temperature" td["component_name"] = "CPU" elif comp in ["System Temp", "MB1 Temperature"]: td["parameter_name"] = "System Temperature" td["component_name"] = "System" elif comp == "DIMMA1 Temp": td["parameter_name"] = "Memory card 1 temperature" td["component_name"] = "Memory card 1" elif comp == "DIMMA2 Temp": td["parameter_name"] = "Memory card 2 temperature" td["component_name"] = "Memory card 2" elif comp == "DIMMA3 Temp": td["parameter_name"] = "Memory card 3 temperature" td["component_name"] = "Memory card 3" elif comp == "FAN1": td["parameter_name"] = "Fan 1 speed" td["component_name"] = "Fan 1" elif comp == "FAN2": td["parameter_name"] = "Fan 2 speed" td["component_name"] = "Fan 2" elif comp == "FAN3": td["parameter_name"] = "Fan 3 speed" td["component_name"] = "Fan 3" elif comp == "CPU_FAN1": td["parameter_name"] = "CPU Fan" td["component_name"] = "CPU Fan" ''' ipmi_status.append(td) except Exception, e: return None, 'Error retrieving IPMI information: %s' % str(e)
class DownloadLogsForm(forms.Form): """ Form to get the info about which log to download""" # ch = [('boot', 'Boot log'), ('dmesg', 'Dmesg log'), ('message', 'Message log'), ('smb', 'Samba logs'), # ('winbind', 'Samba Winbind logs'), ('alerts', 'Alerts log'), ('audit', 'Audit log')] ch = [('system', 'System logs'), ('alerts', 'Alerts log'), ('audit', 'Audit log')] hw_platform, err = config.get_hardware_platform() if hw_platform and hw_platform == 'dell': ch.append(('hardware', 'Hardware log')) log_type = forms.ChoiceField(choices=ch)
def view_disks(request): return_dict = {} type = 'data' try: if "ack" in request.GET: if request.GET["ack"] == "blink": return_dict['ack_message'] = "Disk identification LED successfully activated" elif request.GET["ack"] == "unblink": return_dict['ack_message'] = "Disk identification LED successfully de-activated" ret, err = django_utils.get_request_parameter_values( request, ['type']) if err: raise Exception(err) if ('type' not in ret) or ret['type'] not in ['data', 'os']: type = 'data' else: type = ret['type'] si, err = system_info.load_system_config() if err: raise Exception(err) if not si: raise Exception('Error loading system configuration') hw_platform, err = config.get_hardware_platform() if hw_platform: return_dict['hw_platform'] = hw_platform if hw_platform == 'dell': from integralstor.platforms import dell idrac_url, err = dell.get_idrac_addr() if idrac_url: return_dict['idrac_url'] = idrac_url if type == 'os': os_disk_stats, err = disks.get_os_partition_stats() if err: raise Exception(err) return_dict['os_disk_stats'] = os_disk_stats return_dict['node'] = si return_dict['system_info'] = si return_dict["disk_status"] = si['disks'] # print si['disks'] return_dict['node_name'] = si['fqdn'] if type == 'os': return django.shortcuts.render_to_response('view_os_disks.html', return_dict, context_instance=django.template.context.RequestContext(request)) else: return django.shortcuts.render_to_response('view_data_disks.html', return_dict, context_instance=django.template.context.RequestContext(request)) except Exception, e: return_dict['base_template'] = "storage_base.html" return_dict["page_title"] = 'Disks' return_dict['tab'] = 'view_%s_disks_tab' % type return_dict["error"] = 'Error loading disk information' return_dict["error_details"] = str(e) return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def get_hardware_specific_status(): return_dict = {} try: hw_platform, err = config.get_hardware_platform() if hw_platform: return_dict['hw_platform'] = hw_platform if hw_platform == 'dell': return_dict['hw_platform'] = 'dell' from integralstor.platforms import dell idrac_url, err = dell.get_idrac_addr() if idrac_url: return_dict['idrac_url'] = idrac_url except Exception, e: return None, 'Error retrieving hardware specific status: %s' % str(e)
def get_hardware_specific_status(): return_dict = {} try: hw_platform, err = config.get_hardware_platform() if hw_platform: return_dict['hw_platform'] = hw_platform if hw_platform == 'dell': return_dict['hw_platform'] = 'dell' from integralstor.platforms import dell idrac_url, err = dell.get_idrac_addr() if idrac_url: return_dict['idrac_url'] = idrac_url except Exception, e: return None, 'Error retrieving hardware specific status: %s' % str(e)
def view_hardware_logs(request): return_dict = {} try: hw_platform, err = config.get_hardware_platform() if err: raise Exception(err) if hw_platform and hw_platform != 'dell': raise Exception('Unknown hardware platform') return_dict['hw_platform'] = hw_platform if hw_platform == 'dell': from integralstor.platforms import dell logs_dict, err = dell.get_alert_logs() if logs_dict: return_dict['logs_dict'] = logs_dict return django.shortcuts.render_to_response('view_hardware_logs.html', return_dict, context_instance=django.template.context.RequestContext(request)) except Exception, e: return_dict['base_template'] = "system_base.html" return_dict["page_title"] = 'View and download logs' return_dict['tab'] = 'logs_tab' return_dict["error"] = 'Error loading hardware logs' return_dict["error_details"] = str(e) return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def main(): lg = None try: scripts_log, err = config.get_scripts_log_path() if err: raise Exception(err) lg, err = logger.get_script_logger('System status report generation', scripts_log, level=logging.DEBUG) status_reports_dir, err = config.get_staus_reports_dir_path() if err: raise Exception(err) lck, err = lock.get_lock('generate_system_status_report') if err: raise Exception(err) if not lck: raise Exception('Could not acquire lock.') logger.log_or_print('System status report generation initiated.', lg, level='info') if len(sys.argv) != 2: raise Exception( 'Usage : python generate_system_status_report.py <past_x_days>' ) past_x_days = int(sys.argv[1]) start_time, err = datetime_utils.get_epoch( when='midnight', num_previous_days=past_x_days) if err: raise Exception(err) now, err = datetime_utils.get_epoch(when='now') if err: raise Exception(err) now_local_str, err = datetime_utils.convert_from_epoch( now, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local') if err: raise Exception(err) tmp_file_name = 'integralstor_status_%s' % now_local_str tmp_file_name_with_path = '/tmp/%s' % tmp_file_name with open(tmp_file_name_with_path, 'w') as f: ret, err = generate_global_header(f) # print ret, err f.write('\n') ret, err = generate_dmidecode_section(f) # print ret, err f.write('\n\n') ret, err = generate_cpu_section(f) # print ret, err f.write('\n\n') ret, err = generate_memory_section(f) # print ret, err f.write('\n\n') ret, err = generate_command_based_section( f, 'nmcli con', 'Networking connections') # print ret, err f.write('\n\n') ret, err = generate_command_based_section(f, 'ip addr', 'IP addresses') # print ret, err f.write('\n\n') hw_platform, err = config.get_hardware_platform() # print ret, err if hw_platform: if hw_platform == 'dell': ret, err = generate_dell_hw_status(f) # print ret, err f.write('\n\n') ret, err = generate_disks_status_section(f) # print ret, err f.write('\n\n') ret, err = generate_command_based_section( f, 'df -HT --exclude-type=devtmpfs --exclude-type=tmpfs --exclude-type=zfs', 'OS disk space usage') # print ret, err f.write('\n\n') ret, err = generate_zfs_info_section(f) # print ret, err f.write('\n\n') ret, err = generate_command_based_section(f, 'zpool list', 'ZFS pool space usage') # print ret, err f.write('\n\n') ret, err = generate_command_based_section( f, 'zfs list -t filesystem -o name,used,avail,refer,mountpoint,dedup,compression,quota,xattr,recordsize,acltype', 'ZFS datasets') # print ret, err f.write('\n\n') ret, err = generate_command_based_section( f, 'zfs list -t volume -o name,used,avail,refer,mountpoint,dedup,compression,volsize,volblocksize', 'ZFS zvols') # print ret, err f.write('\n\n') ret, err = generate_command_based_section(f, 'zpool status -v', 'ZFS pool status') # print ret, err f.write('\n\n') ret, err = generate_audits_section(f, start_time, past_x_days) # print ret, err f.write('\n\n') ret, err = generate_alerts_section(f, start_time, past_x_days) # print ret, err f.write('\n\n') try: os.makedirs(status_reports_dir) except: pass final_file_name_with_path = '%s/%s' % (status_reports_dir, tmp_file_name) shutil.move(tmp_file_name_with_path, final_file_name_with_path) d, err = mail.load_email_settings() if not err and d and 'support_email_addresses' in d and d[ 'support_email_addresses']: # Email settings present so send it out to the support email # address email_header = '%s - IntegralSTOR system status report' % socket.getfqdn( ) email_body = 'Please find the latest IntegralSTOR system status report' processed_successfully, err = mail.enqueue( d['support_email_addresses'], email_header, email_body, attachment_file_location=final_file_name_with_path, delete_attachment_file=False) if err: raise Exception(err) except Exception, e: # print str(e) lock.release_lock('generate_system_status_report') logger.log_or_print('Error generating system status report : %s' % e, lg, level='critical') return -1, 'Error generating system status report : %s' % e
def view_dashboard(request, page = None): return_dict = {} try: return_dict["page_title"] = 'Overall system health' return_dict['tab'] = 'system_health_tab' return_dict["error"] = 'Error loading system health data' if request.method != 'GET': raise Exception('Invalid access method. Please use the menus') si, err = system_info.load_system_config() if err: raise Exception(err) if not si: raise Exception('Error loading system configuration') #node_name = si.keys()[0] #node = si[node_name] return_dict['node'] = si # print node.keys() # By default show error page template = "logged_in_error.html" # Chart specific declarations # will return 02, instead of 2. end_epoch, err = datetime_utils.get_epoch(when='now') if err: raise Exception(err) start_epoch = end_epoch - 3 * 60 * 60 start, err = datetime_utils.convert_from_epoch( start_epoch, return_format='str', str_format='%H:%M:%S', to='local') if err: raise Exception(err) end, err = datetime_utils.convert_from_epoch( end_epoch, return_format='str', str_format='%H:%M:%S', to='local') if err: raise Exception(err) todays_date = (datetime.date.today()).strftime('%02d') value_list = [] time_list = [] num_bad_disks = 0 num_hw_raid_bad_disks = 0 num_hw_raid_ctrl_disks = 0 num_smart_ctrl_disks = 0 num_disks = len(si['disks']) disks_ok = True for sn, disk in si['disks'].items(): if 'status' in disk: if 'hw_raid' in disk: if not disk['hw_raid']: num_smart_ctrl_disks += 1 if (disk['status'] is not None and disk['status'].upper() not in ['PASSED', 'OK']): num_bad_disks += 1 disks_ok = False else: num_hw_raid_ctrl_disks += 1 if (disk['status'] is not None and disk['status'].upper() != 'OK'): num_hw_raid_bad_disks += 1 disks_ok = False else: # Assume its a non raid disk num_smart_ctrl_disks += 1 if (disk['status'] is not None and disk['status'].upper() not in ['PASSED', 'OK']): num_bad_disks += 1 disks_ok = False return_dict['num_disks'] = num_disks return_dict['num_bad_disks'] = num_bad_disks return_dict['disks_ok'] = disks_ok return_dict['num_hw_raid_bad_disks'] = num_hw_raid_bad_disks return_dict['num_hw_raid_ctrl_disks'] = num_hw_raid_ctrl_disks return_dict['num_smart_ctrl_disks'] = num_smart_ctrl_disks if 'ipmi_status' in si: num_sensors = len(si['ipmi_status']) num_bad_sensors = 0 ipmi_ok = True for sensor in si['ipmi_status']: if sensor['status'] in ['ok', 'nr', 'na']: continue else: num_bad_sensors += 1 ipmi_ok = False return_dict['num_sensors'] = num_sensors return_dict['num_bad_sensors'] = num_bad_sensors return_dict['ipmi_ok'] = ipmi_ok services_dict, err = services_management.get_sysd_services_status() if err: raise Exception(err) num_services = len(services_dict) num_failed_services = 0 num_active_services = 0 num_inactive_services = 0 services_ok = True if services_dict: for service, service_d in services_dict.items(): if service_d["info"]["status"]["status_str"] == "Active": num_active_services += 1 elif service_d["info"]["status"]["status_str"] == "Inactive": num_inactive_services += 1 elif service_d["info"]["status"]["status_str"] == "Failed": num_failed_services += 1 services_ok = False elif service_d["info"]["status"]["status_str"] == "Unknown State": num_failed_services += 1 services_ok = False return_dict['num_services'] = num_services return_dict['num_active_services'] = num_active_services return_dict['num_inactive_services'] = num_inactive_services return_dict['num_failed_services'] = num_failed_services return_dict['services_ok'] = services_ok else: raise Exception('Error retrieving services status') pools, err = zfs.get_pools() if err: raise Exception(err) num_pools = len(pools) num_bad_pools = 0 num_degraded_pools = 0 num_high_usage_pools = 0 for pool in pools: if pool['usage']['used_percent'] > 75: num_high_usage_pools += 1 if pool['config']['pool']['root']['status']['state'] == 'ONLINE': pass elif pool['config']['pool']['root']['status']['state'] == 'DEGRADED': num_degraded_pools += 1 else: num_bad_pools += 1 return_dict['num_pools'] = num_pools return_dict['num_bad_pools'] = num_bad_pools return_dict['num_degraded_pools'] = num_degraded_pools return_dict['num_high_usage_pools'] = num_high_usage_pools load_avg_ok = True if (si["load_avg"]["5_min"] > si["load_avg"]["cpu_cores"]) or (si["load_avg"]["15_min"] > si["load_avg"]["cpu_cores"]): load_avg_ok = False return_dict['load_avg_ok'] = load_avg_ok shares_list, err = cifs.get_shares_list() if err: raise Exception(err) return_dict['num_cifs_shares'] = len(shares_list) exports_list, err = nfs.load_exports_list() if err: raise Exception(err) return_dict['num_nfs_exports'] = len(exports_list) target_list, err = iscsi_stgt.get_targets() if err: raise Exception(err) return_dict['num_iscsi_targets'] = len(target_list) with open('/proc/uptime', 'r') as f: uptime_seconds = float(f.readline().split()[0]) uptime_str = '%s hours' % ( ':'.join(str(datetime.timedelta(seconds=uptime_seconds)).split(':')[:2])) return_dict['uptime_str'] = uptime_str # CPU status if not page: page = "sys_health" if page == "cpu": return_dict["page_title"] = 'CPU statistics' return_dict['tab'] = 'cpu_tab' return_dict["error"] = 'Error loading CPU statistics' cpu, err = stats.get_system_stats(todays_date, start, end, "cpu") if err: raise Exception(err) value_dict = {} if cpu: for key in cpu.keys(): value_list = [] time_list = [] if key == "date": pass else: if cpu[key]: for a in cpu[key]: time_list.append(a[0]) value_list.append(a[1]) value_dict[key] = value_list return_dict["data_dict"] = value_dict queue, err = stats.get_system_stats( todays_date, start, end, "queue") if err: raise Exception(err) value_dict = {} if queue: for key in queue.keys(): value_list = [] time_list = [] if key == "date": pass else: for a in queue[key]: time_list.append(a[0]) value_list.append(a[1]) value_dict[key] = value_list return_dict["data_dict_queue"] = value_dict return_dict['node'] = si d = {} template = "view_cpu_stats.html" elif page == "sys_health": return_dict["page_title"] = 'Overall system health' return_dict['tab'] = 'system_health_tab' return_dict["error"] = 'Error loading system health data' template = "view_dashboard.html" hw_platform, err = config.get_hardware_platform() if hw_platform: return_dict['hw_platform'] = hw_platform if hw_platform == 'dell': from integralstor.platforms import dell idrac_url, err = dell.get_idrac_addr() if idrac_url: return_dict['idrac_url'] = idrac_url # Memory elif page == "memory": return_dict["page_title"] = 'Memory statistics' return_dict['tab'] = 'memory_tab' return_dict["error"] = 'Error loading memory statistics' mem, err = stats.get_system_stats( todays_date, start, end, "memory") if err: raise Exception(err) if mem: for a in mem["memused"]: time_list.append(a[0]) value_list.append((a[1] / (1024 * 1024))) return_dict['memory_status'] = si['memory'] template = "view_memory_stats.html" # Network elif page == "network": return_dict["page_title"] = 'Network statistics' return_dict['tab'] = 'network_tab' return_dict["error"] = 'Error loading Network statistics' network, err = stats.get_system_stats( todays_date, start, end, "network") if err: raise Exception(err) value_dict = {} if network: for key in network.keys(): value_list = [] time_list = [] if key == "date" or key == "lo": pass else: for a in network[key]["ifutil-percent"]: time_list.append(a[0]) value_list.append(a[1]) value_dict[key] = value_list return_dict["data_dict"] = value_dict return_dict["network_status"] = si['interfaces'] template = "view_network_stats.html" return_dict["labels"] = time_list return_dict["data"] = value_list return django.shortcuts.render_to_response(template, return_dict, context_instance=django.template.context.RequestContext(request)) except Exception, e: return_dict['base_template'] = "monitoring_base.html" return_dict["error_details"] = str(e) return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def download_log(request): """ Download the system log of the type specified in log_type POST param This calls the /sys_log via an http request on that node to get the info""" return_dict = {} try: hw_platform, err = config.get_hardware_platform() if err: raise Exception(err) if hw_platform and hw_platform != 'dell': raise Exception('Unknown hardware platform') return_dict['hw_platform'] = hw_platform form = log_management_forms.DownloadLogsForm(request.POST or None) if request.method == 'POST': if form.is_valid(): cd = form.cleaned_data log_type = cd['log_type'] if log_type in ['alerts', 'audit', 'hardware']: response = django.http.HttpResponse() if log_type == 'alerts': response['Content-disposition'] = 'attachment; filename=alerts_log.txt' all_alerts, err = alerts.get_alerts() if err: raise Exception(err) for alert in all_alerts: if int(alert['repeat_count']) > 1: response.write('Last alert time %s\nAlert message: %s\nRepeated count: %d\n\n' % (alert['last_update_time'], alert['alert_str'], int(alert['repeat_count']))) else: response.write('Last alert time %s\nAlert message: %s\n\n' % (alert['last_update_time'], alert['alert_str'])) response.flush() elif log_type == 'audit': response['Content-disposition'] = 'attachment; filename=audit_log.txt' all_audits, err = audit.get_entries() if err: raise Exception(err) for audit_info in all_audits: response.write('Time : %s \n' % audit_info['time']) response.write('Source IP : %s \n' % audit_info['ip']) response.write('Action : %s \n' % audit_info['action_str']) response.write('\n') response.flush() elif log_type == 'hardware': response['Content-disposition'] = 'attachment; filename=hardware_logs.txt' hw_platform, err = config.get_hardware_platform() if not hw_platform or hw_platform != 'dell': raise Exception('Unknown hardware platform') if hw_platform == 'dell': from integralstor.platforms import dell logs_dict, err = dell.get_alert_logs() if err: raise Exception(err) if not logs_dict: raise Exception('No logs detected!') for timestamp, log_list in logs_dict.items(): for log in log_list: response.write('Time : %s\n' % log['date_time']) response.write( 'Severity : %s\n' % log['Severity']) response.write( 'Description : %s\n' % log['description']) response.write('\n') response.flush() else: raise Exception('Unknown platform') else: scripts_log, err = config.get_scripts_log_path() if err: raise Exception(err) system_logs = [('/var/log/boot.log', 'boot.log'), ('/var/log/dmesg', 'dmesg'), ('/var/log/messages', 'messages'), ('/var/log/smblog.vfs', 'samba'), ('/var/log/samba/log.winbindd', 'winbind'), (scripts_log, 'scripts')] now_local_epoch, err = datetime_utils.get_epoch(when='now') if err: raise Exception(err) now_local_str, err = datetime_utils.convert_from_epoch( now_local_epoch, return_format='str', str_format='%Y_%m_%d_%H_%M', to='local') if err: raise Exception(err) zf_name = 'IntegralSTOR_system_logs_%s.zip' % now_local_str try: out = io.BytesIO() zf = zipfile.ZipFile(out, 'w') for entry in system_logs: zf.write(entry[0], arcname=entry[1]) #zf.write(file_name, arcname=display_name) zf.close() except Exception as e: raise Exception( "Error compressing log file : %s" % str(e)) response = django.http.HttpResponse( out.getvalue(), content_type='application/x-compressed') response['Content-disposition'] = 'attachment; filename=%s' % ( zf_name) return response # either a get or an invalid form so send back form return_dict['form'] = form return django.shortcuts.render_to_response('download_log_form.html', return_dict, context_instance=django.template.context.RequestContext(request)) except Exception, e: return_dict['base_template'] = "system_base.html" return_dict["page_title"] = 'Download system logs' return_dict['tab'] = 'logs_tab' return_dict["error"] = 'Error downloading system logs' return_dict["error_details"] = str(e) return django.shortcuts.render_to_response("logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def main(): lg = None try: scripts_log, err = config.get_scripts_log_path() if err: raise Exception(err) lg, err = logger.get_script_logger( 'Poll for alerts', scripts_log, level=logging.DEBUG) lck, err = lock.get_lock('integralstor_poll_for_alerts') if err: raise Exception(err) if not lck: raise Exception('Could not acquire lock. Exiting.') logger.log_or_print('Poll for alerts initiated.', lg, level='info') now = int(time.time()) db_path, err = config.get_db_path() if err: raise Exception(err) tasks_query = "select * from tasks where last_run_time > '%d' and (status = 'error-retrying' or status = 'failed');" % ( now - 110) # print "\ntasks_query: ", tasks_query rows, err = db.get_multiple_rows(db_path, tasks_query) # print "\nrows: ", rows if err: raise Exception(err) alert_list = None if rows: for row in rows: if row['status'] == 'error-retrying': alert_list.append({'subsystem_type_id': 7, 'severity_type_id': 2, 'component': row['description'], 'alert_str': "Task: %s failed but will be retried." % row['description']}) elif row['status'] == 'failed': alert_list.append({'subsystem_type_id': 7, 'severity_type_id': 3, 'component': row['description'], 'alert_str': "Task: %s failed." % row['description']}) # print "\nalert_list: ", alert_list hw_platform, err = config.get_hardware_platform() if hw_platform: if hw_platform == 'dell': from integralstor.platforms import dell alerts_dict, err = dell.get_alert_logs() if alerts_dict: current_time = int(time.time()) for time_stamp, alerts_list in alerts_dict.items(): for alert_dict in alerts_list: if alert_dict['Severity'] == 'Critical': if (current_time - time_stamp) < (60 * 60): alert_list.append({'subsystem_type_id': 5, 'severity_type_id': 3, 'component': 'Dell Hardware component', 'alert_str': alert_dict['description']}) # print time_stamp, alert_dict if alert_list: alerts.record_alerts(alert_list) lock.release_lock('integralstor_poll_for_alerts') except Exception, e: print "Error generating alerts : %s ! Exiting." % str(e) logger.log_or_print('Error polling for alerts : %s' % e, lg, level='critical') return -1
def get_disk_info(disk_name): """Get all details about a disk given its name """ # Given a disk name like sda, get the disk info return_list = [] try: disk_id, err = get_disk_id(disk_name) if err: raise Exception(err) capacity, err = get_capacity(disk_name) if err: raise Exception(err) rotational, err = is_rotational(disk_name) if err: raise Exception(err) os_partitions, err = get_os_partitions() if err: raise Exception(err) partitions, err = get_partitions(disk_name) if err: raise Exception(err) raid = False hw_platform, err = config.get_hardware_platform() if hw_platform and hw_platform == 'dell': from platforms import dell raid_to_unix_mapping, err = dell.get_hardware_raid_to_unix_mapping() if raid_to_unix_mapping and '/dev/%s' % disk_name in raid_to_unix_mapping.keys(): # Its a hardware RAID so pull the underlying info raid = True controller_id = raid_to_unix_mapping['/dev/%s' % disk_name]['controller_number'] device_id = raid_to_unix_mapping['/dev/%s' % disk_name]['device_id'] disk_list, err = dell.get_hardware_raid_hard_drives( controller_id, device_id) if disk_list: for disk in disk_list: disk_dict = {} disk_dict['serial_number'] = disk['serial_number'] #disk_dict['name'] = disk_name #disk_dict['path'] = '/dev/%s' % disk_name disk_dict['id'] = disk_id disk_dict['capacity'] = capacity disk_dict['rotational'] = rotational disk_dict['hw_raid'] = True return_list.append(disk_dict) if not raid: disk_dict = {} disk_dict['hw_raid'] = False disk_dict['id'] = disk_id disk_dict['capacity'] = capacity serial_number, err = get_serial_number(disk_name) if err: raise Exception(err) disk_dict['serial_number'] = serial_number disk_dict['rotational'] = rotational return_list.append(disk_dict) for disk_dict in return_list: disk_dict['os_device'] = False if partitions: for partition in partitions: if partition['name'] in os_partitions: disk_dict['os_device'] = True except Exception, e: return None, "Error getting disk information : %s" % str(e)
def get_disk_status(disk_name): """Get the smart status of dell hw raid status if applicable of a disk given its name """ return_list = [] try: hw_platform, err = config.get_hardware_platform() hw_raid = False disk_hw_info_all = {} if hw_platform and hw_platform == 'dell': from platforms import dell disk_hw_info_tmp, err = dell.get_all_disks(0) iter = 1 while disk_hw_info_tmp: disk_hw_info_all.update(disk_hw_info_tmp) disk_hw_info_tmp, err = dell.get_all_disks(iter) iter += 1 raid_to_unix_mapping, err = dell.get_hardware_raid_to_unix_mapping() if raid_to_unix_mapping and '/dev/%s' % disk_name in raid_to_unix_mapping.keys(): # Its a hardware RAID so pull the underlying info hw_raid = True controller_id = raid_to_unix_mapping['/dev/%s' % disk_name]['controller_number'] device_id = raid_to_unix_mapping['/dev/%s' % disk_name]['device_id'] disk_list, err = dell.get_hardware_raid_hard_drives( controller_id, device_id) if disk_list: for disk in disk_list: disk_dict = {} disk_dict['serial_number'] = disk['serial_number'] disk_dict['hw_raid'] = True disk_dict['status'] = disk['status'] return_list.append(disk_dict) if not hw_raid: disk_dict = {} serial_number, err = get_serial_number(disk_name) if err: raise Exception(err) disk_dict['serial_number'] = serial_number status, err = get_smart_status(disk_name) if err: raise Exception(err) disk_dict['status'] = status return_list.append(disk_dict) partitions, err = get_partitions(disk_name) if err: raise Exception(err) scsi_info, err = get_disk_scsi_info(disk_name) if err: raise Exception(err) for disk_dict in return_list: if partitions: disk_dict['partitions'] = partitions if scsi_info: disk_dict['scsi_info'] = scsi_info disk_dict['name'] = disk_name disk_dict['path'] = '/dev/%s' % disk_name if hw_platform and hw_platform == 'dell': if disk_hw_info_all and disk_dict['serial_number'] in disk_hw_info_all.keys(): serial_number = disk_dict['serial_number'] disk_dict['target_id'] = disk_hw_info_all[serial_number]['target_id'] disk_dict['controller_number'] = disk_hw_info_all[serial_number]['controller_number'] disk_dict['enclosure_id'] = disk_hw_info_all[serial_number]['enclosure_id'] disk_dict['channel'] = disk_hw_info_all[serial_number]['channel'] except Exception, e: return None, "Error getting disk status : %s" % str(e)
def get_ipmi_status(): """Hardware status using IPMI.""" ipmi_status = [] try: fil = os.popen("ipmitool sdr") str4 = fil.read() lines = re.split("\r?\n", str4) ipmi_status = [] for line in lines: l = line.rstrip() if not l: continue # print l comp_list = l.split('|') comp = comp_list[0].strip() status = comp_list[2].strip() hw_platform, err = config.get_hardware_platform() if status != "ns": if hw_platform and hw_platform == 'dell': if ('fan' not in comp.lower()) and ('PS' not in comp) and ( 'voltage' not in comp.lower() ) and ('current' not in comp.lower()) and (comp not in [ 'Inlet Temp', 'CMOS Battery', 'Pwr Consumption', 'Temp' ]): continue td = {} td["parameter_name"] = comp_list[0].strip() td["reading"] = comp_list[1].strip() td["status"] = comp_list[2].strip() ''' if comp in ["CPU Temp", "CPU1 Temperature"]: td["parameter_name"] = "CPU Temperature" td["component_name"] = "CPU" elif comp in ["System Temp", "MB1 Temperature"]: td["parameter_name"] = "System Temperature" td["component_name"] = "System" elif comp == "DIMMA1 Temp": td["parameter_name"] = "Memory card 1 temperature" td["component_name"] = "Memory card 1" elif comp == "DIMMA2 Temp": td["parameter_name"] = "Memory card 2 temperature" td["component_name"] = "Memory card 2" elif comp == "DIMMA3 Temp": td["parameter_name"] = "Memory card 3 temperature" td["component_name"] = "Memory card 3" elif comp == "FAN1": td["parameter_name"] = "Fan 1 speed" td["component_name"] = "Fan 1" elif comp == "FAN2": td["parameter_name"] = "Fan 2 speed" td["component_name"] = "Fan 2" elif comp == "FAN3": td["parameter_name"] = "Fan 3 speed" td["component_name"] = "Fan 3" elif comp == "CPU_FAN1": td["parameter_name"] = "CPU Fan" td["component_name"] = "CPU Fan" ''' ipmi_status.append(td) except Exception, e: return None, 'Error retrieving IPMI information: %s' % str(e)
def view_disks(request): return_dict = {} type = 'data' try: if "ack" in request.GET: if request.GET["ack"] == "blink": return_dict[ 'ack_message'] = "Disk identification LED successfully activated" elif request.GET["ack"] == "unblink": return_dict[ 'ack_message'] = "Disk identification LED successfully de-activated" ret, err = django_utils.get_request_parameter_values(request, ['type']) if err: raise Exception(err) if ('type' not in ret) or ret['type'] not in ['data', 'os']: type = 'data' else: type = ret['type'] si, err = system_info.load_system_config() if err: raise Exception(err) if not si: raise Exception('Error loading system configuration') hw_platform, err = config.get_hardware_platform() if hw_platform: return_dict['hw_platform'] = hw_platform if hw_platform == 'dell': from integralstor.platforms import dell idrac_url, err = dell.get_idrac_addr() if idrac_url: return_dict['idrac_url'] = idrac_url if type == 'os': os_disk_stats, err = disks.get_os_partition_stats() if err: raise Exception(err) return_dict['os_disk_stats'] = os_disk_stats return_dict['node'] = si return_dict['system_info'] = si return_dict["disk_status"] = si['disks'] # print si['disks'] return_dict['node_name'] = si['fqdn'] if type == 'os': return django.shortcuts.render_to_response( 'view_os_disks.html', return_dict, context_instance=django.template.context.RequestContext( request)) else: return django.shortcuts.render_to_response( 'view_data_disks.html', return_dict, context_instance=django.template.context.RequestContext( request)) except Exception, e: return_dict['base_template'] = "storage_base.html" return_dict["page_title"] = 'Disks' return_dict['tab'] = 'view_%s_disks_tab' % type return_dict["error"] = 'Error loading disk information' return_dict["error_details"] = str(e) return django.shortcuts.render_to_response( "logged_in_error.html", return_dict, context_instance=django.template.context.RequestContext(request))
def main(): lg = None try: scripts_log, err = config.get_scripts_log_path() if err: raise Exception(err) lg, err = logger.get_script_logger('Poll for alerts', scripts_log, level=logging.DEBUG) lck, err = lock.get_lock('integralstor_poll_for_alerts') if err: raise Exception(err) if not lck: raise Exception('Could not acquire lock. Exiting.') logger.log_or_print('Poll for alerts initiated.', lg, level='info') now = int(time.time()) db_path, err = config.get_db_path() if err: raise Exception(err) tasks_query = "select * from tasks where last_run_time > '%d' and (status = 'error-retrying' or status = 'failed');" % ( now - 110) # print "\ntasks_query: ", tasks_query rows, err = db.get_multiple_rows(db_path, tasks_query) # print "\nrows: ", rows if err: raise Exception(err) alert_list = None if rows: for row in rows: if row['status'] == 'error-retrying': alert_list.append({ 'subsystem_type_id': 7, 'severity_type_id': 2, 'component': row['description'], 'alert_str': "Task: %s failed but will be retried." % row['description'] }) elif row['status'] == 'failed': alert_list.append({ 'subsystem_type_id': 7, 'severity_type_id': 3, 'component': row['description'], 'alert_str': "Task: %s failed." % row['description'] }) # print "\nalert_list: ", alert_list hw_platform, err = config.get_hardware_platform() if hw_platform: if hw_platform == 'dell': from integralstor.platforms import dell alerts_dict, err = dell.get_alert_logs() if alerts_dict: current_time = int(time.time()) for time_stamp, alerts_list in alerts_dict.items(): for alert_dict in alerts_list: if alert_dict['Severity'] == 'Critical': if (current_time - time_stamp) < (60 * 60): alert_list.append({ 'subsystem_type_id': 5, 'severity_type_id': 3, 'component': 'Dell Hardware component', 'alert_str': alert_dict['description'] }) # print time_stamp, alert_dict if alert_list: alerts.record_alerts(alert_list) lock.release_lock('integralstor_poll_for_alerts') except Exception, e: print "Error generating alerts : %s ! Exiting." % str(e) logger.log_or_print('Error polling for alerts : %s' % e, lg, level='critical') return -1