def create_list_index_file_and_suspend(self, index: int = 0, save_path="", suspend=True): """ the start method will stop if create the file :param suspend: bool, stop the flow immediately, next method will not exec :param index: the order in method_name_list :param save_path: abs_path :return: None """ if not save_path: save_path = self.default_save_path path = os.path.dirname(save_path) if not os.path.exists(path): os.makedirs(path, exist_ok=True) log.warning("Create path {}".format(path)) self.default_save_path = save_path file_operator.YamlOperator(save_path).write(str(index)) if suspend and index != -1: log.warning( "Create {} Index: {}, Flow {} wil be stop immediately!".format( save_path, index, self.__class_name)) self.suspend_exec() return
def __update_result(self): step = { 'step_name': '', 'result': 'PASS', 'expect': '', 'actual': '', 'note': '' } step.update({'step_name': self.event_method_name}) if not self.flag: if self.capture: path = get_current_dir( "Test_Report/img/{}__{}_exception.png".format( self.case_name, self.event_method_name)) log.warning(f"capture path: {path}") capture_screen(path) step.update({ 'result': 'Fail', 'expect': self.error_msg.get("expect", ""), 'actual': self.error_msg.get("actual", ""), 'note': '{}__{}_exception.png'.format(self.case_name, self.event_method_name) if self.capture else "" }) else: step.update(self.success_msg) update_cases_result(self.yml_path, self.case_name, step) return
def deleteDetails(subject,body,db): try: city="" nodeList="" match=re.match(r'.*City:(.*)',str(subject),re.M|re.I) if match: city=match.group(1) #nodeList=str(match.group(2)).split(",") else: return 0 bodyMsg=body.split("\n") for line in bodyMsg: if "node" in line.lower(): match=re.match(r'.*Nodes\s*Down\s*:(.*)<o:p>.*',str(line),re.DOTALL) if match: nodes=match.group(1) nodes=nodes.split(",") delete(city,nodes,db) return 1 else: log.warning("Parsing bosy message doesn't match pattern. Body message %s",str(body)) return 0 except Exception as e: log.info("Exception occured while deleting records .Error info %s",str(e)) return -1
def deleteDetails(subject, body, db): try: city = "" nodeList = "" match = re.match(r'.*City:(.*)', str(subject), re.M | re.I) if match: city = match.group(1) #nodeList=str(match.group(2)).split(",") else: return 0 bodyMsg = body.split("\n") for line in bodyMsg: if "node" in line.lower(): match = re.match(r'.*Nodes\s*Down\s*:(.*)<o:p>.*', str(line), re.DOTALL) if match: nodes = match.group(1) nodes = nodes.split(",") delete(city, nodes, db) return 1 else: log.warning( "Parsing bosy message doesn't match pattern. Body message %s", str(body)) return 0 except Exception as e: log.info("Exception occured while deleting records .Error info %s", str(e)) return -1
def import_cert(self, file_path): dst = f"/etc/ThinProCertificates/{os.path.basename(file_path)}" shutil.copy(file_path, dst) file_name = os.path.basename(file_path).replace(".cer", ".pem") source = f"/etc/ssl/certs/{file_name}" dist = f"/etc/ThinProCertificates/{file_name}" if file_path in self._certs: log.warning(f'Cert: {file_path} has imported !') return log.info(f"Start Import Cert {file_path}") CertificateUtil._certs.append(file_path) pipe = os.popen(f'hptc-cert-mgr --non-interactive -i {file_path}') if '802' not in file_path: pipe.read() shutil.copy(source, dist) return try: time.sleep(3) input_window = self.cert_password input_window.cert_blank.send_keys(self._cert_password) self.cert_ok.click() time.sleep(10) log.info(f"Import Cert {file_path} Success!") except PicNotFoundError as e: raise CaseRunningError(f"Import Cert Fail! {str(e)}") finally: self.close()
def createDBObject(): try: db = DBCode() return db except Exception as e: log.warning( "NodeDown -- Exception occured during Database Connection %s", str(e))
def nodePingtest(ipaddress): try: status,result=cm.getstatusoutput("ping -c100 "+str(ipaddress)) log.info("Node ping result : %s",str(result)) if status == 0: log.info("Ping to device got passed") return "0" else: return str(result) except Exception as e: log.warning("Ping test for Node Reboot got exception . Error msg : %s",str(e)) return "1"
def setGOCFlag(db,values,val1,val2,val3,val4,val5,val6,val7,AEndIntf,BEndIntf,pingA,pingB,val8,val9,slaStates,phNumber): try: db.updateTable(values) msg="" sev="Sev2" message="" circuitProvider=val7 if "down" in val6.lower(): msg="Down" elif "loss" in val6.lower(): msg="Packet Loss. Loss Percentage is :"+val6+"%" else: msg=slaStates fetchedValues,fetchedValues1,flag=db.getInternalCircuitCountMatch(val7) if flag: circuitProvider = "" sev="Sev1" for val in fetchedValues: slaState="" for val1 in fetchedValues1: if str(val[9]) == str(val1[0]): circuitProvider=circuitProvider+","+str(val1[0]) if "down" in str(val1[2]).lower(): slaState="Down" elif "loss" in str(val1[2].lower()): slaState="Packet Loss. Loss Percentage is :"+str(val1[2])+"%" else: slaState=slaStates break phNumber1="" if "NO" in str(val[13]).strip(): phNumber1="" else: phNumber1=str(val[13]).strip() message=message+"\n AEnd Device :"+str(val[4])+" Interface :"+str(val[5])+"\n ZEnd Device :"+str(val[6])+" Interface :"+str(val[7])+"\n Circuit Provider :"+str(val[8])+"\n CircuitID :"+str(val[1])+" OrderID :"+str(val[2])+"\n SLAStatus :"+slaState+"\n Phone Number :"+phNumber1+"\n\n" else: message="\n AEnd Device :" +val1+" Interface : "+val2+"\n ZEnd Device :"+val3+" Interface :"+val4+"\n Circuit Provider :"+val5+"\n CircuitID :"+val8+" OrderID :"+val9+"\n SLASTatus :"+msg+"\n Phone Number :"+phNumber if pingA.strip() == "Fail": message+="\n\n Access to the routers got failed due to Authentication failed." else: try: if int(AEndIntf) == 0 and int(BEndIntf) == 0 and pingA == "0" and int(pingB) == 0: message+="\n\n Connection to device got timed-out" except Exception as e: message+="\n\n Ping result from Device " +AEndIntf+" to Device "+BEndIntf+" \n "+pingA+"\n\n Ping result from Device "+BEndIntf+" to Device "+AEndIntf+" \n "+pingB+"\n\n" subject=sev+" Circuit ID :"+circuitProvider msg="Team" + "\n"+"\n"+ "Please find the details about the devices\n\n"+message sendMail(subject,msg,sev) except Exception as e: log.warning("Exception(SETGocFLag Module) message : %s",str(e))
def nodePingtest(ipaddress): try: status, result = cm.getstatusoutput("ping -c100 " + str(ipaddress)) log.info("Node ping result : %s", str(result)) if status == 0: log.info("Ping to device got passed") return "0" else: return str(result) except Exception as e: log.warning("Ping test for Node Reboot got exception . Error msg : %s", str(e)) return "1"
def sendMail(subject,message,to_addr_list): try: smtpserver= "relay.emc-corp.net:25" #to_addr_list = common.getEmailList(sev.lower()) from_addr_list = "Network-engineering" log.info("Send mail content. Subject %s .message %s",str(subject),str(message)) msg = MIMEText(message) msg['Subject'] = subject msg['From'] = from_addr_list msg['To'] = ",".join(to_addr_list) server = smtplib.SMTP(smtpserver) server.ehlo() send = server.sendmail(from_addr_list, to_addr_list, msg.as_string()) server.quit() except Exception as e: log.warning("Exception occured during send mail . Error Msg : %s",str(e))
def sendMail(subject, message, sev): try: smtpserver = "relay.emc-corp.net:25" to_addr_list = common.getEmailList(sev.lower()) from_addr_list = "Network-engineering" log.info("Send mail content. Subject %s .message %s", str(subject), str(message)) msg = MIMEText(message) msg['Subject'] = subject msg['From'] = from_addr_list msg['To'] = ",".join(to_addr_list) server = smtplib.SMTP(smtpserver) server.ehlo() send = server.sendmail(from_addr_list, to_addr_list, msg.as_string()) server.quit() except Exception as e: log.warning("Send mail got error %s", str(e))
def sendMail(subject,message): try: smtpserver= "relay.emc-corp.net:25" #to_addr_list = ["*****@*****.**"] to_addr_list = common.getEmailList('exception') from_addr_list = "Network-engineering" log.info("Node -- Send mail content. Subject %s .message %s",str(subject),str(message)) msg = MIMEText(message) msg['Subject'] = subject msg['From'] = from_addr_list msg['To'] = ",".join(to_addr_list) server = smtplib.SMTP(smtpserver) server.ehlo() send = server.sendmail(from_addr_list, to_addr_list, msg.as_string()) log.info("Node-- Mail has been sent to %s",str(to_addr_list)) server.quit() except Exception as e: log.warning("Node -- Send mail got error %s",str(e))
def sendMail(subject, message): try: smtpserver = "relay.emc-corp.net:25" #to_addr_list = ["*****@*****.**"] to_addr_list = common.getEmailList('exception') from_addr_list = "Network-engineering" log.info("Node -- Send mail content. Subject %s .message %s", str(subject), str(message)) msg = MIMEText(message) msg['Subject'] = subject msg['From'] = from_addr_list msg['To'] = ",".join(to_addr_list) server = smtplib.SMTP(smtpserver) server.ehlo() send = server.sendmail(from_addr_list, to_addr_list, msg.as_string()) log.info("Node-- Mail has been sent to %s", str(to_addr_list)) server.quit() except Exception as e: log.warning("Node -- Send mail got error %s", str(e))
def check_window_has_opened(self): if self.close_windows_name and isinstance(self.close_windows_name, tuple): window = self.close_windows_name[-1] result = check_window(window) if not result: log.warning( "Start WorkAround Using Pic instead of 'CTRL+ALT+S'") menu = self.menu menu.click() time.sleep(6) pyautogui.typewrite(self.name.replace("_", " ")) time.sleep(2) pyautogui.press('enter') time.sleep(3) log.info("end") else: log.info(f"Check Window: {window} Exist") else: log.warning("No window name")
def get_global_config(*keys): """ :params: keys, tuple if key not exist, raise ValueError """ file_dict = {} path = get_current_dir('Test_Data/td_common/global_config.yml') if os.path.exists(path): file_obj = YamlOperator(path) file_dict = file_obj.read() else: log.warning("Not Exist {}".format(path)) if not keys: return file_dict new_value = copy.deepcopy(file_dict) for i in keys: value = new_value.get(i, None) if isinstance(new_value, dict) else None if not value: index = keys.index(i) raise ValueError("Key not Exist, origin: {}".format(" -> ".join( keys[:index + 1]))) new_value = value return new_value
def delete(city, nodes, dbs): try: print "city", city record = dbs.selectNodeState(city.strip()) dbNodeValue = "" print "yet to delete nodes : -", nodes for rec in record: dbNodeValue = str(rec[2]).split(",") dbIpaddress = str(rec[4]).split(",") print "DBNode", dbNodeValue for node in nodes: if str(node).strip() in str(rec[2]).strip(): index = dbNodeValue.index(str(node).strip()) del dbIpaddress[index] dbNodeValue.remove(str(node).strip()) print "DBNode:", dbNodeValue if dbNodeValue: value = ",".join(dbNodeValue) Count = len(dbNodeValue) ipadd = ",".join(dbIpaddress) values = [] values.append(str(rec[0])) values.append(Count) values.append(str(value)) values.append('NO') values.append(str(ipadd)) dbs.updateNodeStateTable(values) log.info('Node record got updated.values are %s', str(values)) else: dbs.deleteRowNode(str(rec[0])) log.info('Node %s info got deleted.', str(rec[0])) except Exception as e: log.warning("Exception occured while deleting records.Error info %s", str(e))
def delete(city,nodes,dbs): try: print "city",city record=dbs.selectNodeState(city.strip()) dbNodeValue="" print "yet to delete nodes : -",nodes for rec in record: dbNodeValue=str(rec[2]).split(",") dbIpaddress=str(rec[4]).split(",") print "DBNode",dbNodeValue for node in nodes: if str(node).strip() in str(rec[2]).strip(): index=dbNodeValue.index(str(node).strip()) del dbIpaddress[index] dbNodeValue.remove(str(node).strip()) print "DBNode:",dbNodeValue if dbNodeValue: value=",".join(dbNodeValue) Count=len(dbNodeValue) ipadd=",".join(dbIpaddress) values=[] values.append(str(rec[0])) values.append(Count) values.append(str(value)) values.append('NO') values.append(str(ipadd)) dbs.updateNodeStateTable(values) log.info('Node record got updated.values are %s',str(values)) else: dbs.deleteRowNode(str(rec[0])) log.info('Node %s info got deleted.',str(rec[0])) except Exception as e: log.warning("Exception occured while deleting records.Error info %s",str(e))
def pingTest(device1, device2, aIntf, bIntf, latencyValue, latency): try: username, password = common.getRouterCredentials() #password="******" delayFactor, loop = common.getSendCommandDelayFactor() timeout = getTimeout() slaState = "0" AEnddevice = ConnectHandler(device_type="cisco_ios", ip=device1, username=username, password=password) command = 'sh run interface ' + aIntf AEnddevice.clear_buffer() output1 = AEnddevice.send_command(command, int(delayFactor), int(loop)) output1 = str(output1) log.info("Device %s. Command %s. Output %s", str(device1), str(command), str(output1)) match = re.match(r'.*ip\s+address\s+(.*)\s+255.*', output1, re.DOTALL) if match: AEndIntf = match.group(1) BEnddevice = ConnectHandler(device_type="cisco_ios", ip=device2, username=username, password=password) command = 'sh run interface ' + bIntf BEnddevice.clear_buffer() output1 = BEnddevice.send_command(command, int(delayFactor), int(loop)) output1 = str(output1) log.info("Device %s. Command %s. Output %s", str(device2), str(command), str(output1)) match = re.match(r'.*ip\s+address\s+(.*)\s+255.*', output1, re.DOTALL) if match: BEndIntf = match.group(1) command = 'ping ' + BEndIntf.strip() + ' source ' + AEndIntf.strip( ) + ' repeat 30 timeout ' + timeout AEnddevice.clear_buffer() output1 = AEnddevice.send_command(command, int(delayFactor), int(loop)) output1 = str(output1) pingA = output1 log.info("Device %s.Ping Command %s. Output %s", str(device1), str(command), str(output1)) Alatency = 1 pingResultA = -1 match = re.match(r'.*rate\s+is\s+(.*)\s+percent.*', output1, re.DOTALL) if match: pingResultA = match.group(1) try: if "latency" in latency.lower(): lValue = latencyValue.split(",") match1 = re.match( r'.*min/avg/max\s*=\s*(\d+)/(\d+)/(\d+).*', output1, re.DOTALL) if match1: AminValue = match1.group(1) AavgValue = match1.group(2) AmaxValue = match1.group(3) if int(lValue[0]) > int(AavgValue) or int( lValue[2]) < int(AavgValue): if int(lValue[0]) > int(AavgValue) and int( lValue[2]) < int(AavgValue): slaState = "Latency-Above & Below threshold" elif int(lValue[0]) > int(AavgValue): slaState = "Latency-Below threshold" else: slaState = "Latency-Above threshold" Alatency = 0 except Exception as e: log.warning( "parsing Latency Message got failed.Error info : %s", str(e)) else: if "Invalid" in output1: pingResultA = "0" elif pingResultA == -1: pingResultA = output1 # pingResultA="0" command = 'ping ' + AEndIntf.strip() + ' source ' + BEndIntf.strip( ) + ' time ' + timeout + ' repeat 30' BEnddevice.clear_buffer() output1 = BEnddevice.send_command(command, int(delayFactor), int(loop)) output1 = str(output1) pingB = output1 print "PingB", pingB log.info("Device %s. Ping Command %s. Output %s", str(device2), str(command), str(output1)) Blatency = 1 pingResultB = -1 match = re.match(r'.*rate\s+is\s+(.*)\s+percent.*', output1, re.DOTALL) if match: pingResultB = match.group(1) try: if "latency" in latency.lower(): lValue = latencyValue.split(",") match1 = re.match( r'.*min/avg/max\s*=\s*(\d+)/(\d+)/(\d+).*', output1, re.DOTALL) if match1: BminValue = match1.group(1) BavgValue = match1.group(2) BmaxValue = match1.group(3) if int(lValue[0]) > int(BavgValue) or int( lValue[2]) < int(BavgValue): if int(lValue[0]) > int( BavgValue) and lValue[2] < int(BavgValue): slaState = "Latency-Above & Below threshold" elif int(lValue[0]) > int(BavgValue): slaState = "Latency-Below threshold" else: slaState = "Latency-Above threshold" Blatency = 0 except Exception as e: log.warning( "parsing Latency Message got failed.Error info : %s", str(e)) else: if "Invalid" in output1: pingResultB = "0" elif pingResultB == -1: pingResultB = output1 try: log.info( "Ping result success rate for A ENd Device : %s , For Z End Device : %s ", str(pingResultA), str(pingResultB)) if int(pingResultA.strip()) > 99 and int(pingResultB.strip()) > 99: if Blatency == 0 or Alatency == 0: return 0, AEndIntf, BEndIntf, pingA, pingB, slaState else: return 1, AEndIntf, BEndIntf, pingA, pingB, slaState else: return 0, AEndIntf, BEndIntf, pingA, pingB, slaState except Exception as e: return 0, AEndIntf, BEndIntf, pingA, pingB, slaState except Exception as e: if "Authentication" in str(e): subject = "Authentication failed on routers" + device1 + "," + device2 message = "Error while login to Routers.Authentication failed" sendMail(subject, message, 'exception') return 0, 0, 0, "Fail", 0, "" elif "timed-out" in str(e): log.warning("Device timed out error %s", str(e)) return 0, 0, 0, "0", 0, "" else: log.warning("Error occured while doing pingtest %s", str(e))
def setGOCFlag(db, values, val1, val2, val3, val4, val5, val6, val7, AEndIntf, BEndIntf, pingA, pingB, val8, val9, slaStates, phNumber): try: db.updateTable(values) msg = "" sev = "Sev2" message = "" circuitProvider = val7 if "down" in val6.lower(): msg = "Down" elif "loss" in val6.lower(): msg = "Packet Loss. Loss Percentage is :" + val6 + "%" else: msg = slaStates fetchedValues, fetchedValues1, flag = db.getInternalCircuitCountMatch( val7) if flag: circuitProvider = "" sev = "Sev1" for val in fetchedValues: slaState = "" for val1 in fetchedValues1: if str(val[9]) == str(val1[0]): circuitProvider = circuitProvider + "," + str(val1[0]) if "down" in str(val1[2]).lower(): slaState = "Down" elif "loss" in str(val1[2].lower()): slaState = "Packet Loss. Loss Percentage is :" + str( val1[2]) + "%" else: slaState = slaStates break phNumber1 = "" if "NO" in str(val[13]).strip(): phNumber1 = "" else: phNumber1 = str(val[13]).strip() message = message + "\n AEnd Device :" + str( val[4] ) + " Interface :" + str(val[5]) + "\n ZEnd Device :" + str( val[6] ) + " Interface :" + str( val[7] ) + "\n Circuit Provider :" + str( val[8] ) + "\n CircuitID :" + str(val[1]) + " OrderID :" + str( val[2] ) + "\n SLAStatus :" + slaState + "\n Phone Number :" + phNumber1 + "\n\n" else: message = "\n AEnd Device :" + val1 + " Interface : " + val2 + "\n ZEnd Device :" + val3 + " Interface :" + val4 + "\n Circuit Provider :" + val5 + "\n CircuitID :" + val8 + " OrderID :" + val9 + "\n SLASTatus :" + msg + "\n Phone Number :" + phNumber if pingA.strip() == "Fail": message += "\n\n Access to the routers got failed due to Authentication failed." else: try: if int(AEndIntf) == 0 and int( BEndIntf) == 0 and pingA == "0" and int(pingB) == 0: message += "\n\n Connection to device got timed-out" except Exception as e: message += "\n\n Ping result from Device " + AEndIntf + " to Device " + BEndIntf + " \n " + pingA + "\n\n Ping result from Device " + BEndIntf + " to Device " + AEndIntf + " \n " + pingB + "\n\n" subject = sev + " Circuit ID :" + circuitProvider msg = "Team" + "\n" + "\n" + "Please find the details about the devices\n\n" + message sendMail(subject, msg, sev) except Exception as e: log.warning("Exception(SETGocFLag Module) message : %s", str(e))
def checkAfterTimeout(db): while 1: try: record=db.selectTable('CoreCircuitStates') seconds=getDateTime() for rec1 in record: if str(rec1[3]) == 'YES': #log.info('GOC Flag set as YES.so move to next record') continue diff=Decimal(str(seconds)) - Decimal(str(rec1[4])) if int(diff) >= 60: setFlag=[] SqlQuery="" row=db.selectCoreRouterDetails(rec1[0]) for rec in row: log.info('Going to perform ping from both ends') result,AEndIntf,BEndIntf,pingA,pingB,slaState=pingTest(str(rec[10]),str(rec[11]),str(rec[5]),str(rec[7]),str(rec[12]),str(rec1[1])) if result == 0: #setFlag.append(rec1[0]) #setFlag.append('YES') #setFlag.append('0') if slaState.strip() == "0": SqlQuery="update CoreCircuitStates set GOCFlag = LTRIM(\'YES\'), Time = LTRIM(\'0\') where InternalCircuitID = LTRIM(\'"+str(rec1[0])+"\')" else: SqlQuery="update CoreCircuitStates set GOCFlag = LTRIM(\'YES\'), Time = LTRIM(\'0\') , SLAState = LTRIM(\'"+str(slaState)+"\') where InternalCircuitID = LTRIM(\'"+str(rec1[0])+"\')" phNumber="" if "NO" in str(rec[13]).strip(): phNumber="" else: phNumber=str(rec[13]).strip() setGOCFlag(db,SqlQuery,str(rec[4]),str(rec[5]),str(rec[6]),str(rec[7]),str(rec[8]),str(rec1[2]),str(rec1[0]),AEndIntf,BEndIntf,pingA,pingB,str(rec[1]),str(rec[2]),slaState,phNumber) else: log.info("Ping Result got passed.so going to delete InternalCircuitID state details from CoreCircuitState table") try: db.deleteRow(str(rec1[0]).strip()) except Exception as e: log.warning("Error occured while deleting record in CoreCircuitStates table.Error info : %s",str(e)) if "down" in rec1[2]: setFlag.append("UP") db.updateSLAState(setFlag) else: db.updateTime(setFlag) except Exception as e: #log.warning("Error Occured") pass
def read(username, password, db): # Login to INBOX imap = imaplib.IMAP4_SSL("outlook.office365.com", 993) imap.login(username, password) imap.select('EMC-Core-Mon') #imap.select('INBOX') for line in open('/opt/CoreMonitoringApp/Common/secretKey', 'r').readlines(): # Print all unread messages from a certain sender of interest passwordStr = str(line).strip() #print "MailID",line.strip() hash_object = hashlib.md5(passwordStr.encode()) hashPass = hash_object.hexdigest() status, response = imap.search(None, '(UNSEEN)') unread_msg_nums = response[0].split() for e_id in unread_msg_nums: #print "full msg : ",e_id _, response = imap.fetch(e_id, '(UID BODY[TEXT])') subject = 0 body = 0 flag = 0 fromMailID = "" typemsg, data = imap.fetch( e_id, '(RFC822.SIZE BODY[HEADER.FIELDS (SUBJECT)])') subject = data[0][1].lstrip('Subject: ').strip() + ' ' body = response[0][1] typemssg, data = imap.fetch( e_id, '(RFC822.SIZE BODY[HEADER.FIELDS (FROM)])') fromMailID = data[0][1].lstrip('From: ').strip() print "From MailID :-", fromMailID mailID = "" match = re.match(r'\s*\"(.*)\"\s*\<.*', str(fromMailID), re.M | re.I) nodeStringPresent = 0 cidStringPresent = 0 if match: mailID = str(match.group(1).strip()) else: mailID = fromMailID #print body print subject print "hashvalue", hashPass mdHash = 0 mdStringPresent = 0 if "md5" in str(body).lower(): bodyMsg = body.split("\n") for line in bodyMsg: #print "line",line if "md5" in line.lower(): mdStringPresent = 1 match = re.match( r'.*MD5hash\s*:(\s*.*<span.*white">|\s*)([a-zA-Z0-9]+)(</span><o:p>.*|</span><span .*>.*|<o:p>)', str(line), re.M | re.I) if match: hashValue = str(match.group(2)).strip() print hashValue print hashPass if hashValue.strip() == hashPass.strip(): mdHash = 1 print "matched", mdHash break if mdHash == 0: if mdStringPresent == 0: to_addr_list = str(common.getAdditionalMailID()).split(",") to_addr_list.append(str(fromMailID)) #to_addr_list.append('*****@*****.**') subjectHeading = subject message = "Hi " + mailID + ",\n\n The MD5 Hash key is missing from the email sent,Please add the correct MD5 Hash when sending the Acknowledgement Mail.\n\n\n Example: MD5hash: XXXXXXXXXXXXXXXXXXXXXXXX \n\n Thanks\n Core Montiorng App\n [email protected]" sendMail(subjectHeading, message, to_addr_list) else: print mdHash to_addr_list = str(common.getAdditionalMailID()).split(",") to_addr_list.append(str(fromMailID)) #to_addr_list.append('*****@*****.**') subjectHeading = subject message = "Hi " + mailID + ",\n\n The MD5 HASH sent for the acknowledge of Circut ID doesn't match.Please enter the correct MD5 hash.Please do not reply to this email.\n\n Thanks\n Core Montirong App\n [email protected]" sendMail(subjectHeading, message, to_addr_list) log.warning( 'Unable to match MDSHash value.Please use proper hash value in the mail.' ) continue #print circuitIDs[1] if "acknowledge" in str(body).lower(): flag = 1 log.info("Acknowledge string present in the Body message: ") if flag == 1: if "node" in str(subject).lower(): nodeStringPresent = 1 success = deleteDetails(subject, body, db) if success == 0: to_addr_list = str( common.getAdditionalMailID()).split(',') to_addr_list.append(str(fromMailID)) subjectHeading = "Nodes Down,Device-Type: XXXXXX, City: XXXXXXXX" message = "Hi " + str( mailID ) + ", \n\n The EMail Acknowledge format for Node Down is like below. \n\n Nodes Down:XXXX,XXXXX\n Status: Acknowledge\n MD5hash: xxxxxxxxxxxxxxxxxx.Please do not reply to this mail. \n\n Thanks\n Core Motinoring App\n [email protected]" sendMail(subjectHeading, message, to_addr_list) else: #bodyMsg=body.split("\n") #for line in bodyMsg: match = re.match( r'(.*Circuit\s*ID\s*|[a-zA-Z]*\s*:\s*CID|\s*CID\s*):(.*)', str(subject), re.M | re.I) #print "line",line if match: circuitIDs = str(match.group(2)).split(',') CIDs = circuitIDs #print CIDs #log.info("Going to delete CircuitID record from CoreCircuitState table which has been resolved .") for cid in CIDs: check = db.checkInTable(str(cid).strip()) if int(check) == 1: db.deleteRow(cid.strip()) log.info( "Record Deleted Successfully. Internal Circuit Id : %s ", str(cid)) else: #sendMail() to_addr_list = str( common.getAdditionalMailID()).split(",") to_addr_list.append(fromMailID) #to_addr_list.append('*****@*****.**') subject = "Circuit ID:" + str(cid) message = " Hi Team, \n\n The Circuit ID sent " + str( cid ) + " for acknowledgement does not exists in the Circuit State Table.Please enter the correct CircuitID or check if the incident has already been approved.\n\n\n Thanks,\n Core Monitoring App\n [email protected] " sendMail(subject, message, to_addr_list) else: to_addr_list = str( common.getAdditionalMailID()).split(',') to_addr_list.append(str(fromMailID)) subjectHeading = subject message = "Hi " + str( mailID ) + "\n\n The subject mail should like either \"Circuit ID: XXXXXXXXXXXXXXXXXX\" or \"CID: XXXXXXXXXXXXXXX\". Please check the Email Acknowledge mail format and send.\n\n Thanks\n Core Monitoring App\n [email protected]" sendMail(subjectHeading, message, to_addr_list) # print "Internal Circuit ID "+cid.strip()+" Record Deleted Succeffully from mCoreCircuitStates Table" #db.deleteRow(value) # Mark them as seen for e_id in unread_msg_nums: imap.store(e_id, '+FLAGS', '\Seen')
def createDBObject(): try: db = DBCode() return db except Exception as e: log.warning("Exception occured during Database Connection %s",str(e))
def read(username, password,db): # Login to INBOX imap = imaplib.IMAP4_SSL("outlook.office365.com", 993) imap.login(username, password) imap.select('EMC-Core-Mon') #imap.select('INBOX') for line in open('/opt/CoreMonitoringApp/Common/secretKey','r').readlines(): # Print all unread messages from a certain sender of interest passwordStr=str(line).strip() #print "MailID",line.strip() hash_object=hashlib.md5(passwordStr.encode()) hashPass=hash_object.hexdigest() status, response = imap.search(None, '(UNSEEN)') unread_msg_nums = response[0].split() for e_id in unread_msg_nums: #print "full msg : ",e_id _, response = imap.fetch(e_id, '(UID BODY[TEXT])') subject=0 body=0 flag=0 fromMailID="" typemsg,data = imap.fetch(e_id,'(RFC822.SIZE BODY[HEADER.FIELDS (SUBJECT)])') subject = data[0][1].lstrip('Subject: ').strip() + ' ' body = response[0][1] typemssg,data= imap.fetch(e_id,'(RFC822.SIZE BODY[HEADER.FIELDS (FROM)])') fromMailID = data[0][1].lstrip('From: ').strip() print "From MailID :-",fromMailID mailID="" match=re.match(r'\s*\"(.*)\"\s*\<.*',str(fromMailID),re.M|re.I) nodeStringPresent=0 cidStringPresent=0 if match: mailID=str(match.group(1).strip()) else: mailID=fromMailID #print body print subject print "hashvalue",hashPass mdHash=0 mdStringPresent=0 if "md5" in str(body).lower(): bodyMsg=body.split("\n") for line in bodyMsg: #print "line",line if "md5" in line.lower(): mdStringPresent = 1 match=re.match(r'.*MD5hash\s*:(\s*.*<span.*white">|\s*)([a-zA-Z0-9]+)(</span><o:p>.*|</span><span .*>.*|<o:p>)',str(line),re.M|re.I) if match: hashValue=str(match.group(2)).strip() print hashValue print hashPass if hashValue.strip() == hashPass.strip(): mdHash=1 print "matched",mdHash break if mdHash == 0: if mdStringPresent == 0: to_addr_list=str(common.getAdditionalMailID()).split(",") to_addr_list.append(str(fromMailID)) #to_addr_list.append('*****@*****.**') subjectHeading=subject message="Hi "+mailID+",\n\n The MD5 Hash key is missing from the email sent,Please add the correct MD5 Hash when sending the Acknowledgement Mail.\n\n\n Example: MD5hash: XXXXXXXXXXXXXXXXXXXXXXXX \n\n Thanks\n Core Montiorng App\n [email protected]" sendMail(subjectHeading,message,to_addr_list) else: print mdHash to_addr_list=str(common.getAdditionalMailID()).split(",") to_addr_list.append(str(fromMailID)) #to_addr_list.append('*****@*****.**') subjectHeading=subject message="Hi "+mailID+",\n\n The MD5 HASH sent for the acknowledge of Circut ID doesn't match.Please enter the correct MD5 hash.Please do not reply to this email.\n\n Thanks\n Core Montirong App\n [email protected]" sendMail(subjectHeading,message,to_addr_list) log.warning('Unable to match MDSHash value.Please use proper hash value in the mail.') continue #print circuitIDs[1] if "acknowledge" in str(body).lower(): flag = 1 log.info("Acknowledge string present in the Body message: ") if flag == 1: if "node" in str(subject).lower(): nodeStringPresent=1 success=deleteDetails(subject,body,db) if success == 0: to_addr_list=str(common.getAdditionalMailID()).split(',') to_addr_list.append(str(fromMailID)) subjectHeading="Nodes Down,Device-Type: XXXXXX, City: XXXXXXXX" message="Hi "+str(mailID)+", \n\n The EMail Acknowledge format for Node Down is like below. \n\n Nodes Down:XXXX,XXXXX\n Status: Acknowledge\n MD5hash: xxxxxxxxxxxxxxxxxx.Please do not reply to this mail. \n\n Thanks\n Core Motinoring App\n [email protected]" sendMail(subjectHeading,message,to_addr_list) else: #bodyMsg=body.split("\n") #for line in bodyMsg: match=re.match(r'(.*Circuit\s*ID\s*|[a-zA-Z]*\s*:\s*CID|\s*CID\s*):(.*)',str(subject),re.M|re.I) #print "line",line if match: circuitIDs=str(match.group(2)).split(',') CIDs=circuitIDs #print CIDs #log.info("Going to delete CircuitID record from CoreCircuitState table which has been resolved .") for cid in CIDs: check=db.checkInTable(str(cid).strip()) if int(check) == 1: db.deleteRow(cid.strip()) log.info("Record Deleted Successfully. Internal Circuit Id : %s ",str(cid)) else: #sendMail() to_addr_list=str(common.getAdditionalMailID()).split(",") to_addr_list.append(fromMailID) #to_addr_list.append('*****@*****.**') subject="Circuit ID:"+str(cid) message=" Hi Team, \n\n The Circuit ID sent "+str(cid)+" for acknowledgement does not exists in the Circuit State Table.Please enter the correct CircuitID or check if the incident has already been approved.\n\n\n Thanks,\n Core Monitoring App\n [email protected] " sendMail(subject,message,to_addr_list) else: to_addr_list=str(common.getAdditionalMailID()).split(',') to_addr_list.append(str(fromMailID)) subjectHeading=subject message="Hi "+str(mailID)+"\n\n The subject mail should like either \"Circuit ID: XXXXXXXXXXXXXXXXXX\" or \"CID: XXXXXXXXXXXXXXX\". Please check the Email Acknowledge mail format and send.\n\n Thanks\n Core Monitoring App\n [email protected]" sendMail(subjectHeading,message,to_addr_list) # print "Internal Circuit ID "+cid.strip()+" Record Deleted Succeffully from mCoreCircuitStates Table" #db.deleteRow(value) # Mark them as seen for e_id in unread_msg_nums: imap.store(e_id, '+FLAGS', '\Seen')
def pingTest(device1,device2,aIntf,bIntf,latencyValue,latency): try: username,password=common.getRouterCredentials() #password="******" delayFactor,loop = common.getSendCommandDelayFactor() timeout=getTimeout() slaState="0" AEnddevice = ConnectHandler(device_type="cisco_ios", ip=device1, username=username, password=password) command='sh run interface '+aIntf AEnddevice.clear_buffer() output1 = AEnddevice.send_command(command,int(delayFactor),int(loop)) output1 = str(output1) log.info("Device %s. Command %s. Output %s",str(device1),str(command),str(output1)) match=re.match(r'.*ip\s+address\s+(.*)\s+255.*',output1,re.DOTALL) if match: AEndIntf=match.group(1) BEnddevice = ConnectHandler(device_type="cisco_ios", ip=device2, username=username, password=password) command='sh run interface '+bIntf BEnddevice.clear_buffer() output1 = BEnddevice.send_command(command,int(delayFactor),int(loop)) output1 = str(output1) log.info("Device %s. Command %s. Output %s",str(device2),str(command),str(output1)) match=re.match(r'.*ip\s+address\s+(.*)\s+255.*',output1,re.DOTALL) if match: BEndIntf=match.group(1) command='ping '+BEndIntf.strip()+' source '+AEndIntf.strip()+' repeat 30 timeout '+timeout AEnddevice.clear_buffer() output1 = AEnddevice.send_command(command,int(delayFactor),int(loop)) output1 = str(output1) pingA = output1 log.info("Device %s.Ping Command %s. Output %s",str(device1),str(command),str(output1)) Alatency=1 pingResultA=-1 match=re.match(r'.*rate\s+is\s+(.*)\s+percent.*',output1,re.DOTALL) if match: pingResultA=match.group(1) try: if "latency" in latency.lower(): lValue=latencyValue.split(",") match1=re.match(r'.*min/avg/max\s*=\s*(\d+)/(\d+)/(\d+).*',output1,re.DOTALL) if match1: AminValue=match1.group(1) AavgValue=match1.group(2) AmaxValue=match1.group(3) if int(lValue[0]) > int(AavgValue) or int(lValue[2]) < int(AavgValue): if int(lValue[0]) > int(AavgValue) and int(lValue[2]) < int(AavgValue): slaState="Latency-Above & Below threshold" elif int(lValue[0]) > int(AavgValue): slaState="Latency-Below threshold" else: slaState="Latency-Above threshold" Alatency=0 except Exception as e: log.warning("parsing Latency Message got failed.Error info : %s",str(e)) else: if "Invalid" in output1: pingResultA="0" elif pingResultA == -1: pingResultA = output1 # pingResultA="0" command='ping '+AEndIntf.strip()+' source '+BEndIntf.strip()+' time '+timeout+' repeat 30' BEnddevice.clear_buffer() output1 = BEnddevice.send_command(command,int(delayFactor),int(loop)) output1 = str(output1) pingB = output1 print "PingB",pingB log.info("Device %s. Ping Command %s. Output %s",str(device2),str(command),str(output1)) Blatency=1 pingResultB=-1 match=re.match(r'.*rate\s+is\s+(.*)\s+percent.*',output1,re.DOTALL) if match: pingResultB=match.group(1) try: if "latency" in latency.lower(): lValue=latencyValue.split(",") match1=re.match(r'.*min/avg/max\s*=\s*(\d+)/(\d+)/(\d+).*',output1,re.DOTALL) if match1: BminValue=match1.group(1) BavgValue=match1.group(2) BmaxValue=match1.group(3) if int(lValue[0]) > int(BavgValue) or int(lValue[2]) < int(BavgValue): if int(lValue[0]) > int(BavgValue) and lValue[2] < int(BavgValue) : slaState="Latency-Above & Below threshold" elif int(lValue[0]) > int(BavgValue): slaState="Latency-Below threshold" else: slaState="Latency-Above threshold" Blatency=0 except Exception as e: log.warning("parsing Latency Message got failed.Error info : %s",str(e)) else: if "Invalid" in output1: pingResultB="0" elif pingResultB == -1: pingResultB = output1 try: log.info("Ping result success rate for A ENd Device : %s , For Z End Device : %s ",str(pingResultA),str(pingResultB)) if int(pingResultA.strip()) > 99 and int(pingResultB.strip()) > 99: if Blatency == 0 or Alatency == 0: return 0,AEndIntf,BEndIntf,pingA,pingB,slaState else: return 1,AEndIntf,BEndIntf,pingA,pingB,slaState else: return 0,AEndIntf,BEndIntf,pingA,pingB,slaState except Exception as e: return 0,AEndIntf,BEndIntf,pingA,pingB,slaState except Exception as e: if "Authentication" in str(e): subject="Authentication failed on routers"+device1+","+device2 message="Error while login to Routers.Authentication failed" sendMail(subject,message,'exception') return 0,0,0,"Fail",0,"" elif "timed-out" in str(e): log.warning("Device timed out error %s",str(e)) return 0,0,0,"0",0,"" else: log.warning("Error occured while doing pingtest %s",str(e))
def checkAfterTimeout(db): while 1: try: record = db.selectTable('CoreCircuitStates') seconds = getDateTime() for rec1 in record: if str(rec1[3]) == 'YES': #log.info('GOC Flag set as YES.so move to next record') continue diff = Decimal(str(seconds)) - Decimal(str(rec1[4])) if int(diff) >= 60: setFlag = [] SqlQuery = "" row = db.selectCoreRouterDetails(rec1[0]) for rec in row: log.info('Going to perform ping from both ends') result, AEndIntf, BEndIntf, pingA, pingB, slaState = pingTest( str(rec[10]), str(rec[11]), str(rec[5]), str(rec[7]), str(rec[12]), str(rec1[1])) if result == 0: #setFlag.append(rec1[0]) #setFlag.append('YES') #setFlag.append('0') if slaState.strip() == "0": SqlQuery = "update CoreCircuitStates set GOCFlag = LTRIM(\'YES\'), Time = LTRIM(\'0\') where InternalCircuitID = LTRIM(\'" + str( rec1[0]) + "\')" else: SqlQuery = "update CoreCircuitStates set GOCFlag = LTRIM(\'YES\'), Time = LTRIM(\'0\') , SLAState = LTRIM(\'" + str( slaState ) + "\') where InternalCircuitID = LTRIM(\'" + str( rec1[0]) + "\')" phNumber = "" if "NO" in str(rec[13]).strip(): phNumber = "" else: phNumber = str(rec[13]).strip() setGOCFlag(db, SqlQuery, str(rec[4]), str(rec[5]), str(rec[6]), str(rec[7]), str(rec[8]), str(rec1[2]), str(rec1[0]), AEndIntf, BEndIntf, pingA, pingB, str(rec[1]), str(rec[2]), slaState, phNumber) else: log.info( "Ping Result got passed.so going to delete InternalCircuitID state details from CoreCircuitState table" ) try: db.deleteRow(str(rec1[0]).strip()) except Exception as e: log.warning( "Error occured while deleting record in CoreCircuitStates table.Error info : %s", str(e)) if "down" in rec1[2]: setFlag.append("UP") db.updateSLAState(setFlag) else: db.updateTime(setFlag) except Exception as e: #log.warning("Error Occured") pass
def checkDataBase(dbs): try: record = dbs.selectTable('nodeState') for rec in record: flag = str(rec[3]) if flag.strip() == 'YES': Count = len(str(rec[2]).split(',')) if int(Count) == 1: result = nodePingtest(str(rec[4]).strip()) if result == "0": log.info( "Skipping mail operations and delete info from nodeState table." ) self.db.deleteRowNode(str(rec[0])) print 'Node info got deleted.', str(rec[0]) continue elif result == "1": msg = "" subject = "Sev2 Node Down, Device-Type :" + str( rec[5]) + ", City:" + str(rec[0]) msg += " Device Name :" + str( rec[2] ) + " IP-Address :" + str( rec[4] ) + " Device-Type : Core City : " + str( rec[0] ) + " State : Packet Loss\n Ping Result \n Got exception while doing ping test, please refer logs for more info " message = "Team \n\n\n Please investigate this node down incident \n\n " + msg else: msg = "" subject = "Sev2 Node Down, Device-Type :" + str( rec[5]) + ", City:" + str(rec[0]) msg += " Device Name :" + str( rec[2] ) + " IP-Address :" + str( rec[4] ) + " Device-Type : Core City : " + str( rec[0] ) + " State : Packet Loss\n Ping Result \n " + str( result) message = "Team \n\n\n Please investigate this node down incident \n\n " + msg else: # subject="Sev1 Nodes Down, Device-Type :"+str(rec[5])+" ,City:"+str(rec[0]) nodeNames = str(rec[2]).split(',') ipadd = str(rec[4]).split(',') msg = "" removeNodes = [] counting = 0 for i in range(0, len(nodeNames)): result = nodePingtest(str(ipadd[i]).strip()) if result == "0": removeNodes.append(str(nodeNames[i])) elif reult == "1": counting = counting + 1 msg += " Device Name :" + str( nodeNames[i] ) + " IP-Address :" + str( ipadd[i] ) + " Device-Type : Core City : " + str( rec[0] ) + " State : Packet Loss\n Ping Result \n Got exception while doing ping test, please refer logs for more info " else: counting = counting + 1 msg += " Device Name :" + str( nodeNames[i] ) + " IP-Address :" + str( ipadd[i] ) + " Device-Type : Core City : " + str( rec[0] ) + " State : Packet Loss \n Ping Result \n " + str( result) if counting > 1: subject = "Sev1 Nodes Down, Device-Type :" + str( rec[5]) + " ,City:" + str(rec[0]) elif counting == 1: subject = "Sev2 Nodes Down, Device-Type :" + str( rec[5]) + " ,City:" + str(rec[0]) else: log.info( "Skipping mail operations and delete info from nodeState table." ) self.db.deleteRowNode(str(rec[0])) print 'Node info got deleted.', str(rec[0]) continue message = "Team \n\n\n Please find details about down nodes \n\n " + msg if removeNodes: dbNodeValue = str(rec[2]).split(",") dbIpaddress = str(rec[4]).split(",") for node in removeNodes: index = dbNodeValue.index(str(node)) del dbIpaddress[index] dbNodeValue.remove(str(node)) value = ",".join(dbNodeValue) Count1 = len(dbNodeValue) ipadd = ",".join(dbIpaddress) values = [] values.append(str(rec[0])) values.append(str(Count1)) values.append(str(value)) values.append('NO') values.append(str(ipadd)) dbs.updateNodeStateTable(values) print 'Node record got updated.values are', str(values) sendMail(subject, message) continue sendMail(subject, message) value = [] value.append(str(rec[0])) value.append('NO') dbs.updateNodeStateTableAfterMailSend(value) except Exception as e: log.warning( "Node Down-- Exception occured(CheckDatabse Module). Error info : %s", str(e))
def remote_start(case_name, result_file, test_data, pic_name, user_manager, parameters, performance): global session, rdp_server, user if parameters['vdi'].upper() == 'RDP': session = parameters.get('session') rdp_server = user_manager.get_a_available_key(session.lower()) conn = vdi_connection.RDPLinux(user=user, parameters=parameters, rdp_server=rdp_server) elif parameters['vdi'].upper() == 'VIEW': conn = vdi_connection.ViewLinux(user=user, parameters=parameters) elif parameters['vdi'].upper() == 'CITRIX': conn = vdi_connection.CitrixLinux(user=user, parameters=parameters) else: conn = None log.info('init connection instance') first_collect_flag = False if conn: # VDI Test conn_flag = False for count in range(2): logon = conn.logon(parameters['session']) if logon is True: pymouse.PyMouse().click(1, 1) log.info('successfully logon session: {}'.format(parameters['session'])) steps = { 'step_name': 'logon vdi', 'result': 'Pass', 'expect': '', # can be string or pic path 'actual': 'logon successfully', 'note': ''} common_function.update_cases_result(result_file, case_name, steps) conn_flag = True pic_sim, diff_res, first_collect_flag = check_layout(os.path.join(test_data, 'td_multiple_display', '{}_pic'.format(parameters['vdi']).lower(), pic_name)) if pic_sim >= 0.99: log.info('check layout pass') steps = { 'step_name': 'check vdi layout', 'result': 'Pass', 'expect': 'similarity > 99%', # can be string or pic path 'actual': 'layout and resolution set correctly: {}'.format(pic_sim), 'note': ''} common_function.update_cases_result(result_file, case_name, steps) else: log.error('check layout Fail, autual: {}'.format(pic_sim)) shutil.copy(common_function.get_current_dir('temp.png'), common_function.get_current_dir('Test_Report', 'img', '{}.jpg'.format(case_name))) steps = { 'step_name': 'check vdi layout', 'result': 'Fail', 'expect': 'similarity > 99%', # can be string or pic path 'actual': 'img/{}.png'.format(case_name), # can be string or pic path 'note': 'actual result: {}'.format(pic_sim)} common_function.update_cases_result(result_file, case_name, steps) if not os.path.exists(common_function.get_current_dir('Test_Report', 'img')): os.mkdir(common_function.get_current_dir('Test_Report', 'img')) try: common_function.check_free_memory("beforeremote") picture_operator.save_from_data( '{}'.format(common_function.get_current_dir('Test_Report', 'img', '{}.png'.format( case_name))), diff_res) common_function.check_free_memory("afterremote") except: save_fail_path = common_function.get_current_dir('Test_Report', 'img', 'save_fail_remote_{}.txt'.format(case_name)) f = open(save_fail_path, "w") f.write(traceback.format_exc()) f.close() finally: log.debug(gc.garbage) gc.collect() log.info("collecting performance data") path = "{}/Test_Report/performance/".format(common_function.get_current_dir()) if not os.path.exists(path): os.makedirs(path) performance.file = path + "{}.txt".format("".join(case_name.split("_")[2:-4]).replace(" ", "")) performance.start() conn.logoff() break elif isinstance(logon, str): log.warning('logon fail in cycle {}'.format(count)) # wait 2min try again time.sleep(120) if not user: log.info("user get none before, try to get a new user") user = user_manager.get_a_available_key() conn.user = user if parameters['vdi'].upper() == 'RDP' and not rdp_server: log.info("try to get rdp_server") session = parameters.get('session') rdp_server = user_manager.get_a_available_key(session.lower()) conn.server = rdp_server conn.domain_list = [rdp_server] continue else: log.error("logon fail in cycle {}& white list can't catch it, break!".format(count)) break if not conn_flag: log.error('logon fail, and try count finished') steps = { 'step_name': 'logon vdi', 'result': 'Fail', 'expect': '', # can be string or pic path 'actual': 'Fail to logon within 6 mins', # can be string or pic path 'note': ''} common_function.update_cases_result(result_file, case_name, steps) return False, first_collect_flag return True, first_collect_flag
def checkDataBase(dbs): try: record=dbs.selectTable('nodeState') for rec in record: flag=str(rec[3]) if flag.strip() == 'YES': Count=len(str(rec[2]).split(',')) if int(Count) == 1: result=nodePingtest(str(rec[4]).strip()) if result == "0": log.info("Skipping mail operations and delete info from nodeState table.") self.db.deleteRowNode(str(rec[0])) print 'Node info got deleted.',str(rec[0]) continue elif result == "1": msg="" subject="Sev2 Node Down, Device-Type :"+str(rec[5])+", City:"+str(rec[0]) msg+=" Device Name :"+str(rec[2])+" IP-Address :"+str(rec[4])+" Device-Type : Core City : "+str(rec[0])+" State : Packet Loss\n Ping Result \n Got exception while doing ping test, please refer logs for more info " message="Team \n\n\n Please investigate this node down incident \n\n "+msg else: msg="" subject="Sev2 Node Down, Device-Type :"+str(rec[5])+", City:"+str(rec[0]) msg+=" Device Name :"+str(rec[2])+" IP-Address :"+str(rec[4])+" Device-Type : Core City : "+str(rec[0])+" State : Packet Loss\n Ping Result \n "+str(result) message="Team \n\n\n Please investigate this node down incident \n\n "+msg else: # subject="Sev1 Nodes Down, Device-Type :"+str(rec[5])+" ,City:"+str(rec[0]) nodeNames=str(rec[2]).split(',') ipadd=str(rec[4]).split(',') msg="" removeNodes=[] counting=0 for i in range(0,len(nodeNames)): result=nodePingtest(str(ipadd[i]).strip()) if result == "0": removeNodes.append(str(nodeNames[i])) elif reult == "1": counting = counting + 1 msg+=" Device Name :"+str(nodeNames[i])+" IP-Address :"+str(ipadd[i])+" Device-Type : Core City : "+str(rec[0])+" State : Packet Loss\n Ping Result \n Got exception while doing ping test, please refer logs for more info " else: counting=counting+1 msg+=" Device Name :"+str(nodeNames[i])+" IP-Address :"+str(ipadd[i])+" Device-Type : Core City : "+str(rec[0])+" State : Packet Loss \n Ping Result \n "+str(result) if counting > 1: subject="Sev1 Nodes Down, Device-Type :"+str(rec[5])+" ,City:"+str(rec[0]) elif counting == 1: subject="Sev2 Nodes Down, Device-Type :"+str(rec[5])+" ,City:"+str(rec[0]) else: log.info("Skipping mail operations and delete info from nodeState table.") self.db.deleteRowNode(str(rec[0])) print 'Node info got deleted.',str(rec[0]) continue message="Team \n\n\n Please find details about down nodes \n\n "+msg if removeNodes: dbNodeValue=str(rec[2]).split(",") dbIpaddress=str(rec[4]).split(",") for node in removeNodes: index=dbNodeValue.index(str(node)) del dbIpaddress[index] dbNodeValue.remove(str(node)) value=",".join(dbNodeValue) Count1=len(dbNodeValue) ipadd=",".join(dbIpaddress) values=[] values.append(str(rec[0])) values.append(str(Count1)) values.append(str(value)) values.append('NO') values.append(str(ipadd)) dbs.updateNodeStateTable(values) print'Node record got updated.values are',str(values) sendMail(subject,message) continue sendMail(subject,message) value=[] value.append(str(rec[0])) value.append('NO') dbs.updateNodeStateTableAfterMailSend(value) except Exception as e: log.warning("Node Down-- Exception occured(CheckDatabse Module). Error info : %s",str(e))