def get_pin_status_mobile_call(astro_id, pin_id, restart_time): try: count = 0 while count < MAX_STATUS_COUNT: count += 1 time_from = dt.strftime( restart_time - datetime.timedelta(hours=count * 20), TIME_FORMAT) time_to = dt.strftime(restart_time, TIME_FORMAT) payload = { 'query': 'Message_ACTION:CHANGE_DEVICE_STATUS AND ASTRO_ID:' + astro_id + ' AND Message_PIN_ID:' + pin_id + ' AND ENV:PROD', 'from': time_from, 'to': time_to, 'fields': 'DYFO_TIME,Message_NEW_STATUS' } data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE) if data['total_results'] > 0: return { 'mobile_call_time': data['messages'][0]['message']['DYFO_TIME'], 'pin_status': data['messages'][0]['message']['Message_NEW_STATUS'] } return {'mobile_call_time': 'Not Found', 'pin_status': 'Not Found'} except Exception as e: logger.error("Exeption in getting Mobile Pin statuses" + str(e))
def handle_UpdateRestarts(data): """ Saves Restarts to DB using Deserialization :param data: :return: """ try: restart_alerts = {} for data_item in data: rs = RestartsSerializer(data=data_item) if rs.is_valid(): try: rs.save() rsa = check_restart_alert(rs.data) if rsa is not "": restart_alerts[rs.data['astro_id']] = rsa except Exception as e: logger.info("Exception in saving" + str(e)) else: logger.info("Errors in saving" + str(rs.errors)) return restart_alerts except Exception as e: logger.error("Exception in handling restarts. Exception" + str(e))
def handle_UpdateQInterrupts(interrupts, house_id): """ Saves Severe Quick Interrupts to Db :param interrupts: :param house_id: :return: """ try: qi_alerts = {} for data_item in interrupts: rs = QuickInterruptSerializer(data=data_item) if rs.is_valid(): try: rs.save() qi_alerts[rs.data['astro_id']] = check_quick_interrupt_alert(rs.data) except Exception as e: logger.info("Exception in saving" + str(e)) else: logger.info("Errors in saving" + str(rs.errors)) return qi_alerts except Exception as e: logger.error("Exception in handling quick interrupts. Exception" + str(e))
def handle_UpdateLongCutoff(data, house_id): """ Checks if previously Inactive astros are still Inactive or not. If they have reconnected, mark them as active. Makes DB entries for other Long Cutoffs. :param obj: :param house_id: :return: """ try: ld_objects = LongDisconnect.objects.filter(house_id=house_id) ldo_alert = {} for ld in ld_objects: if ld.astro_id not in data: ld.delete() for key, val in data.iteritems(): try: ldo = LongDisconnect.objects.get(astro_id=key) except: ldo = LongDisconnect.objects.create(house_id=house_id, astro_id=key) ldo_alert[key] = check_long_disconnect_alert(key, val) ldo.disconnected_at = val ldo.status = "INACTIVE" ldo.save() # ldo_alert=check_long_disconnect_alert(key, val) return ldo_alert except Exception as e: logger.error("Exception in updating long cut off. Exception " + str(e))
def authorization(email): try: a = Auth() return a.authenticate(email, 'telescope') except Exception as e: logger.error("Exception in authorization. Exception" + str(e)) raise e
def get_last_healthcheck(astro_id, restart_time): try: count = 0 while count < MAX_HEALTHCHECK_COUNT: count += 1 time_from = dt.strftime( restart_time - datetime.timedelta(hours=count * 12), TIME_FORMAT) time_to = dt.strftime(restart_time, TIME_FORMAT) payload = { 'query': 'Message_ASTRO_ID:' + astro_id + ' AND ENV:PROD', 'from': time_from, 'to': time_to, 'fields': 'DYFO_TIME', 'filter': 'streams:' + ASTRO_STATS_STREAM_ID } healthcheck_data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE) if healthcheck_data['total_results'] > 0: return healthcheck_data['messages'][0]['message']['DYFO_TIME'] return None except Exception as e: logger.error( "Exception in getting last helathcheck of astro from Graylog. Exception " + str(e)) raise e
def checkAstroRestarts(self, range): try: self.checkWifiDisconnections(range) restarts = [] payload = {'query': 'HOUSE_ID:' + self.house_id + ' AND Message_ACTION:ASTRO_INITIATED AND ENV:PROD', 'range': range, 'fields': 'ASTRO_ID,DYFO_TIME,Message_ASTRO_SKETCH_VERSION, Message_RESTART_REASON'} data = make_graylog_request(payload, GRAYLOG_SEARCH_RELATIVE) if data == None or 'messages' not in data: return restarts for item in data['messages']: try: message = item['message'] astro_id = message['ASTRO_ID'] restart_time = message['DYFO_TIME'] # sketch_version = message['Message_ASTRO_SKETCH_VERSION'] if astro_id not in self.astros: continue restart_time_obj = dt.strptime(restart_time, DYFO_TIME_FORMAT_INPUT) # device_type = get_device_type(sketch_version) mobile_pin_status, interrupt_pin_status = get_pin_statuses(restart_time_obj, self.astros[astro_id][1], astro_id) wifi_histogram = {} wifi_disconnections = [] # , wifi_histogram = self.checkWifiDisconnections(range, astro_id) code_version = House.objects.get(house_id=self.house_id).code_version if "Message_RESTART_REASON" in message: reason=RESTART_REASON.get(message["Message_RESTART_REASON"],"First time up") comment="Derived from astro" else: analysis = HealthAnalysis(restart_time_obj, wifi_histogram, wifi_disconnections, self.house_wifi_disconnections, code_version) reason, comment = analysis.analyse(astro_id, restart_time_obj) restart = {} restart['house_id'] = self.house_id restart['astro_id'] = astro_id restart['restart_time'] = restart_time_obj restart['wifi_disconnections'] = wifi_disconnections restart['reason'] = reason restart['comment'] = comment restart['mobile_pin_status'] = mobile_pin_status restart['interrupt_pin_status'] = interrupt_pin_status restart['device_type'] = self.astros[astro_id][1] restarts.append(restart) except Exception as e: logger.error("Exception in getting restart. Exception " + str(e)) return restarts except Exception as e: logger.error("Exception in checking restarts. Exception" + str(e)) raise e
def check_quick_interrupt_alert(qi): try: return "Quick Interrupt Detected on Astro " + str( qi['astro_id']) + " on Pin " + str( qi['pin']) + " with count" + str(qi['count']) + ".\n" except Exception as e: logger.error("Exception in checking quick interrupt alert" + str(e)) raise e
def handle_UpdateAll(): try: houses = get_house_statuses_from_drona() blacklisted = BLACKLISTED for house in houses: if house['display_id'] in blacklisted: continue try: h = House.objects.get(house_id=house['display_id']) except: h = House.objects.create(house_id=house['display_id']) if (not house['master_status']) and h.status == 'ACTIVE': h.status = 'ERRORED' + " " + datetime.datetime.now(pytz.timezone('Asia/Calcutta')).strftime( "%Y-%m-%d %H:%M:%S") if house['master_status']: h.status = 'ACTIVE' h.code_version = house['code_version'] h.save() houses = House.objects.filter(to_update=True) alerts = {} for house in houses: dyfo_house = DyfoHouseHealth(house.house_id) data = dyfo_house.checkAstroRestarts(DAY_SECONDS) alert = handle_UpdateRestarts(data) if alert: if house.house_id not in alerts: alerts[house.house_id] = {} alerts[house.house_id]['name'] = house.house_name alerts[house.house_id]['Restarts'] = alert data = dyfo_house.checkQuickInterrupt(DAY_SECONDS) alert = handle_UpdateQInterrupts(data, house.house_id) if alert: if house.house_id not in alerts: alerts[house.house_id] = {} alerts[house.house_id]['name'] = house.house_name alerts[house.house_id]['Quick_Interrupts'] = alert data = dyfo_house.checkLongDisconnect() alert = handle_UpdateLongCutoff(data, house.house_id) if alert: if house.house_id not in alerts: alerts[house.house_id] = {} alerts[house.house_id]['name'] = house.house_name alerts[house.house_id]['Long_Disconnect'] = alert if alerts: send_alert(alerts) update_last_update() except Exception as e: logger.error("Exception in updating Houses. Exception " + str(e)) raise e
def get_astro_states_from_drona(): try: res = rest_call.get(MASTER_SERVER_URL + ":" + str(MASTER_SERVER_PORT) + "/device/astro_state_choices", timeout=20) return json.loads(res.text)["choices"] except Exception as e: logger.error( "Exception in getting astro states choice list from Drona. Exception " + str(e)) raise e
def getConfigValue(key): """ Get Configuration value from Db :param key: :return: """ try: c = Configs.objects.get(key=key) return c.value except Exception as e: logger.error("Exception in getting config value of " + key + "from Database. Exception " + str(e))
def get_start_time(): """ Returns time based on Dispaly interval defined in Db :return: """ try: return datetime.datetime.now() + datetime.timedelta( hours=5) - datetime.timedelta( hours=int(getConfigValue(DATA_DISPLAY_RANGE))) except Exception as e: logger.error( "Exception in getting start time for event according to user defined display range" + str(e))
def checkLongDisconnect(self): # print("Long Disconnected Live Astros") try: long_disconnects = {} for key, val in self.astros.iteritems(): if val[0] is not None and (dt.now() + datetime.timedelta(hours=5, minutes=30) - dt.strptime(val[0], DRONA_TIME_FORMAT)) > datetime.timedelta( hours=4): long_disconnects[key] = val[0] return long_disconnects except Exception as e: logger.error("Exception in checking long disconnect. Exception" + str(e)) raise e
def check_restart_alert(rs): try: time = get_start_time() rscount = Restart.objects.filter(astro_id=rs['astro_id'], restart_time__gt=time).count() if rscount >= RESTART_ALERT_THRESHOLD: return str(rs["astro_id"]) + " restarted " + str( rscount) + " times in the past " + getConfigValue( DATA_DISPLAY_RANGE ) + " hours from the time of this alert.\n" return "" except Exception as e: logger.error("Exception in checking restart alert" + str(e)) raise e
def update_last_update(): """ Updates the Last Updates attribute in Db :return: """ try: c = Configs.objects.get(key=LAST_UPDATED) c.value = datetime.datetime.now( pytz.timezone('Asia/Calcutta')).strftime("%Y-%m-%d %H:%M:%S") c.save() except Exception as e: logger.error("Exception in updating last updated time. Exception " + str(e)) raise e
def get_house_statuses_from_drona(): """ Status of master from Drona :return: """ try: houses = rest_call.get(MASTER_SERVER_URL + ":" + str(MASTER_SERVER_PORT) + "/house/all", timeout=20) houses = json.loads(houses.text) return houses except Exception as e: logger.error( "Exception in getting House Status from Drona. Exception " + str(e))
def make_graylog_request(payload, uri): """ :param payload: :param uri: :return: """ try: url = GRAYLOG_SERVER_URL + uri + "?{}".format( urllib.urlencode(payload)) response = graylog_rest_call.get( url, retry=True, headers={'Accept': 'application/json'}, timeout=5) return json.loads(response.text, object_pairs_hook=OrderedDict) except Exception as e: logger.error("Exception in making graylog request. Exception " + str(e)) raise e
def send_alert(alerts): try: alert_string = "" for house, val in alerts.iteritems(): alert_string += "<b>" + house + "</b>" + "<br>" name = val.pop('name') alert_string += "<b>" + name + "</b>" + "<br>" for type, values in val.iteritems(): alert_string += "<b>" + type + "</b>" + "<br>" for astro, values2 in values.iteritems(): alert_string += values2 + "<br>" alert_string += "Visit " + TELESCOPE_URL + " for detailed information." user = yamjam()['keeper_user'] m = Mailer(user) m.send_mail("Lense Alert", alert_string, ['*****@*****.**']) except Exception as e: logger.error("Exception in sending alert" + str(e))
def get_last_wifi_disconnect(astro_id, restart_time): try: time_from = dt.strftime(restart_time - datetime.timedelta(hours=12), TIME_FORMAT) time_to = dt.strftime(restart_time, TIME_FORMAT) payload = { 'query': 'Message_ASTRO_ID:' + astro_id + ' AND ENV:PROD AND Message_ACTION:ASTRO_ALERT AND Message_STATUS:Connected', 'from': time_from, 'to': time_to, 'fields': 'DYFO_TIME' } reconnect_log_data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE) payload = { 'query': 'Message_ASTRO_ID:' + astro_id + ' AND ENV:PROD AND Message_ACTION:ASTRO_ALERT AND Message_STATUS:Disconnected', 'from': time_from, 'to': time_to, 'fields': 'DYFO_TIME' } disconnect_log_data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE) if disconnect_log_data['total_results'] > 0: if reconnect_log_data['total_results'] == 0 or dt.strptime( disconnect_log_data['messages'][0]['message'] ['DYFO_TIME'], DYFO_TIME_FORMAT_INPUT) > dt.strptime( reconnect_log_data['messages'][0]['message']['DYFO_TIME'], DYFO_TIME_FORMAT_INPUT): return disconnect_log_data['messages'][0]['message'][ 'DYFO_TIME'] elif (restart_time - dt.strptime( reconnect_log_data['messages'][0]['message']['DYFO_TIME'], DYFO_TIME_FORMAT_INPUT)).seconds < 60: return disconnect_log_data['messages'][0]['message'][ 'DYFO_TIME'] return None except Exception as e: logger.error("Exception in getting last wifi disconnect from Graylog") raise e
def get_severe_quick_interrupts(interrupt_bin, astros, house_id): """ :param interrupt_bin: :return: """ try: q_interrupt = [] for astro_id, val1 in interrupt_bin.iteritems(): for pin_id, val2 in val1.iteritems(): for epoch, count in val2.iteritems(): if count >= int(getConfigValue(QUICK_INTERRUPT_THRESHOLD)): mobile_pin_status, interrupt_pin_status = get_pin_statuses( datetime.datetime.fromtimestamp(int(epoch)), astros[astro_id][1], astro_id) q_interrupt.append({ 'astro_id': astro_id, 'house_id': house_id, 'time': datetime.datetime.fromtimestamp(int(epoch)), 'pin': pin_id, 'count': count, 'mobile_pin_state': mobile_pin_status.pop(str(pin_id), {}), 'other_mobile_pin_state': mobile_pin_status, 'interrupt_pin_state': interrupt_pin_status.pop(str(pin_id), {}), 'other_interrupt_pin_state': interrupt_pin_status }) return q_interrupt except Exception as e: logger.error( "Exception in extracting severe quick interrupts from interrupt bin. Exception " + str(e)) raise e
def checkQuickInterrupt(self, range): try: interrupt_bin = {} counter = 0 while (1): payload = {'query': 'HOUSE_ID:' + self.house_id + ' AND ENV:PROD', 'range': range, 'fields': 'DYFO_TIME,Message_ASTRO_ID,Message_PIN_D', 'filter': 'streams:' + QUICK_INTERRUPT_STREAM_ID, 'offset': 150 * counter} data = make_graylog_request(payload, GRAYLOG_SEARCH_RELATIVE) if data == None or 'messages' not in data or len(data['messages']) == 0: break for element in data['messages']: dyfo_time = element['message']['DYFO_TIME'] astro_id = element['message']['Message_ASTRO_ID'] pin_id = element['message']['Message_PIN_D'] if astro_id not in self.astros: continue epoch = int(time.mktime(time.strptime(dyfo_time, DYFO_TIME_FORMAT_INPUT))) epoch = epoch - (epoch % int(getConfigValue(INTERRUPT_BIN_VALUE))) if astro_id not in interrupt_bin: interrupt_bin[astro_id] = {} if pin_id not in interrupt_bin[astro_id]: interrupt_bin[astro_id][pin_id] = {} if epoch not in interrupt_bin[astro_id][pin_id]: interrupt_bin[astro_id][pin_id][epoch] = 0 interrupt_bin[astro_id][pin_id][epoch] += 1 counter += 1 # print(interrupt_bin) final_interrupts = get_severe_quick_interrupts(interrupt_bin, self.astros, self.house_id) return final_interrupts except Exception as e: logger.error("Exception in checking quick interrupts. Exception" + str(e)) raise e
def get_pin_status_interrupt(astro_id, pin_id, restart_time): try: count = 0 while count < MAX_STATUS_COUNT: count += 1 time_from = dt.strftime( restart_time - datetime.timedelta(hours=count * 20), TIME_FORMAT) time_to = dt.strftime(restart_time, TIME_FORMAT) payload = { 'query': 'Message_INTERRUPT_DETECTED_pi_id:' + astro_id + ' AND ENV:PROD AND (Message_INTERRUPT_DETECTED_pins_statuses:\"{status=1, pin_id=' + pin_id + '}\" OR Message_INTERRUPT_DETECTED_pins_statuses:\"{status=0, pin_id=' + pin_id + '}\")', 'from': time_from, 'to': time_to, 'fields': 'DYFO_TIME,Message_INTERRUPT_DETECTED_pins_statuses' } data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE) if data['total_results'] > 0: return { 'interrupt_call_time': data['messages'][0]['message']['DYFO_TIME'], 'pin_status': str(data['messages'][0]['message'] ['Message_INTERRUPT_DETECTED_pins_statuses']).split( ',')[0].split('=')[1] } return {'interrupt_call_time': 'Not Found', 'pin_status': 'Not Found'} except Exception as e: logger.error("Exception in getting Interrupt Pin statuses" + str(e)) raise e
def get_last_mqtt_disconnect(astro_id, restart_time): try: time_from = dt.strftime(restart_time - datetime.timedelta(hours=12), TIME_FORMAT) time_to = dt.strftime(restart_time, TIME_FORMAT) payload = { 'query': 'Message_ASTRO_ID:' + astro_id + ' AND ENV:PROD AND Message_ACTION:ASTRO_MQTT_RECONNECTED', 'from': time_from, 'to': time_to, 'fields': 'DYFO_TIME, Message_RECONNECTED_AT' } reconnect_log_data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE) payload = { 'query': 'Message_ASTRO_ID:' + astro_id + ' AND ENV:PROD AND Message_ACTION:ASTRO_MQTT_DISCONNECTED', 'from': time_from, 'to': time_to, 'fields': 'DYFO_TIME, Message_DISCONNECTED_AT' } disconnect_log_data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE) if disconnect_log_data['total_results'] > 0: if reconnect_log_data['total_results'] == 0 or dt.strptime( disconnect_log_data['messages'][0]['message'] ['Message_DISCONNECTED_AT'], DYFO_TIME_FORMAT_INPUT) > dt.strptime( reconnect_log_data['messages'][0]['message'] ['Message_RECONNECTED_AT'], DYFO_TIME_FORMAT_INPUT): return disconnect_log_data['messages'][0]['message'][ 'Message_DISCONNECTED_AT'] return None except Exception as e: logger.error("Exception in getting last mqtt disconnect from Graylog") raise e
def get_live_astros_from_drona(house_id): """ Get live astros from Drona :param house_id: :return: """ try: try: astro_state = House.objects.get(house_id=house_id)['astro_state'] except: astro_state = 'live' loaded_astros = rest_call.get( MASTER_SERVER_URL + ":" + str(MASTER_SERVER_PORT) + "/device/astro/all?house_did=" + house_id, timeout=20) loaded_astros = json.loads(loaded_astros.text) if astro_state == "all": astros = { item['display_id']: [item['inactive_since'], item['type']] for item in loaded_astros if item['type'] in ['single', 'double', 'triple'] } else: astros = { item['display_id']: [item['inactive_since'], item['type']] for item in loaded_astros if item['state'] == astro_state and item['type'] in ['single', 'double', 'triple'] } return astros except Exception as e: logger.error( "Exception while getting user defined astros type from drona. Exception " + str(e)) raise e
def handle_UpdateHouse(): """ Updates the status as well as Counts of all attributes(restarts, long cutoff, quick interrupt) for a House. :return: """ try: houses = get_house_statuses_from_drona() blacklisted = BLACKLISTED time = get_start_time() for house in houses: if house['display_id'] in blacklisted: continue try: h = House.objects.get(house_id=house['display_id']) except: h = House.objects.create(house_id=house['display_id']) if (not house['master_status']) and h.status == 'ACTIVE': h.status = 'ERRORED' + " " + datetime.datetime.now(pytz.timezone('Asia/Calcutta')).strftime( "%Y-%m-%d %H:%M:%S") if house['master_status']: h.status = 'ACTIVE' h.house_name = house['name'] h.restart_count = Restart.objects.filter(house_id=house['display_id'], restart_time__gt=time).count() # qis = QuickInterruptSerializer(QuickInterrupt.objects.filter(house_id=key), many=True, # context={'time_constrain': time}) # c = 0 # for q in qis.data: # c += len(q['interrupts']) h.qi_count = QuickInterrupt.objects.filter(house_id=house['display_id'], time__gt=time).count() h.ld_count = LongDisconnect.objects.filter(house_id=house['display_id'], status="INACTIVE").count() h.save() return houses except Exception as e: logger.error("Exception in updating house information from local db. Exception" + str(e)) raise e
def check_long_disconnect_alert(astro_id, disconnected_at): try: return "Long Disconnect Detected for Astro " + astro_id + " since " + disconnected_at + ".\n" except Exception as e: logger.error("Exception in checking long disconnect error" + str(e)) raise e
def handle(self, *args, **options): try: handle_UpdateAll() except Exception as e: logger.error("Exception in Updating data. Exception " + str(e))