def get_pin_status_mobile_call(astro_id, pin_id, restart_time):
try:
count = 0
while count < MAX_STATUS_COUNT:
count += 1
time_from = dt.strftime(
restart_time - datetime.timedelta(hours=count * 20),
TIME_FORMAT)
time_to = dt.strftime(restart_time, TIME_FORMAT)
payload = {
'query':
'Message_ACTION:CHANGE_DEVICE_STATUS AND ASTRO_ID:' +
astro_id + ' AND Message_PIN_ID:' + pin_id + ' AND ENV:PROD',
'from':
time_from,
'to':
time_to,
'fields':
'DYFO_TIME,Message_NEW_STATUS'
}
data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE)
if data['total_results'] > 0:
return {
'mobile_call_time':
data['messages'][0]['message']['DYFO_TIME'],
'pin_status':
data['messages'][0]['message']['Message_NEW_STATUS']
}
return {'mobile_call_time': 'Not Found', 'pin_status': 'Not Found'}
except Exception as e:
logger.error("Exeption in getting Mobile Pin statuses" + str(e))
def handle_UpdateRestarts(data):
"""
Saves Restarts to DB using Deserialization
:param data:
:return:
"""
try:
restart_alerts = {}
for data_item in data:
rs = RestartsSerializer(data=data_item)
if rs.is_valid():
try:
rs.save()
rsa = check_restart_alert(rs.data)
if rsa is not "":
restart_alerts[rs.data['astro_id']] = rsa
except Exception as e:
logger.info("Exception in saving" + str(e))
else:
logger.info("Errors in saving" + str(rs.errors))
return restart_alerts
except Exception as e:
logger.error("Exception in handling restarts. Exception" + str(e))
def handle_UpdateQInterrupts(interrupts, house_id):
"""
Saves Severe Quick Interrupts to Db
:param interrupts:
:param house_id:
:return:
"""
try:
qi_alerts = {}
for data_item in interrupts:
rs = QuickInterruptSerializer(data=data_item)
if rs.is_valid():
try:
rs.save()
qi_alerts[rs.data['astro_id']] = check_quick_interrupt_alert(rs.data)
except Exception as e:
logger.info("Exception in saving" + str(e))
else:
logger.info("Errors in saving" + str(rs.errors))
return qi_alerts
except Exception as e:
logger.error("Exception in handling quick interrupts. Exception" + str(e))
def handle_UpdateLongCutoff(data, house_id):
"""
Checks if previously Inactive astros are still Inactive or not.
If they have reconnected, mark them as active.
Makes DB entries for other Long Cutoffs.
:param obj:
:param house_id:
:return:
"""
try:
ld_objects = LongDisconnect.objects.filter(house_id=house_id)
ldo_alert = {}
for ld in ld_objects:
if ld.astro_id not in data:
ld.delete()
for key, val in data.iteritems():
try:
ldo = LongDisconnect.objects.get(astro_id=key)
except:
ldo = LongDisconnect.objects.create(house_id=house_id, astro_id=key)
ldo_alert[key] = check_long_disconnect_alert(key, val)
ldo.disconnected_at = val
ldo.status = "INACTIVE"
ldo.save()
# ldo_alert=check_long_disconnect_alert(key, val)
return ldo_alert
except Exception as e:
logger.error("Exception in updating long cut off. Exception " + str(e))
def authorization(email):
try:
a = Auth()
return a.authenticate(email, 'telescope')
except Exception as e:
logger.error("Exception in authorization. Exception" + str(e))
raise e
def get_last_healthcheck(astro_id, restart_time):
try:
count = 0
while count < MAX_HEALTHCHECK_COUNT:
count += 1
time_from = dt.strftime(
restart_time - datetime.timedelta(hours=count * 12),
TIME_FORMAT)
time_to = dt.strftime(restart_time, TIME_FORMAT)
payload = {
'query': 'Message_ASTRO_ID:' + astro_id + ' AND ENV:PROD',
'from': time_from,
'to': time_to,
'fields': 'DYFO_TIME',
'filter': 'streams:' + ASTRO_STATS_STREAM_ID
}
healthcheck_data = make_graylog_request(payload,
GRAYLOG_SEARCH_ABSOLUTE)
if healthcheck_data['total_results'] > 0:
return healthcheck_data['messages'][0]['message']['DYFO_TIME']
return None
except Exception as e:
logger.error(
"Exception in getting last helathcheck of astro from Graylog. Exception "
+ str(e))
raise e
def checkAstroRestarts(self, range):
try:
self.checkWifiDisconnections(range)
restarts = []
payload = {'query': 'HOUSE_ID:' + self.house_id + ' AND Message_ACTION:ASTRO_INITIATED AND ENV:PROD',
'range': range,
'fields': 'ASTRO_ID,DYFO_TIME,Message_ASTRO_SKETCH_VERSION, Message_RESTART_REASON'}
data = make_graylog_request(payload, GRAYLOG_SEARCH_RELATIVE)
if data == None or 'messages' not in data:
return restarts
for item in data['messages']:
try:
message = item['message']
astro_id = message['ASTRO_ID']
restart_time = message['DYFO_TIME']
# sketch_version = message['Message_ASTRO_SKETCH_VERSION']
if astro_id not in self.astros:
continue
restart_time_obj = dt.strptime(restart_time, DYFO_TIME_FORMAT_INPUT)
# device_type = get_device_type(sketch_version)
mobile_pin_status, interrupt_pin_status = get_pin_statuses(restart_time_obj,
self.astros[astro_id][1],
astro_id)
wifi_histogram = {}
wifi_disconnections = []
# , wifi_histogram = self.checkWifiDisconnections(range, astro_id)
code_version = House.objects.get(house_id=self.house_id).code_version
if "Message_RESTART_REASON" in message:
reason=RESTART_REASON.get(message["Message_RESTART_REASON"],"First time up")
comment="Derived from astro"
else:
analysis = HealthAnalysis(restart_time_obj, wifi_histogram, wifi_disconnections,
self.house_wifi_disconnections, code_version)
reason, comment = analysis.analyse(astro_id, restart_time_obj)
restart = {}
restart['house_id'] = self.house_id
restart['astro_id'] = astro_id
restart['restart_time'] = restart_time_obj
restart['wifi_disconnections'] = wifi_disconnections
restart['reason'] = reason
restart['comment'] = comment
restart['mobile_pin_status'] = mobile_pin_status
restart['interrupt_pin_status'] = interrupt_pin_status
restart['device_type'] = self.astros[astro_id][1]
restarts.append(restart)
except Exception as e:
logger.error("Exception in getting restart. Exception " + str(e))
return restarts
except Exception as e:
logger.error("Exception in checking restarts. Exception" + str(e))
raise e
def check_quick_interrupt_alert(qi):
try:
return "Quick Interrupt Detected on Astro " + str(
qi['astro_id']) + " on Pin " + str(
qi['pin']) + " with count" + str(qi['count']) + ".\n"
except Exception as e:
logger.error("Exception in checking quick interrupt alert" + str(e))
raise e
def handle_UpdateAll():
try:
houses = get_house_statuses_from_drona()
blacklisted = BLACKLISTED
for house in houses:
if house['display_id'] in blacklisted:
continue
try:
h = House.objects.get(house_id=house['display_id'])
except:
h = House.objects.create(house_id=house['display_id'])
if (not house['master_status']) and h.status == 'ACTIVE':
h.status = 'ERRORED' + " " + datetime.datetime.now(pytz.timezone('Asia/Calcutta')).strftime(
"%Y-%m-%d %H:%M:%S")
if house['master_status']:
h.status = 'ACTIVE'
h.code_version = house['code_version']
h.save()
houses = House.objects.filter(to_update=True)
alerts = {}
for house in houses:
dyfo_house = DyfoHouseHealth(house.house_id)
data = dyfo_house.checkAstroRestarts(DAY_SECONDS)
alert = handle_UpdateRestarts(data)
if alert:
if house.house_id not in alerts:
alerts[house.house_id] = {}
alerts[house.house_id]['name'] = house.house_name
alerts[house.house_id]['Restarts'] = alert
data = dyfo_house.checkQuickInterrupt(DAY_SECONDS)
alert = handle_UpdateQInterrupts(data, house.house_id)
if alert:
if house.house_id not in alerts:
alerts[house.house_id] = {}
alerts[house.house_id]['name'] = house.house_name
alerts[house.house_id]['Quick_Interrupts'] = alert
data = dyfo_house.checkLongDisconnect()
alert = handle_UpdateLongCutoff(data, house.house_id)
if alert:
if house.house_id not in alerts:
alerts[house.house_id] = {}
alerts[house.house_id]['name'] = house.house_name
alerts[house.house_id]['Long_Disconnect'] = alert
if alerts:
send_alert(alerts)
update_last_update()
except Exception as e:
logger.error("Exception in updating Houses. Exception " + str(e))
raise e
def get_astro_states_from_drona():
try:
res = rest_call.get(MASTER_SERVER_URL + ":" + str(MASTER_SERVER_PORT) +
"/device/astro_state_choices",
timeout=20)
return json.loads(res.text)["choices"]
except Exception as e:
logger.error(
"Exception in getting astro states choice list from Drona. Exception "
+ str(e))
raise e
def getConfigValue(key):
"""
Get Configuration value from Db
:param key:
:return:
"""
try:
c = Configs.objects.get(key=key)
return c.value
except Exception as e:
logger.error("Exception in getting config value of " + key +
"from Database. Exception " + str(e))
def get_start_time():
"""
Returns time based on Dispaly interval defined in Db
:return:
"""
try:
return datetime.datetime.now() + datetime.timedelta(
hours=5) - datetime.timedelta(
hours=int(getConfigValue(DATA_DISPLAY_RANGE)))
except Exception as e:
logger.error(
"Exception in getting start time for event according to user defined display range"
+ str(e))
def checkLongDisconnect(self):
# print("Long Disconnected Live Astros")
try:
long_disconnects = {}
for key, val in self.astros.iteritems():
if val[0] is not None and (dt.now() + datetime.timedelta(hours=5, minutes=30) - dt.strptime(val[0],
DRONA_TIME_FORMAT)) > datetime.timedelta(
hours=4):
long_disconnects[key] = val[0]
return long_disconnects
except Exception as e:
logger.error("Exception in checking long disconnect. Exception" + str(e))
raise e
def check_restart_alert(rs):
try:
time = get_start_time()
rscount = Restart.objects.filter(astro_id=rs['astro_id'],
restart_time__gt=time).count()
if rscount >= RESTART_ALERT_THRESHOLD:
return str(rs["astro_id"]) + " restarted " + str(
rscount) + " times in the past " + getConfigValue(
DATA_DISPLAY_RANGE
) + " hours from the time of this alert.\n"
return ""
except Exception as e:
logger.error("Exception in checking restart alert" + str(e))
raise e
def update_last_update():
"""
Updates the Last Updates attribute in Db
:return:
"""
try:
c = Configs.objects.get(key=LAST_UPDATED)
c.value = datetime.datetime.now(
pytz.timezone('Asia/Calcutta')).strftime("%Y-%m-%d %H:%M:%S")
c.save()
except Exception as e:
logger.error("Exception in updating last updated time. Exception " +
str(e))
raise e
def get_house_statuses_from_drona():
"""
Status of master from Drona
:return:
"""
try:
houses = rest_call.get(MASTER_SERVER_URL + ":" +
str(MASTER_SERVER_PORT) + "/house/all",
timeout=20)
houses = json.loads(houses.text)
return houses
except Exception as e:
logger.error(
"Exception in getting House Status from Drona. Exception " +
str(e))
def make_graylog_request(payload, uri):
"""
:param payload:
:param uri:
:return:
"""
try:
url = GRAYLOG_SERVER_URL + uri + "?{}".format(
urllib.urlencode(payload))
response = graylog_rest_call.get(
url, retry=True, headers={'Accept': 'application/json'}, timeout=5)
return json.loads(response.text, object_pairs_hook=OrderedDict)
except Exception as e:
logger.error("Exception in making graylog request. Exception " +
str(e))
raise e
def send_alert(alerts):
try:
alert_string = ""
for house, val in alerts.iteritems():
alert_string += "<b>" + house + "</b>" + "<br>"
name = val.pop('name')
alert_string += "<b>" + name + "</b>" + "<br>"
for type, values in val.iteritems():
alert_string += "<b>" + type + "</b>" + "<br>"
for astro, values2 in values.iteritems():
alert_string += values2 + "<br>"
alert_string += "Visit " + TELESCOPE_URL + " for detailed information."
user = yamjam()['keeper_user']
m = Mailer(user)
m.send_mail("Lense Alert", alert_string, ['*****@*****.**'])
except Exception as e:
logger.error("Exception in sending alert" + str(e))
def get_last_wifi_disconnect(astro_id, restart_time):
try:
time_from = dt.strftime(restart_time - datetime.timedelta(hours=12),
TIME_FORMAT)
time_to = dt.strftime(restart_time, TIME_FORMAT)
payload = {
'query': 'Message_ASTRO_ID:' + astro_id +
' AND ENV:PROD AND Message_ACTION:ASTRO_ALERT AND Message_STATUS:Connected',
'from': time_from,
'to': time_to,
'fields': 'DYFO_TIME'
}
reconnect_log_data = make_graylog_request(payload,
GRAYLOG_SEARCH_ABSOLUTE)
payload = {
'query': 'Message_ASTRO_ID:' + astro_id +
' AND ENV:PROD AND Message_ACTION:ASTRO_ALERT AND Message_STATUS:Disconnected',
'from': time_from,
'to': time_to,
'fields': 'DYFO_TIME'
}
disconnect_log_data = make_graylog_request(payload,
GRAYLOG_SEARCH_ABSOLUTE)
if disconnect_log_data['total_results'] > 0:
if reconnect_log_data['total_results'] == 0 or dt.strptime(
disconnect_log_data['messages'][0]['message']
['DYFO_TIME'], DYFO_TIME_FORMAT_INPUT) > dt.strptime(
reconnect_log_data['messages'][0]['message']['DYFO_TIME'],
DYFO_TIME_FORMAT_INPUT):
return disconnect_log_data['messages'][0]['message'][
'DYFO_TIME']
elif (restart_time - dt.strptime(
reconnect_log_data['messages'][0]['message']['DYFO_TIME'],
DYFO_TIME_FORMAT_INPUT)).seconds < 60:
return disconnect_log_data['messages'][0]['message'][
'DYFO_TIME']
return None
except Exception as e:
logger.error("Exception in getting last wifi disconnect from Graylog")
raise e
def get_severe_quick_interrupts(interrupt_bin, astros, house_id):
"""
:param interrupt_bin:
:return:
"""
try:
q_interrupt = []
for astro_id, val1 in interrupt_bin.iteritems():
for pin_id, val2 in val1.iteritems():
for epoch, count in val2.iteritems():
if count >= int(getConfigValue(QUICK_INTERRUPT_THRESHOLD)):
mobile_pin_status, interrupt_pin_status = get_pin_statuses(
datetime.datetime.fromtimestamp(int(epoch)),
astros[astro_id][1], astro_id)
q_interrupt.append({
'astro_id':
astro_id,
'house_id':
house_id,
'time':
datetime.datetime.fromtimestamp(int(epoch)),
'pin':
pin_id,
'count':
count,
'mobile_pin_state':
mobile_pin_status.pop(str(pin_id), {}),
'other_mobile_pin_state':
mobile_pin_status,
'interrupt_pin_state':
interrupt_pin_status.pop(str(pin_id), {}),
'other_interrupt_pin_state':
interrupt_pin_status
})
return q_interrupt
except Exception as e:
logger.error(
"Exception in extracting severe quick interrupts from interrupt bin. Exception "
+ str(e))
raise e
def checkQuickInterrupt(self, range):
try:
interrupt_bin = {}
counter = 0
while (1):
payload = {'query': 'HOUSE_ID:' + self.house_id + ' AND ENV:PROD', 'range': range,
'fields': 'DYFO_TIME,Message_ASTRO_ID,Message_PIN_D',
'filter': 'streams:' + QUICK_INTERRUPT_STREAM_ID,
'offset': 150 * counter}
data = make_graylog_request(payload, GRAYLOG_SEARCH_RELATIVE)
if data == None or 'messages' not in data or len(data['messages']) == 0:
break
for element in data['messages']:
dyfo_time = element['message']['DYFO_TIME']
astro_id = element['message']['Message_ASTRO_ID']
pin_id = element['message']['Message_PIN_D']
if astro_id not in self.astros:
continue
epoch = int(time.mktime(time.strptime(dyfo_time, DYFO_TIME_FORMAT_INPUT)))
epoch = epoch - (epoch % int(getConfigValue(INTERRUPT_BIN_VALUE)))
if astro_id not in interrupt_bin:
interrupt_bin[astro_id] = {}
if pin_id not in interrupt_bin[astro_id]:
interrupt_bin[astro_id][pin_id] = {}
if epoch not in interrupt_bin[astro_id][pin_id]:
interrupt_bin[astro_id][pin_id][epoch] = 0
interrupt_bin[astro_id][pin_id][epoch] += 1
counter += 1
# print(interrupt_bin)
final_interrupts = get_severe_quick_interrupts(interrupt_bin, self.astros, self.house_id)
return final_interrupts
except Exception as e:
logger.error("Exception in checking quick interrupts. Exception" + str(e))
raise e
def get_pin_status_interrupt(astro_id, pin_id, restart_time):
try:
count = 0
while count < MAX_STATUS_COUNT:
count += 1
time_from = dt.strftime(
restart_time - datetime.timedelta(hours=count * 20),
TIME_FORMAT)
time_to = dt.strftime(restart_time, TIME_FORMAT)
payload = {
'query':
'Message_INTERRUPT_DETECTED_pi_id:' + astro_id +
' AND ENV:PROD AND (Message_INTERRUPT_DETECTED_pins_statuses:\"{status=1, pin_id='
+ pin_id +
'}\" OR Message_INTERRUPT_DETECTED_pins_statuses:\"{status=0, pin_id='
+ pin_id + '}\")',
'from':
time_from,
'to':
time_to,
'fields':
'DYFO_TIME,Message_INTERRUPT_DETECTED_pins_statuses'
}
data = make_graylog_request(payload, GRAYLOG_SEARCH_ABSOLUTE)
if data['total_results'] > 0:
return {
'interrupt_call_time':
data['messages'][0]['message']['DYFO_TIME'],
'pin_status':
str(data['messages'][0]['message']
['Message_INTERRUPT_DETECTED_pins_statuses']).split(
',')[0].split('=')[1]
}
return {'interrupt_call_time': 'Not Found', 'pin_status': 'Not Found'}
except Exception as e:
logger.error("Exception in getting Interrupt Pin statuses" + str(e))
raise e
def get_last_mqtt_disconnect(astro_id, restart_time):
try:
time_from = dt.strftime(restart_time - datetime.timedelta(hours=12),
TIME_FORMAT)
time_to = dt.strftime(restart_time, TIME_FORMAT)
payload = {
'query': 'Message_ASTRO_ID:' + astro_id +
' AND ENV:PROD AND Message_ACTION:ASTRO_MQTT_RECONNECTED',
'from': time_from,
'to': time_to,
'fields': 'DYFO_TIME, Message_RECONNECTED_AT'
}
reconnect_log_data = make_graylog_request(payload,
GRAYLOG_SEARCH_ABSOLUTE)
payload = {
'query': 'Message_ASTRO_ID:' + astro_id +
' AND ENV:PROD AND Message_ACTION:ASTRO_MQTT_DISCONNECTED',
'from': time_from,
'to': time_to,
'fields': 'DYFO_TIME, Message_DISCONNECTED_AT'
}
disconnect_log_data = make_graylog_request(payload,
GRAYLOG_SEARCH_ABSOLUTE)
if disconnect_log_data['total_results'] > 0:
if reconnect_log_data['total_results'] == 0 or dt.strptime(
disconnect_log_data['messages'][0]['message']
['Message_DISCONNECTED_AT'],
DYFO_TIME_FORMAT_INPUT) > dt.strptime(
reconnect_log_data['messages'][0]['message']
['Message_RECONNECTED_AT'], DYFO_TIME_FORMAT_INPUT):
return disconnect_log_data['messages'][0]['message'][
'Message_DISCONNECTED_AT']
return None
except Exception as e:
logger.error("Exception in getting last mqtt disconnect from Graylog")
raise e
def get_live_astros_from_drona(house_id):
"""
Get live astros from Drona
:param house_id:
:return:
"""
try:
try:
astro_state = House.objects.get(house_id=house_id)['astro_state']
except:
astro_state = 'live'
loaded_astros = rest_call.get(
MASTER_SERVER_URL + ":" + str(MASTER_SERVER_PORT) +
"/device/astro/all?house_did=" + house_id,
timeout=20)
loaded_astros = json.loads(loaded_astros.text)
if astro_state == "all":
astros = {
item['display_id']: [item['inactive_since'], item['type']]
for item in loaded_astros
if item['type'] in ['single', 'double', 'triple']
}
else:
astros = {
item['display_id']: [item['inactive_since'], item['type']]
for item in loaded_astros if item['state'] == astro_state
and item['type'] in ['single', 'double', 'triple']
}
return astros
except Exception as e:
logger.error(
"Exception while getting user defined astros type from drona. Exception "
+ str(e))
raise e
def handle_UpdateHouse():
"""
Updates the status as well as Counts of all attributes(restarts, long cutoff, quick interrupt) for a House.
:return:
"""
try:
houses = get_house_statuses_from_drona()
blacklisted = BLACKLISTED
time = get_start_time()
for house in houses:
if house['display_id'] in blacklisted:
continue
try:
h = House.objects.get(house_id=house['display_id'])
except:
h = House.objects.create(house_id=house['display_id'])
if (not house['master_status']) and h.status == 'ACTIVE':
h.status = 'ERRORED' + " " + datetime.datetime.now(pytz.timezone('Asia/Calcutta')).strftime(
"%Y-%m-%d %H:%M:%S")
if house['master_status']:
h.status = 'ACTIVE'
h.house_name = house['name']
h.restart_count = Restart.objects.filter(house_id=house['display_id'], restart_time__gt=time).count()
# qis = QuickInterruptSerializer(QuickInterrupt.objects.filter(house_id=key), many=True,
# context={'time_constrain': time})
# c = 0
# for q in qis.data:
# c += len(q['interrupts'])
h.qi_count = QuickInterrupt.objects.filter(house_id=house['display_id'], time__gt=time).count()
h.ld_count = LongDisconnect.objects.filter(house_id=house['display_id'], status="INACTIVE").count()
h.save()
return houses
except Exception as e:
logger.error("Exception in updating house information from local db. Exception" + str(e))
raise e
def check_long_disconnect_alert(astro_id, disconnected_at):
try:
return "Long Disconnect Detected for Astro " + astro_id + " since " + disconnected_at + ".\n"
except Exception as e:
logger.error("Exception in checking long disconnect error" + str(e))
raise e
def handle(self, *args, **options):
try:
handle_UpdateAll()
except Exception as e:
logger.error("Exception in Updating data. Exception " + str(e))
