def check_disk_space_scratch(self): if song_monitor_config.mon_disk_space == 1: # output = os.popen("df").readlines()[-4] for line in os.popen("df").readlines(): if "scratch" in line.split()[-1]: scratch_device, scratch_size, scratch_used, scratch_available, scratch_percent, scratch_mountpoint = line.split( ) scratch_free_ds_percent = (float(scratch_available) / float(scratch_size)) * 100. scratch_used_ds_percent = (float(scratch_used) / float(scratch_size)) * 100. if float( scratch_available ) < song_monitor_config.scratch_ds_limit and self.email_ds_scratch_alarm == 0: print clock.timename( ), " There is not much space left on scratch: %f GB\n" % ( float(scratch_available) / 1000000.) value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="scratch out of disk space!", message= "scratch has %sGB disk space left. Please clean now!\n\nSend at: %s\n\n" % ((float(scratch_available) / 1000000.), clock.obstimeUT())) self.email_ds_scratch_alarm = 1 elif float( scratch_available ) < song_monitor_config.scratch_ds_soft_limit and self.email_ds_scratch == 0: print clock.timename( ), " There is not much space left on scratch: %f GB\n" % ( float(scratch_available) / 1000000.) value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="scratch out of disk space!", message= "scratch has %sGB disk space left. Please clean soon!\n\nSend at: %s\n\n" % ((float(scratch_available) / 1000000.), clock.obstimeUT())) self.email_ds_scratch = 1 elif float( scratch_available ) < song_monitor_config.scratch_ds_soft_limit and self.email_ds_scratch == 1: print clock.timename( ), " There is not much space left on scratch: %f GB" % ( float(scratch_available) / 1000000.) elif float(scratch_available ) > song_monitor_config.scratch_ds_soft_limit: print clock.timename( ), " scratch has sufficient disk space left: %f GB" % ( float(scratch_available) / 1000000.) self.email_ds_scratch = 0 return 1 else: return 1
def check_machines(self): for m in song_monitor_config.machines: output = self.timeout_command(["ssh", m, "date"], 15) print clock.timename(), "Output from %s was: %s" % ( m, str(output).strip()) if output == None: print clock.timename(), "The check on %s had a timeout..." % ( m) print clock.timename(), "Now sending an e-mail..." send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="%s is not responding" % (m), message= "%s is not responding on the ssh connection from hw.sstenerife.prv. It might be hanging...\n\nPlease check!!" % (m)) if song_monitor_config.send_sms == "yes": value = send_song_mail.send_mail().send_sms( receiver=m_conf.wakeup_sms_who, message="%s is not responding. Please check!" % (m)) return 1
def check_iodine_heaters(self): if song_monitor_config.mon_iodine_heaters == 1: #### Check test iodine cell heater try: output = self.timeout_command([ "ssh", "10.8.0.111", "python /home/obs/mads/i2_usb.py tact?" ], 10) except Exception, e: print clock.timename( ), " Could not get test iodine heater data" if self.notify_test_i2_heater_delay < time.time(): value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Could not check iodine test cell heater!", message= "The monitor caught an exception when trying to check the test iodine cell heater.\n\n Please check the machine: IP: 10.8.0.111 as obs to see if it is running. Use the script > python /home/obs/mads/i2_usb.py' to check and enable the heater!\n\nSend at: %s\n\n" % (clock.obstimeUT())) self.notify_test_i2_heater_delay = time.time() + ( 24. * 60. * 60.) # Add a delay of 1 day else: if output == None: print clock.timename( ), " Did not get data... now sending an e-mail..." if self.notify_test_i2_heater_delay < time.time(): value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Iodine test cell not heated!", message= "The monitor did not get a reply from the small hp machine to which the iodine test cells heater is connected.\n\n Please check the machine: IP: 10.8.0.111 as obs to see if it is running. Use the script > python /home/obs/mads/i2_usb.py' to check and enable the heater!\n\nSend at: %s\n\n" % (clock.obstimeUT())) self.notify_test_i2_heater_delay = time.time() + ( 24. * 60. * 60.) # Add a delay of 1 day else: #print float(output[0].strip(">\r\n C")) try: temp = float(output.split()[1]) except Exception, e: print clock.timename( ), " Did not get meaningful data from test iodine heater... now sending an e-mail..." if self.notify_test_i2_heater_delay < time.time(): value = send_song_mail.send_mail( ).sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Iodine test cell not heated!", message= "The monitor did not get a meaningful reply from the heater.\n\n Please log onto the machine: IP: 10.8.0.111 as obs and use the script > python /home/obs/mads/i2_usb.py' to check and enable the heater!\n\nSend at: %s\n\n" % (clock.obstimeUT())) self.notify_test_i2_heater_delay = time.time() + ( 24. * 60. * 60.) # Add a delay of 1 day else:
def check_disk_space_hw(self): if song_monitor_config.mon_disk_space == 1: df = subprocess.Popen(["df", song_monitor_config.outstream], stdout=subprocess.PIPE) output = df.communicate()[0] hw_device, hw_size, hw_used, hw_available, hw_percent, hw_mountpoint = output.split( "\n")[1].split() hw_free_ds_percent = (float(hw_available) / float(hw_size)) * 100. hw_used_ds_percent = (float(hw_used) / float(hw_size)) * 100. if float( hw_available ) < song_monitor_config.hw_ds_limit and self.email_ds_alarm == 0: print clock.timename( ), " There is not much space left on hw: %f GB" % ( float(hw_available) / 1000000.) print clock.timename( ), " Free disk space in percent: %f\n" % hw_free_ds_percent value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="hw out of disk space!", message= "hw has less than 15 percent disk space left. Please check soon!\n\nSend at: %s\n\n" % clock.obstimeUT()) self.email_ds_alarm = 1 elif hw_free_ds_percent < song_monitor_config.hw_ds_soft_limit_percent and self.email_ds == 0: print clock.timename( ), " There is not much space left on hw: %f GB" % ( float(hw_available) / 1000000.) print clock.timename( ), " There is less than 15 percent disk space left on hw!" print clock.timename( ), " Free disk space in percent: %f\n" % hw_free_ds_percent value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="hw out of disk space!", message= "hw has less than 15 percent disk space left. Please check soon!\n\nSend at: %s\n\n" % clock.obstimeUT()) self.email_ds = 1 elif hw_free_ds_percent > song_monitor_config.hw_ds_soft_limit_percent: print clock.timename( ), " hw has sufficient disk space left: %f GB" % ( float(hw_available) / 1000000.) self.email_ds = 0 return 1 else: return 1
def check_if_monitor_runs(self): delay_time = 0 while RUNNING: if past_time + 600 < time.time() and self.monitor_hangs_send_mail == 0: print clock.timename(), " It has been 10 minutes since the monitor went through a loop!" send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS",subject="Monitor hangs!",message="The monitor daemon is hanging somewhere and a restart is most likely needed. Please check if things looks fine!\n\nThis is what will be send out if the connection to the telescope is lost from the Astelco side\n\nSend at: %s" % clock.obstimeUT()) send_song_mail.send_mail().send_sms(receiver=m_conf.wakeup_sms_who, message="The monitor daemon hangs for some reason!") if song_monitor_config.send_to_support == "yes": send_song_mail.send_mail().sending_an_email(reciever=['support'],sender="SONG_MS",subject="Monitor hangs!",message="The monitor daemon is hanging somewhere and a restart is most likely needed. Please check if things looks fine!\n\nThis is what will be send out if the connection to the telescope is lost from the Astelco side\n\nSend at: %s" % clock.obstimeUT()) self.monitor_hangs_send_mail = 1 delay_time = time.time() + 3600 # Wait an hour to send another SMS and emails. elif past_time + 600 > time.time() and self.monitor_hangs_send_mail == 0: loop_time = time.time() - past_time print clock.timename(), " Monitor runs as it should... loop past %s seconds ago" % loop_time else: if delay_time < time.time() and self.monitor_hangs_send_mail == 1: self.monitor_hangs_send_mail = 0 time.sleep(30)
def run(self): """ """ done_log_param = 0 global RUNNING RUNNING = True #self.selector = selector.SELECTOR() #self.insertor = insertor self.sun_handle = song_star_checker.sun_pos( site=m_conf.song_site) # Tenerife sun handle self.daily_update = 0 self.daily_sun_insert = 0 val = beating_heart.start_heartbeat(job_id=m_conf.conductor_id) over_all_obs_state = get_database_values(table_name='tel_dome', fields=['extra_param_1']) gc.disable() process = psutil.Process(os.getpid()) print("Memory used in percent : %0.3f" % float(process.memory_percent())) mem = process.memory_info() mem_old = (float(mem.vms) / float(2**20)) print "" print clock.timename(), "The Conductor was started!\n" print clock.timename( ), "The over all observing state was: ", over_all_obs_state[ 'extra_param_1'] while RUNNING: if int( float( datetime.datetime.strftime( datetime.datetime.utcnow(), "%H"))) < 9 and self.sun_handle.sun_alt( unit='f') < 0.0: yesterday = datetime.datetime.utcnow() - datetime.timedelta( days=1) folder_date = yesterday.strftime('%Y-%m-%d') else: folder_date = datetime.datetime.strftime( datetime.datetime.utcnow(), "%Y-%m-%d") # print "Checking the date: %s" % (str(folder_date)) ###################### DAY TIME CHECK ################################ if int( float( datetime.datetime.strftime( datetime.datetime.utcnow(), "%H"))) < 10 and self.sun_handle.sun_alt( unit='f') > 1.0: try: daytime_obs = check_daytime.Check_Daytime(folder_date) except Exception, e: print clock.timename(), e else: if daytime_obs == 0 and self.daily_sun_insert == 0: print clock.timename( ), "The Solar observations were not inserted yet. Will do now" try: #insertor.insert_solar_observations() insertor.insert_solar_observations_soda() except Exception, e: print clock.timename( ), "Did not call the check-for-gaps function" print clock.timename(), e else: self.daily_sun_insert = 1 if conf.check_for_timecritical == "yes" and self.daily_update == 0: st = datetime.datetime.utcnow() se = datetime.datetime.utcnow() + datetime.timedelta( days=1) try: selected_target = selector.SELECTOR().determine_next( timegap=[st, se], project_critical_type=["timecritical"]) except Exception, e: print clock.timename( ), "Problem in the selector.determine_next function..." print clock.timename(), e else: if selected_target != None: # Checking if they are already inserted: ids = [] names = [] for tmp_or in selected_target: ids.append(tmp_or[1]) names.append(tmp_or[2]) check_val = check_daytime.check_timecritical( folder_date, ids) if check_val == 0: for tcrit_target in selected_target: insertor.insert_timecritical_OR( pre_obs_spec_id=tcrit_target[1]) try: send_song_mail.send_mail( ).sending_an_email( reciever=["mads", "frank"], sender="SONG_MS", subject="Time critical OR inserted", message= "The conductor has inserted time critical OR(s) for the coming night.\nObject(s): %s\n" % str(names)) except Exception, e: print "Could not send e-mail on time critical insertion..."
done_log_param = 1 if done_log_param == 1 and int( float(time.strftime("%H", time.gmtime()))) > 12: done_log_param = 0 self.daily_update = 0 self.daily_sun_insert = 0 process = psutil.Process(os.getpid()) print("Memory used in percent : %0.3f" % float(process.memory_percent())) if float(process.memory_percent()) > 60: send_song_mail.send_mail().sending_an_email( reciever=["mads"], sender="SONG_MS", subject="Conductor using all memory!", message= "The conductor has used more than 60 percent of the total memory on the song VM!\n\nPlease restart to clear memory ASAP!" ) os.system( "python /home/madsfa/subversion/central_trunk/conductor/conductor.py -t ; python /home/madsfa/subversion/central_trunk/conductor/conductor.py -s" ) mem = process.memory_info() print "Memory used in MB: %.1f" % ((float(mem.vms) / float(2**20))) print "Memory increase in MB: %.2f" % ( (float(mem.vms) / float(2**20)) - mem_old) mem_old = (float(mem.vms) / float(2**20)) gc.collect()
class Check_Time(object): """ @brief: This class handles all checks on the time of day. """ def __init__(self): """ Initialization of the time checks. """ self.mon_time_value = song_monitor_config.mon_time # 1 = do things, 0 = do nothing. self.sun_handle = song_star_checker.sun_pos( site=m_conf.song_site) # site=1: Tenerife self.verbose = song_monitor_config.verbose self.perform_actions = mon_actions.Do_Actions() self.skycam_start = m_conf.start_skycam_movie self.local_mode = 0 self.sms_send = 0 self.check_ors = 0 self.open_mirror_covers = 0 self.bad_weather_sms = 0 self.scheduler_val = 0 self.local_mode_sms = 0 self.x_mas_notify = 0 def check_sun(self, side_port_group_1, side_port_group_2, side_port_group_3, side_port_group_4, telescope_state, slit_state, flap_state, mirror_cover_state, who_did_it_tel, who_did_it_sp): """ Checks the Suns position and returns a value corresponding to this. """ self.side_port_group_1 = side_port_group_1 self.side_port_group_2 = side_port_group_2 self.side_port_group_3 = side_port_group_3 self.side_port_group_4 = side_port_group_4 self.telescope_state = telescope_state self.slit_state = slit_state self.flap_state = flap_state self.mirror_cover_state = mirror_cover_state self.who_did_it_tel = who_did_it_tel self.who_did_it_sp = who_did_it_sp sun_alt = self.sun_handle.sun_alt() if str(sun_alt)[0] == "-": sun_alt_d = float(str(sun_alt).split(":")[0]) - float( str(sun_alt).split(":")[1]) / 60.0 - float( str(sun_alt).split(":")[2]) / 3600.0 elif str(sun_alt)[0] != "-": sun_alt_d = float(str(sun_alt).split(":")[0]) + float( str(sun_alt).split(":")[1]) / 60.0 + float( str(sun_alt).split(":")[2]) / 3600.0 ################################# #### Get time to next sun set ### tmp_time_str2 = datetime.datetime.strptime( str(self.sun_handle.sun_set_next()), "%Y/%m/%d %H:%M:%S") time_diff = tmp_time_str2 - datetime.datetime.utcnow() hours_to_next_sun_set = int( time_diff.days) * 24. + time_diff.seconds / (24. * 3600.) * 24 #### Get time from previous sun set ### tmp_time_str2 = datetime.datetime.strptime( str(self.sun_handle.sun_set_pre()), "%Y/%m/%d %H:%M:%S") time_diff2 = datetime.datetime.utcnow() - tmp_time_str2 hours_from_pre_sun_set = int( time_diff2.days) * 24. + time_diff2.seconds / (24. * 3600.) * 24 ################################# ### GET DUST VALUE: ############################## try: conn = psycopg2.connect( "host='%s' dbname='%s' user='******' password='******'" % (m_conf.db_host, m_conf.data_db, m_conf.db_user, m_conf.db_password)) curr = conn.cursor() stmt = 'SELECT dust_level FROM weather_station WHERE ins_at = (SELECT max(ins_at) FROM weather_station)' curr.execute(stmt) results = curr.fetchone() dust_value = results[0] except Exception, e: print clock.timename(), " Could not get dust value from database!" dust_value = 0.011 if song_monitor_config.mon_telescope_actions == 1: #### Get telescope poitning direktion #### tel_value = get_db_values.db_connection().get_fields_site01( "tel_dome", ["tel_az"]) tel_az = tel_value["tel_az"] # if float(self.telescope_state) == float(-3.0) and self.local_mode == 0: # print clock.timename(), " The telescope was put in local mode" # if song_monitor_config.send_notifications == "yes": # send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS",subject="Telescope in local mode!", message="The telescope was put in local mode!\n\nSend at: %s\n\n" % clock.obstimeUT()) # if song_monitor_config.send_notify_sms == "yes": # send_song_mail.send_mail().send_sms(receiver=["Mads"], message="Someone has put the telescope into local mode") # self.local_mode = 1 # elif float(self.telescope_state) == float(0.0) and self.local_mode == 1: # print clock.timename(), " The telescope was put back in remote mode" # if song_monitor_config.send_notifications == "yes": # send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS", subject="Telescope back in remote mode!", message="The telescope was put back in remote mode!\n\nSend at: %s\n\n" % clock.obstimeUT()) # self.local_mode = 0 if float(sun_alt_d) >= ( float(m_conf.obs_sun_alt) + 2.0 ) and hours_to_next_sun_set >= m_conf.open_time_tel and hours_from_pre_sun_set >= m_conf.open_time_tel: ####################################### #### Close dome and park telescope #### ####################################### if self.skycam_start == 0: print clock.timename(), " Resetting skycam starting parameter" self.skycam_start = 1 try: self.sc2_handle.kill() except Exception, e: print e print clock.timename( ), " Could not kill the skycam 2 process" if song_monitor_config.mon_telescope_actions == 1 and float( self.telescope_state) != float(0.0) and float( self.telescope_state) != float(-3.0): # if float(self.slit_state) != 0.0 or float(self.flap_state) != 0.0 or float(self.mirror_cover_state) != 0.0: if float(self.slit_state) != 0.0 or float( self.flap_state) != 0.0 or ( float(self.mirror_cover_state) != 0.0 and song_monitor_config.allow_mc_open != "yes"): status_tel = self.perform_actions.shutdown_telescope() if status_tel == "done": self.who_did_it_tel = "time" print clock.timename( ), " The dome was closed due to daylight" dome_check = self.perform_actions.check_shutdown() if dome_check == 1: print clock.timename( ), " The dome was closed correctly" elif dome_check == 2: print clock.timename( ), " The dome was closed but mirror cover open" if song_monitor_config.send_notifications == "yes": send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Mirror covers were open!", message= "The dome was closed but mirror covers open!\n\nSend at: %s\n\n" % clock.obstimeUT()) else: if song_monitor_config.send_to_wake_up == "yes": send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Telescope Closing Error!", message= "The dome was not closed correctly after shutdown!\n\nSend at: %s\n\n" % clock.obstimeUT()) if song_monitor_config.send_sms == "yes" and self.sms_send == 0: send_song_mail.send_mail().send_sms( receiver=m_conf.wakeup_sms_who, message="The dome was not closed corretly...") if song_monitor_config.send_to_support == "yes" and self.sms_send == 0: send_song_mail.send_mail().sending_an_email( reciever=['support'], sender="SONG_MS", subject="Telescope Closing Error!", message= "The dome was not closed correctly after shutdown!\n\nSend at: %s\n\n" % clock.obstimeUT()) self.sms_send = 1 ###### Stop sigu and pugu ######## print clock.timename( ), " Stopping the slit and pupil guiders" try: status_sigu_pugu = self.perform_actions.stop_sigu_and_pugu( ) except Exception, e: print clock.timename( ), " Could not connect to Sigu and/or Pugu. They might be stopped already!" if status_sigu_pugu != 1: print clock.timename( ), " Something went wrong when trying to stop sigu and pugu" else: if self.who_did_it_tel == "free": self.who_did_it_tel = "time" self.time_delay_telescope = time.time() + ( song_checker_config.telescope_delay_time * 60) ### Maybe the sms send parameter should be placed elsewhere... self.sms_send = 0
status_tel = self.perform_actions.startup_telescope() if status_tel == "done": print clock.timename( ), " The telescope is now ready for observing" #if song_monitor_config.send_notifications == "yes": #send_song_mail.send_mail().sending_an_email(reciever=["mads"],sender="SONG_MS",subject="Observations startup!",message="The Sun has set and observation can be started!") if hours_to_next_sun_set < m_conf.open_time_tel and hours_to_next_sun_set > m_conf.open_time_tel - 1: try: if song_monitor_config.send_notify_sms == "yes" and self.who_did_it_tel == "time": print clock.timename( ), " A SMS will be sent about telescope startup..." send_song_mail.send_mail().send_sms( receiver=m_conf.notify_sms_who, message= "Weather is great at Tenerife and the telescope has opend up!" ) except Exception, e: print clock.timename(), e print clock.timename( ), " Could not send startup SMS!" if status_tel == "local_mode" and self.local_mode_sms == 0: print clock.timename( ), " A SMS will be sent about telescope in local mode..." send_song_mail.send_mail().send_sms( receiver=m_conf.notify_sms_who, message= "The telescope is put into local mode and cannot start up!" )
# print clock.timename(), "Still checking things... ", threading.activeCount() if conf.check_mirror_covers == 1: mir_cover_state = comm2tcs_read.GET_TSI().get_auxiliary_cover_realpos(sender="observer") if float(mir_cover_state) == 0.0: comm2tcs_write.SET_TSI().set_auxiliary_cover_targetpos(param=1, sender="observer") print clock.timename(), "Now trying to open the mirror covers... " sys.stdout.flush() time_out = time.time() + 180.0 while mir_cover_state != '1.0': mir_cover_state = comm2tcs_read.GET_TSI().get_auxiliary_cover_realpos(sender="observer") time.sleep(5) if time.time() > time_out: print clock.timename(), "The while loop has timed out and the mirror covers are most likely open!" if conf.send_notifications == 1: print send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS",subject="Mirror cover opening timeout!",message="The mirror cover loop timed out in the beginning of the observing script.\n\nSend at: %s\n\n" % clock.obstimeUT()) break elif mir_cover_state == '1.0': print clock.timename(), "The mirror covers are now open" sys.stdout.flush() ### CHECK IF THAR and do one if needed! # do_thar = 0 # if obs_req_values["obs_mode"] == "none-iodine" and song_checker.Checker().check_last_thar_spectrum() == 1: # #### Acquire ONE Thorium Argon spectrum # acq_thar = threading.Thread(target=do_action.acquire_one_thar, args=(obs_req_nr, "acq")) # Acquire a Thorium Argon spectre while slewing... # acq_thar.start() # do_thar = 1 print clock.timename(), "Number of threads: ", threading.activeCount()
########################### #### Shut down telescope ### if song_monitor_config.mon_telescope_actions == 1: if float(self.slit_state) != 0.0 or float( self.flap_state) != 0.0 or ( float(self.mirror_cover_state) != 0.0 and song_monitor_config.allow_mc_open != "yes"): status_tel = self.perform_actions.shutdown_telescope() if song_monitor_config.send_notifications == "yes": send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Telescope Shutdown!", message= "The telescope is now being shut down!\n\nThe weather has turned bad and the telescope will be shut down!\n\nSend at: %s\n\n" % clock.obstimeUT()) if status_tel == "done": self.who_did_it_tel = "weather" self.time_delay_telescope = time.time() + ( song_checker_config.telescope_delay_time * 60) print clock.timename( ), " Weather check deduced output: ", output2 print clock.timename( ), " The telescope was shut down due to bad weather" dome_check = self.perform_actions.check_shutdown() if dome_check == 1:
def handle_cooler(self, side_port_group_1, side_port_group_2, side_port_group_3, side_port_group_4, telescope_state, slit_state, flap_state, mirror_cover_state, who_did_it_tel, who_did_it_sp): """ Checks the dome and side port state and will turn the cooler on or off depending on the state of these. """ ### If any of the dome openings are open the cooler will be stoped: if float(side_port_group_1) != 0.0 or float( side_port_group_2) != 0.0 or float( side_port_group_3) != 0.0 or float( side_port_group_4) != 0.0 or float( slit_state) != 0.0 or float(flap_state) != 0.0: if song_monitor_config.mon_cooler_actions == 1 and float( self.cooler_state) == 1.0: status_cooler = self.perform_actions.stop_cooler() if status_cooler != "done" and song_monitor_config.send_notifications == "yes": send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Cooler not off!", message= "The dome or side ports were opened and the cooling unit was not switched off!\n\nSend at: %s\n\n" % clock.obstimeUT()) self.cooler_state = self.perform_actions.cooler_status() elif song_monitor_config.mon_cooler_actions == 0 and float( self.cooler_state) == 1.0: print clock.timename( ), " The cooler whould have been stopped now" self.cooler_state = 0 ### If all dome openings are closed the cooler are allowed to start: elif float(side_port_group_1) == 0.0 and float( side_port_group_2) == 0.0 and float( side_port_group_3) == 0.0 and float( side_port_group_4) == 0.0 and float( slit_state) == 0.0 and float(flap_state) == 0.0: if song_monitor_config.mon_cooler_actions == 1 and float( self.cooler_state) == 0.0: status_cooler = self.perform_actions.start_cooler( m_conf.cooling_temp) if status_cooler != "done" and song_monitor_config.send_notifications == "yes": send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Cooler not on!", message= "The cooling unit was not switched on!\n\nSend at: %s\n\n" % clock.obstimeUT()) self.cooler_state = self.perform_actions.cooler_status() elif song_monitor_config.mon_cooler_actions == 0 and float( self.cooler_state) == 0.0: print clock.timename( ), " The cooler whould have been started now" self.cooler_state = 1 return 1
except Exception, e: pass print("Starting daemon...!") daemon.start() elif opt == "-t": daemon.stop() print "The daemon is stoped!" else: print("Option %s not supported!" % (opt)) if __name__ == '__main__': try: main() except Exception, e: print e print "The scheduler has crashed at: ", clock.obstimeUT() send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Scheduler Crash!", message= "The scheduler daemon has crashed!\n\nCheck the log file to see why!\n\nMaybe a simple restart helps!!\nLog onto hw as the user obs and type\nor_scheduler -s\nThis should start the scheduler. Check the log file /home/obs/logs/or_scheduler.log if it starts correctly." ) send_song_mail.send_mail().send_sms( receiver=m_conf.wakeup_sms_who, message= "The scheduler daemon was stopped for some reason. You got mail!")
def check_memory_hw(self): if song_monitor_config.mon_memory_hw == 1: try: import psutil except Exception, e: print e print "psutil was not installed..." return 0 else: # psutil.virtual_memory(): # vmem(total=8392966144L, available=7489146880L, percent=10.8, used=7220899840L, free=1172066304L, active=2213416960, inactive=4481515520, buffers=375693312L, cached=5941387264) mem = psutil.virtual_memory() swap = psutil.swap_memory() print clock.timename( ), " HW memory usage: Tot: %iMB, Av: %iMB, Us: %iMB, Free: %iMB, Act: %iMB, InAct: %iMB, Buf: %iMB, Cach: %iMB" % ( int(mem.total / 1.0e6), int(mem.available / 1.0e6), int(mem.used / 1.0e6), int(mem.free / 1.0e6), int(mem.active / 1.0e6), int(mem.inactive / 1.0e6), int(mem.buffers / 1.0e6), int(mem.cached / 1.0e6)) print clock.timename( ), " HW SWAP memory usage: Tot: %iMB, Us: %iMB, Free: %iMB" % ( int(swap.total / 1.0e6), int( swap.used / 1.0e6), int(swap.free / 1.0e6)) if ( swap.free / 1.0e6 ) < song_monitor_config.hw_swap_limit and self.notify_swap_delay < time.time( ): print clock.timename( ), " Swap memory is in use. Swap memory left on hw: %f MB\n" % ( swap.free / 1.0e6) value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="hw out of swap memory!", message= "hw is using swap meory and only %fMB swap memory is left.\n\n Maybe restart some of the python modules to free memory!!\n\n Please be aware!\n\nSend at: %s\n\n" % ((swap.free / 1.0e6), clock.obstimeUT())) #value = send_song_mail.send_mail().send_sms(receiver=m_conf.wakeup_sms_who, message="hw is using up the swap memory. Please be aware!!!") self.notify_swap_delay = time.time() + ( 24. * 60. * 60.) # Add a delay of 1 day elif ( mem.free / 1.0e6 ) < song_monitor_config.hw_mem_limit and self.notify_mem_delay < time.time( ): print clock.timename( ), " There is not much memory left on hw: %f MB\n" % ( mem.free / 1.0e6) value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="hw out of memory!", message= "hw has %fMB memory left.\n\n Maybe restart some of the python modules to free memory!!\n\n Please be aware!\n\nSend at: %s\n\n" % ((mem.free / 1.0e6), clock.obstimeUT())) self.notify_mem_delay = time.time() + ( 24. * 60. * 60.) # Add a delay of 1 day return 1
or time_stamp["ins_at"] + datetime.timedelta(seconds=300) < datetime.datetime.utcnow()): # if float(mean_value) >= float(song_checker_config.spec_box_temp_max) and time.time() > self.delay_time: if self.verbose == 'yes': print clock.timename( ), " The temperature inside the spectrograph box was %s and an email was sendt out" % ( str(mean_value)) if self.mon_house_hold_value == 1: # If the temperature is higher than the allowed limit an email will be sendt out. if song_monitor_config.send_notifications == "yes": if time_stamp["ins_at"] + datetime.timedelta( seconds=300) < datetime.datetime.utcnow(): value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Temperatures too old!", message= "The temperatures (from our tmon sensor) in the database are no longer up to date.\nLast datapoint was inserted at %s.\nPlease do something!!!\n\nSend at: %s\n\n" % (str(time_stamp), clock.obstimeUT())) elif temp_values["temperature_11"] > float( song_checker_config.spec_box_temp_max): value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Spec box temp too high!", message= "The temperature inside the spectrograph box was %s.\nPlease do something!!!\n\nSend at: %s\n\n" % (str(temp_values["temperature_11"]), clock.obstimeUT())) elif temp_values["temperature_11"] < float(
class Check_House_Hold(object): """ @brief: This class handles all checks on the time of day. """ def __init__(self): """ Initialization of the time checks. """ self.mon_house_hold_value = song_monitor_config.mon_house_hold_actions # 1 = do things, 0 = do nothing. self.verbose = song_monitor_config.verbose self.checker = song_checker.Checker() self.delay_time = time.time() self.specbox_delay_time = time.time() self.email_slony = 0 self.email_ds = 0 self.email_ds_alarm = 0 self.email_ds_scratch = 0 self.email_ds_scratch_alarm = 0 self.slony_delay = time.time() self.notify_swap_delay = 0 self.notify_mem_delay = 0 self.notify_test_i2_heater_delay = 0 self.notify_i2_heater_delay = 0 def check_container(self): """ Checks the temperature inside the container. """ try: temp_values = read_value_db.get_fields_with_id( "house_hold", [ "temperature_9", "temperature_10", "temperature_13", "temperature_14" ], "box_id", "2") time_stamp = read_value_db.get_fields_with_id( "house_hold", ["ins_at"], "box_id", "2") except Exception, e: print clock.timename( ), " Could not connect to the database to read container temperatures" return 0 temp_arr = [] for temp in temp_values.values(): if float(temp) != float(1.0): temp_arr.append(float(temp)) ### Calculate the mean temperature of the four messured inside the container: mean_value = numpy.mean(temp_arr) if self.verbose == 'yes' and float(mean_value) >= float( song_checker_config.container_temp): print clock.timename( ), " The temperature inside the container was:", mean_value if float(mean_value) >= float(song_checker_config.container_temp ) and time.time() > self.delay_time: if self.verbose == 'yes': print clock.timename( ), " The temperature inside the container was %s and an email was sendt out" % ( str(mean_value)) # If the temperature is higher than the allowed limit an email will be sendt out. if self.mon_house_hold_value == 1: if song_monitor_config.send_sms == "yes": value = send_song_mail.send_mail().send_sms( receiver=m_conf.wakeup_sms_who, message= "The temperature inside the container is higher than %s. Please do something!!!" % str(song_checker_config.container_temp)) if song_monitor_config.send_to_support == "yes": send_song_mail.send_mail().sending_an_email( reciever=['support'], sender="SONG_MS", subject="Container temp too high!", message= "The temperature inside the container is higheder than %s.\nPlease contact the IAC night operator to tell him to go to the container. Maybe the air condition is failing.!!!\n\nSend at: %s\n\n" % (str( song_checker_config.dome_temp), clock.obstimeUT())) if song_monitor_config.send_notifications == "yes": value = send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="Container temp too high!", message= "The temperature inside the container is higheder than %s.\nPlease contact the IAC night operator to tell him to go to the container. Maybe the air condition is failing.!!!\n\nSend at: %s\n\n" % (str( song_checker_config.dome_temp), clock.obstimeUT())) # Send message to error daemon #ticket_container = error_notifier.serror(20,"Monitor: Container temperature is %s" % str(mean_value)) ###################### # The monitor should stop the telescope, park it and close the dome and close the side ports. ###################### self.delay_time = time.time( ) + 7200 # this is so that the function will not sendt out an email every 10 second. return 1
def check_slony(self): """ @brief: This function will check if data are copied to the central database. """ if song_monitor_config.mon_slony == 1: slony_val = "" time_stamp = 0 try: slony_val, time_stamp = self.checker.check_slony() except Exception, e: print clock.timename(), " Could not check slony" print clock.timename(), e if slony_val == 0: print clock.timename(), " Slony is working fine" if self.email_slony == 0 and slony_val != 0: if slony_val == 1: value = send_song_mail.send_mail().sending_an_email( reciever=["mads", "frank", "eric"], sender="SONG_MS", subject="Slony is down!", message= "Slony is not replicating the ORs from central to tenerife!!!Last weather data was inserted at: %s\n\nSend at: %s\n\n" % (time_stamp, clock.obstimeUT())) print clock.timename( ), " Slony was not replicating the ORs from central to tenerife" elif slony_val == 2: value = send_song_mail.send_mail().sending_an_email( reciever=["mads", "frank", "eric"], sender="SONG_MS", subject="Slony is down!", message= "Slony is not replicating data from tenerife to central!!!\nLast weather data was inserted at: %s\n\nSend at: %s\n\n" % (time_stamp, clock.obstimeUT())) print clock.timename( ), " Slony was not replicating data from tenerife to central" elif slony_val == 3: value = send_song_mail.send_mail().sending_an_email( reciever=["mads", "frank", "eric"], sender="SONG_MS", subject="Slony is down!", message= "Slony is not replicating at all!!!\n\nSend at: %s\n\n" % clock.obstimeUT()) print clock.timename(), " Slony was not replicating at all" elif slony_val == 5: value = send_song_mail.send_mail().sending_an_email( reciever=["mads", "frank", "eric"], sender="SONG_MS", subject="Database is down!", message= "Connection to the central database could not be achieved!!!\n\nSend at: %s\n\n" % clock.obstimeUT()) print clock.timename( ), " Connection to the central database was not achieved" elif slony_val == 4: value = send_song_mail.send_mail().sending_an_email( reciever=["mads", "frank", "eric"], sender="SONG_MS", subject="Database is down!", message= "Connection to the tenerife database could not be achieved!!!\n\nSend at: %s\n\n" % clock.obstimeUT()) print clock.timename( ), " Connection to the tenerife database was not achieved" self.email_slony = 1 self.slony_delay = time.time() elif self.email_slony == 1 and time.time( ) >= self.slony_delay + 86400: # Only allow one email per day. self.email_slony = 0 self.slony_delay = 0 if slony_val == 5: print clock.timename( ), " Connection to the central database was not achieved" elif slony_val == 4: print clock.timename( ), " Connection to the tenerife database was not achieved" return 1
days=1 ) #a date string. Format should be YYYMMDD and it should be the date the observations started!!! ret_val = "" try: print clock.timename( ), "Calling SONGwriter to start the extraction of the spectre of the night: ", obs_night_date.strftime( "%Y%m%d") ret_val = songwriter_xmlrpcclient.all_science( obs_night_date.strftime("%Y%m%d")) # ret_val = songwriter_MFA_client.all_science(obs_night_date.strftime("%Y%m%d")) except Exception, e: print clock.timename(), e send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="SONGwriter problem!", message= "The request to start reducing the data from last night failed...\n\nSend at: %s\n\n" % clock.obstimeUT()) if ret_val != 1: send_song_mail.send_mail().sending_an_email( reciever=m_conf.notify_email_who, sender="SONG_MS", subject="SONGwriter problem!", message= "The request to start reducing the data from last night failed...\n\nSend at: %s\n\n" % clock.obstimeUT()) ########################################################################################## ##########################################################################################
if song_monitor_config.mon_side_ports == 1: # If the config file specifies it side port checks will be carried out. try: song_checker_handle.check_side_ports() except Exception,e: print clock.timename(), " Problem in check_side_ports()" print clock.timename(), e if song_monitor_config.mon_telescope == 1: # If the config file specifies it telescope checks will be carried out. try: song_checker_handle.check_telescope() except Exception,e: print clock.timename(), e print clock.timename(), " Problem in check_telescope()" if song_monitor_config.send_notifications == "yes" and self.telescope_hangs_sms == 0: send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS",subject="Telescope Error!",message="The monitor could not connect to the telescope!\n\nPlease check if something is wrong. Check if the TSI is running on the astelco pc. Check if the tcs_daemon is running on hw. \n\nSend at: %s\n\n" % (clock.obstimeUT())) if song_monitor_config.send_sms == "yes": print clock.timename(), " Sending an sms to: %s " % (m_conf.wakeup_sms_who) send_song_mail.send_mail().send_sms(receiver=m_conf.wakeup_sms_who, message="The monitor could not connect to the telescope!") if song_monitor_config.send_to_support == "yes": send_song_mail.send_mail().sending_an_email(reciever=['support'],sender="SONG_MS",subject="Telescope Error!",message="The monitor could not connect to the telescope!\n\nPlease check if something is wrong. Check if the TSI is running on the astelco pc. Check if the tcs_daemon is running on hw. \n\nSend at: %s\n\n" % (clock.obstimeUT())) self.telescope_hangs_sms = 1 if song_monitor_config.mon_network == 1: # If the config file specifies it netowrk checks will be carried out. try: song_checker_handle.check_network() except Exception,e: print clock.timename(), " Problem in check_network()" print clock.timename(), e