def check_disk_space_scratch(self):
if song_monitor_config.mon_disk_space == 1:
# output = os.popen("df").readlines()[-4]
for line in os.popen("df").readlines():
if "scratch" in line.split()[-1]:
scratch_device, scratch_size, scratch_used, scratch_available, scratch_percent, scratch_mountpoint = line.split(
)
scratch_free_ds_percent = (float(scratch_available) /
float(scratch_size)) * 100.
scratch_used_ds_percent = (float(scratch_used) /
float(scratch_size)) * 100.
if float(
scratch_available
) < song_monitor_config.scratch_ds_limit and self.email_ds_scratch_alarm == 0:
print clock.timename(
), " There is not much space left on scratch: %f GB\n" % (
float(scratch_available) / 1000000.)
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="scratch out of disk space!",
message=
"scratch has %sGB disk space left. Please clean now!\n\nSend at: %s\n\n"
% ((float(scratch_available) / 1000000.),
clock.obstimeUT()))
self.email_ds_scratch_alarm = 1
elif float(
scratch_available
) < song_monitor_config.scratch_ds_soft_limit and self.email_ds_scratch == 0:
print clock.timename(
), " There is not much space left on scratch: %f GB\n" % (
float(scratch_available) / 1000000.)
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="scratch out of disk space!",
message=
"scratch has %sGB disk space left. Please clean soon!\n\nSend at: %s\n\n"
% ((float(scratch_available) / 1000000.),
clock.obstimeUT()))
self.email_ds_scratch = 1
elif float(
scratch_available
) < song_monitor_config.scratch_ds_soft_limit and self.email_ds_scratch == 1:
print clock.timename(
), " There is not much space left on scratch: %f GB" % (
float(scratch_available) / 1000000.)
elif float(scratch_available
) > song_monitor_config.scratch_ds_soft_limit:
print clock.timename(
), " scratch has sufficient disk space left: %f GB" % (
float(scratch_available) / 1000000.)
self.email_ds_scratch = 0
return 1
else:
return 1
def check_machines(self):
for m in song_monitor_config.machines:
output = self.timeout_command(["ssh", m, "date"], 15)
print clock.timename(), "Output from %s was: %s" % (
m, str(output).strip())
if output == None:
print clock.timename(), "The check on %s had a timeout..." % (
m)
print clock.timename(), "Now sending an e-mail..."
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="%s is not responding" % (m),
message=
"%s is not responding on the ssh connection from hw.sstenerife.prv. It might be hanging...\n\nPlease check!!"
% (m))
if song_monitor_config.send_sms == "yes":
value = send_song_mail.send_mail().send_sms(
receiver=m_conf.wakeup_sms_who,
message="%s is not responding. Please check!" % (m))
return 1
def check_iodine_heaters(self):
if song_monitor_config.mon_iodine_heaters == 1:
#### Check test iodine cell heater
try:
output = self.timeout_command([
"ssh", "10.8.0.111",
"python /home/obs/mads/i2_usb.py tact?"
], 10)
except Exception, e:
print clock.timename(
), " Could not get test iodine heater data"
if self.notify_test_i2_heater_delay < time.time():
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Could not check iodine test cell heater!",
message=
"The monitor caught an exception when trying to check the test iodine cell heater.\n\n Please check the machine: IP: 10.8.0.111 as obs to see if it is running. Use the script > python /home/obs/mads/i2_usb.py' to check and enable the heater!\n\nSend at: %s\n\n"
% (clock.obstimeUT()))
self.notify_test_i2_heater_delay = time.time() + (
24. * 60. * 60.) # Add a delay of 1 day
else:
if output == None:
print clock.timename(
), " Did not get data... now sending an e-mail..."
if self.notify_test_i2_heater_delay < time.time():
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Iodine test cell not heated!",
message=
"The monitor did not get a reply from the small hp machine to which the iodine test cells heater is connected.\n\n Please check the machine: IP: 10.8.0.111 as obs to see if it is running. Use the script > python /home/obs/mads/i2_usb.py' to check and enable the heater!\n\nSend at: %s\n\n"
% (clock.obstimeUT()))
self.notify_test_i2_heater_delay = time.time() + (
24. * 60. * 60.) # Add a delay of 1 day
else:
#print float(output[0].strip(">\r\n C"))
try:
temp = float(output.split()[1])
except Exception, e:
print clock.timename(
), " Did not get meaningful data from test iodine heater... now sending an e-mail..."
if self.notify_test_i2_heater_delay < time.time():
value = send_song_mail.send_mail(
).sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Iodine test cell not heated!",
message=
"The monitor did not get a meaningful reply from the heater.\n\n Please log onto the machine: IP: 10.8.0.111 as obs and use the script > python /home/obs/mads/i2_usb.py' to check and enable the heater!\n\nSend at: %s\n\n"
% (clock.obstimeUT()))
self.notify_test_i2_heater_delay = time.time() + (
24. * 60. * 60.) # Add a delay of 1 day
else:
def check_disk_space_hw(self):
if song_monitor_config.mon_disk_space == 1:
df = subprocess.Popen(["df", song_monitor_config.outstream],
stdout=subprocess.PIPE)
output = df.communicate()[0]
hw_device, hw_size, hw_used, hw_available, hw_percent, hw_mountpoint = output.split(
"\n")[1].split()
hw_free_ds_percent = (float(hw_available) / float(hw_size)) * 100.
hw_used_ds_percent = (float(hw_used) / float(hw_size)) * 100.
if float(
hw_available
) < song_monitor_config.hw_ds_limit and self.email_ds_alarm == 0:
print clock.timename(
), " There is not much space left on hw: %f GB" % (
float(hw_available) / 1000000.)
print clock.timename(
), " Free disk space in percent: %f\n" % hw_free_ds_percent
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="hw out of disk space!",
message=
"hw has less than 15 percent disk space left. Please check soon!\n\nSend at: %s\n\n"
% clock.obstimeUT())
self.email_ds_alarm = 1
elif hw_free_ds_percent < song_monitor_config.hw_ds_soft_limit_percent and self.email_ds == 0:
print clock.timename(
), " There is not much space left on hw: %f GB" % (
float(hw_available) / 1000000.)
print clock.timename(
), " There is less than 15 percent disk space left on hw!"
print clock.timename(
), " Free disk space in percent: %f\n" % hw_free_ds_percent
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="hw out of disk space!",
message=
"hw has less than 15 percent disk space left. Please check soon!\n\nSend at: %s\n\n"
% clock.obstimeUT())
self.email_ds = 1
elif hw_free_ds_percent > song_monitor_config.hw_ds_soft_limit_percent:
print clock.timename(
), " hw has sufficient disk space left: %f GB" % (
float(hw_available) / 1000000.)
self.email_ds = 0
return 1
else:
return 1
def check_if_monitor_runs(self): delay_time = 0 while RUNNING: if past_time + 600 < time.time() and self.monitor_hangs_send_mail == 0: print clock.timename(), " It has been 10 minutes since the monitor went through a loop!" send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS",subject="Monitor hangs!",message="The monitor daemon is hanging somewhere and a restart is most likely needed. Please check if things looks fine!\n\nThis is what will be send out if the connection to the telescope is lost from the Astelco side\n\nSend at: %s" % clock.obstimeUT()) send_song_mail.send_mail().send_sms(receiver=m_conf.wakeup_sms_who, message="The monitor daemon hangs for some reason!") if song_monitor_config.send_to_support == "yes": send_song_mail.send_mail().sending_an_email(reciever=['support'],sender="SONG_MS",subject="Monitor hangs!",message="The monitor daemon is hanging somewhere and a restart is most likely needed. Please check if things looks fine!\n\nThis is what will be send out if the connection to the telescope is lost from the Astelco side\n\nSend at: %s" % clock.obstimeUT()) self.monitor_hangs_send_mail = 1 delay_time = time.time() + 3600 # Wait an hour to send another SMS and emails. elif past_time + 600 > time.time() and self.monitor_hangs_send_mail == 0: loop_time = time.time() - past_time print clock.timename(), " Monitor runs as it should... loop past %s seconds ago" % loop_time else: if delay_time < time.time() and self.monitor_hangs_send_mail == 1: self.monitor_hangs_send_mail = 0 time.sleep(30)
def run(self):
"""
"""
done_log_param = 0
global RUNNING
RUNNING = True
#self.selector = selector.SELECTOR()
#self.insertor = insertor
self.sun_handle = song_star_checker.sun_pos(
site=m_conf.song_site) # Tenerife sun handle
self.daily_update = 0
self.daily_sun_insert = 0
val = beating_heart.start_heartbeat(job_id=m_conf.conductor_id)
over_all_obs_state = get_database_values(table_name='tel_dome',
fields=['extra_param_1'])
gc.disable()
process = psutil.Process(os.getpid())
print("Memory used in percent : %0.3f" %
float(process.memory_percent()))
mem = process.memory_info()
mem_old = (float(mem.vms) / float(2**20))
print ""
print clock.timename(), "The Conductor was started!\n"
print clock.timename(
), "The over all observing state was: ", over_all_obs_state[
'extra_param_1']
while RUNNING:
if int(
float(
datetime.datetime.strftime(
datetime.datetime.utcnow(),
"%H"))) < 9 and self.sun_handle.sun_alt(
unit='f') < 0.0:
yesterday = datetime.datetime.utcnow() - datetime.timedelta(
days=1)
folder_date = yesterday.strftime('%Y-%m-%d')
else:
folder_date = datetime.datetime.strftime(
datetime.datetime.utcnow(), "%Y-%m-%d")
# print "Checking the date: %s" % (str(folder_date))
###################### DAY TIME CHECK ################################
if int(
float(
datetime.datetime.strftime(
datetime.datetime.utcnow(),
"%H"))) < 10 and self.sun_handle.sun_alt(
unit='f') > 1.0:
try:
daytime_obs = check_daytime.Check_Daytime(folder_date)
except Exception, e:
print clock.timename(), e
else:
if daytime_obs == 0 and self.daily_sun_insert == 0:
print clock.timename(
), "The Solar observations were not inserted yet. Will do now"
try:
#insertor.insert_solar_observations()
insertor.insert_solar_observations_soda()
except Exception, e:
print clock.timename(
), "Did not call the check-for-gaps function"
print clock.timename(), e
else:
self.daily_sun_insert = 1
if conf.check_for_timecritical == "yes" and self.daily_update == 0:
st = datetime.datetime.utcnow()
se = datetime.datetime.utcnow() + datetime.timedelta(
days=1)
try:
selected_target = selector.SELECTOR().determine_next(
timegap=[st, se],
project_critical_type=["timecritical"])
except Exception, e:
print clock.timename(
), "Problem in the selector.determine_next function..."
print clock.timename(), e
else:
if selected_target != None:
# Checking if they are already inserted:
ids = []
names = []
for tmp_or in selected_target:
ids.append(tmp_or[1])
names.append(tmp_or[2])
check_val = check_daytime.check_timecritical(
folder_date, ids)
if check_val == 0:
for tcrit_target in selected_target:
insertor.insert_timecritical_OR(
pre_obs_spec_id=tcrit_target[1])
try:
send_song_mail.send_mail(
).sending_an_email(
reciever=["mads", "frank"],
sender="SONG_MS",
subject="Time critical OR inserted",
message=
"The conductor has inserted time critical OR(s) for the coming night.\nObject(s): %s\n"
% str(names))
except Exception, e:
print "Could not send e-mail on time critical insertion..."
done_log_param = 1
if done_log_param == 1 and int(
float(time.strftime("%H", time.gmtime()))) > 12:
done_log_param = 0
self.daily_update = 0
self.daily_sun_insert = 0
process = psutil.Process(os.getpid())
print("Memory used in percent : %0.3f" %
float(process.memory_percent()))
if float(process.memory_percent()) > 60:
send_song_mail.send_mail().sending_an_email(
reciever=["mads"],
sender="SONG_MS",
subject="Conductor using all memory!",
message=
"The conductor has used more than 60 percent of the total memory on the song VM!\n\nPlease restart to clear memory ASAP!"
)
os.system(
"python /home/madsfa/subversion/central_trunk/conductor/conductor.py -t ; python /home/madsfa/subversion/central_trunk/conductor/conductor.py -s"
)
mem = process.memory_info()
print "Memory used in MB: %.1f" % ((float(mem.vms) / float(2**20)))
print "Memory increase in MB: %.2f" % (
(float(mem.vms) / float(2**20)) - mem_old)
mem_old = (float(mem.vms) / float(2**20))
gc.collect()
class Check_Time(object):
"""
@brief: This class handles all checks on the time of day.
"""
def __init__(self):
"""
Initialization of the time checks.
"""
self.mon_time_value = song_monitor_config.mon_time # 1 = do things, 0 = do nothing.
self.sun_handle = song_star_checker.sun_pos(
site=m_conf.song_site) # site=1: Tenerife
self.verbose = song_monitor_config.verbose
self.perform_actions = mon_actions.Do_Actions()
self.skycam_start = m_conf.start_skycam_movie
self.local_mode = 0
self.sms_send = 0
self.check_ors = 0
self.open_mirror_covers = 0
self.bad_weather_sms = 0
self.scheduler_val = 0
self.local_mode_sms = 0
self.x_mas_notify = 0
def check_sun(self, side_port_group_1, side_port_group_2,
side_port_group_3, side_port_group_4, telescope_state,
slit_state, flap_state, mirror_cover_state, who_did_it_tel,
who_did_it_sp):
"""
Checks the Suns position and returns a value corresponding to this.
"""
self.side_port_group_1 = side_port_group_1
self.side_port_group_2 = side_port_group_2
self.side_port_group_3 = side_port_group_3
self.side_port_group_4 = side_port_group_4
self.telescope_state = telescope_state
self.slit_state = slit_state
self.flap_state = flap_state
self.mirror_cover_state = mirror_cover_state
self.who_did_it_tel = who_did_it_tel
self.who_did_it_sp = who_did_it_sp
sun_alt = self.sun_handle.sun_alt()
if str(sun_alt)[0] == "-":
sun_alt_d = float(str(sun_alt).split(":")[0]) - float(
str(sun_alt).split(":")[1]) / 60.0 - float(
str(sun_alt).split(":")[2]) / 3600.0
elif str(sun_alt)[0] != "-":
sun_alt_d = float(str(sun_alt).split(":")[0]) + float(
str(sun_alt).split(":")[1]) / 60.0 + float(
str(sun_alt).split(":")[2]) / 3600.0
#################################
#### Get time to next sun set ###
tmp_time_str2 = datetime.datetime.strptime(
str(self.sun_handle.sun_set_next()), "%Y/%m/%d %H:%M:%S")
time_diff = tmp_time_str2 - datetime.datetime.utcnow()
hours_to_next_sun_set = int(
time_diff.days) * 24. + time_diff.seconds / (24. * 3600.) * 24
#### Get time from previous sun set ###
tmp_time_str2 = datetime.datetime.strptime(
str(self.sun_handle.sun_set_pre()), "%Y/%m/%d %H:%M:%S")
time_diff2 = datetime.datetime.utcnow() - tmp_time_str2
hours_from_pre_sun_set = int(
time_diff2.days) * 24. + time_diff2.seconds / (24. * 3600.) * 24
#################################
### GET DUST VALUE: ##############################
try:
conn = psycopg2.connect(
"host='%s' dbname='%s' user='******' password='******'" %
(m_conf.db_host, m_conf.data_db, m_conf.db_user,
m_conf.db_password))
curr = conn.cursor()
stmt = 'SELECT dust_level FROM weather_station WHERE ins_at = (SELECT max(ins_at) FROM weather_station)'
curr.execute(stmt)
results = curr.fetchone()
dust_value = results[0]
except Exception, e:
print clock.timename(), " Could not get dust value from database!"
dust_value = 0.011
if song_monitor_config.mon_telescope_actions == 1:
#### Get telescope poitning direktion ####
tel_value = get_db_values.db_connection().get_fields_site01(
"tel_dome", ["tel_az"])
tel_az = tel_value["tel_az"]
# if float(self.telescope_state) == float(-3.0) and self.local_mode == 0:
# print clock.timename(), " The telescope was put in local mode"
# if song_monitor_config.send_notifications == "yes":
# send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS",subject="Telescope in local mode!", message="The telescope was put in local mode!\n\nSend at: %s\n\n" % clock.obstimeUT())
# if song_monitor_config.send_notify_sms == "yes":
# send_song_mail.send_mail().send_sms(receiver=["Mads"], message="Someone has put the telescope into local mode")
# self.local_mode = 1
# elif float(self.telescope_state) == float(0.0) and self.local_mode == 1:
# print clock.timename(), " The telescope was put back in remote mode"
# if song_monitor_config.send_notifications == "yes":
# send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS", subject="Telescope back in remote mode!", message="The telescope was put back in remote mode!\n\nSend at: %s\n\n" % clock.obstimeUT())
# self.local_mode = 0
if float(sun_alt_d) >= (
float(m_conf.obs_sun_alt) + 2.0
) and hours_to_next_sun_set >= m_conf.open_time_tel and hours_from_pre_sun_set >= m_conf.open_time_tel:
#######################################
#### Close dome and park telescope ####
#######################################
if self.skycam_start == 0:
print clock.timename(), " Resetting skycam starting parameter"
self.skycam_start = 1
try:
self.sc2_handle.kill()
except Exception, e:
print e
print clock.timename(
), " Could not kill the skycam 2 process"
if song_monitor_config.mon_telescope_actions == 1 and float(
self.telescope_state) != float(0.0) and float(
self.telescope_state) != float(-3.0):
# if float(self.slit_state) != 0.0 or float(self.flap_state) != 0.0 or float(self.mirror_cover_state) != 0.0:
if float(self.slit_state) != 0.0 or float(
self.flap_state) != 0.0 or (
float(self.mirror_cover_state) != 0.0
and song_monitor_config.allow_mc_open != "yes"):
status_tel = self.perform_actions.shutdown_telescope()
if status_tel == "done":
self.who_did_it_tel = "time"
print clock.timename(
), " The dome was closed due to daylight"
dome_check = self.perform_actions.check_shutdown()
if dome_check == 1:
print clock.timename(
), " The dome was closed correctly"
elif dome_check == 2:
print clock.timename(
), " The dome was closed but mirror cover open"
if song_monitor_config.send_notifications == "yes":
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Mirror covers were open!",
message=
"The dome was closed but mirror covers open!\n\nSend at: %s\n\n"
% clock.obstimeUT())
else:
if song_monitor_config.send_to_wake_up == "yes":
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Telescope Closing Error!",
message=
"The dome was not closed correctly after shutdown!\n\nSend at: %s\n\n"
% clock.obstimeUT())
if song_monitor_config.send_sms == "yes" and self.sms_send == 0:
send_song_mail.send_mail().send_sms(
receiver=m_conf.wakeup_sms_who,
message="The dome was not closed corretly...")
if song_monitor_config.send_to_support == "yes" and self.sms_send == 0:
send_song_mail.send_mail().sending_an_email(
reciever=['support'],
sender="SONG_MS",
subject="Telescope Closing Error!",
message=
"The dome was not closed correctly after shutdown!\n\nSend at: %s\n\n"
% clock.obstimeUT())
self.sms_send = 1
###### Stop sigu and pugu ########
print clock.timename(
), " Stopping the slit and pupil guiders"
try:
status_sigu_pugu = self.perform_actions.stop_sigu_and_pugu(
)
except Exception, e:
print clock.timename(
), " Could not connect to Sigu and/or Pugu. They might be stopped already!"
if status_sigu_pugu != 1:
print clock.timename(
), " Something went wrong when trying to stop sigu and pugu"
else:
if self.who_did_it_tel == "free":
self.who_did_it_tel = "time"
self.time_delay_telescope = time.time() + (
song_checker_config.telescope_delay_time * 60)
### Maybe the sms send parameter should be placed elsewhere...
self.sms_send = 0
status_tel = self.perform_actions.startup_telescope()
if status_tel == "done":
print clock.timename(
), " The telescope is now ready for observing"
#if song_monitor_config.send_notifications == "yes":
#send_song_mail.send_mail().sending_an_email(reciever=["mads"],sender="SONG_MS",subject="Observations startup!",message="The Sun has set and observation can be started!")
if hours_to_next_sun_set < m_conf.open_time_tel and hours_to_next_sun_set > m_conf.open_time_tel - 1:
try:
if song_monitor_config.send_notify_sms == "yes" and self.who_did_it_tel == "time":
print clock.timename(
), " A SMS will be sent about telescope startup..."
send_song_mail.send_mail().send_sms(
receiver=m_conf.notify_sms_who,
message=
"Weather is great at Tenerife and the telescope has opend up!"
)
except Exception, e:
print clock.timename(), e
print clock.timename(
), " Could not send startup SMS!"
if status_tel == "local_mode" and self.local_mode_sms == 0:
print clock.timename(
), " A SMS will be sent about telescope in local mode..."
send_song_mail.send_mail().send_sms(
receiver=m_conf.notify_sms_who,
message=
"The telescope is put into local mode and cannot start up!"
)
# print clock.timename(), "Still checking things... ", threading.activeCount() if conf.check_mirror_covers == 1: mir_cover_state = comm2tcs_read.GET_TSI().get_auxiliary_cover_realpos(sender="observer") if float(mir_cover_state) == 0.0: comm2tcs_write.SET_TSI().set_auxiliary_cover_targetpos(param=1, sender="observer") print clock.timename(), "Now trying to open the mirror covers... " sys.stdout.flush() time_out = time.time() + 180.0 while mir_cover_state != '1.0': mir_cover_state = comm2tcs_read.GET_TSI().get_auxiliary_cover_realpos(sender="observer") time.sleep(5) if time.time() > time_out: print clock.timename(), "The while loop has timed out and the mirror covers are most likely open!" if conf.send_notifications == 1: print send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS",subject="Mirror cover opening timeout!",message="The mirror cover loop timed out in the beginning of the observing script.\n\nSend at: %s\n\n" % clock.obstimeUT()) break elif mir_cover_state == '1.0': print clock.timename(), "The mirror covers are now open" sys.stdout.flush() ### CHECK IF THAR and do one if needed! # do_thar = 0 # if obs_req_values["obs_mode"] == "none-iodine" and song_checker.Checker().check_last_thar_spectrum() == 1: # #### Acquire ONE Thorium Argon spectrum # acq_thar = threading.Thread(target=do_action.acquire_one_thar, args=(obs_req_nr, "acq")) # Acquire a Thorium Argon spectre while slewing... # acq_thar.start() # do_thar = 1 print clock.timename(), "Number of threads: ", threading.activeCount()
###########################
#### Shut down telescope ###
if song_monitor_config.mon_telescope_actions == 1:
if float(self.slit_state) != 0.0 or float(
self.flap_state) != 0.0 or (
float(self.mirror_cover_state) != 0.0
and song_monitor_config.allow_mc_open != "yes"):
status_tel = self.perform_actions.shutdown_telescope()
if song_monitor_config.send_notifications == "yes":
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Telescope Shutdown!",
message=
"The telescope is now being shut down!\n\nThe weather has turned bad and the telescope will be shut down!\n\nSend at: %s\n\n"
% clock.obstimeUT())
if status_tel == "done":
self.who_did_it_tel = "weather"
self.time_delay_telescope = time.time() + (
song_checker_config.telescope_delay_time * 60)
print clock.timename(
), " Weather check deduced output: ", output2
print clock.timename(
), " The telescope was shut down due to bad weather"
dome_check = self.perform_actions.check_shutdown()
if dome_check == 1:
def handle_cooler(self, side_port_group_1, side_port_group_2,
side_port_group_3, side_port_group_4, telescope_state,
slit_state, flap_state, mirror_cover_state,
who_did_it_tel, who_did_it_sp):
"""
Checks the dome and side port state and will turn the cooler on or off depending on the state of these.
"""
### If any of the dome openings are open the cooler will be stoped:
if float(side_port_group_1) != 0.0 or float(
side_port_group_2) != 0.0 or float(
side_port_group_3) != 0.0 or float(
side_port_group_4) != 0.0 or float(
slit_state) != 0.0 or float(flap_state) != 0.0:
if song_monitor_config.mon_cooler_actions == 1 and float(
self.cooler_state) == 1.0:
status_cooler = self.perform_actions.stop_cooler()
if status_cooler != "done" and song_monitor_config.send_notifications == "yes":
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Cooler not off!",
message=
"The dome or side ports were opened and the cooling unit was not switched off!\n\nSend at: %s\n\n"
% clock.obstimeUT())
self.cooler_state = self.perform_actions.cooler_status()
elif song_monitor_config.mon_cooler_actions == 0 and float(
self.cooler_state) == 1.0:
print clock.timename(
), " The cooler whould have been stopped now"
self.cooler_state = 0
### If all dome openings are closed the cooler are allowed to start:
elif float(side_port_group_1) == 0.0 and float(
side_port_group_2) == 0.0 and float(
side_port_group_3) == 0.0 and float(
side_port_group_4) == 0.0 and float(
slit_state) == 0.0 and float(flap_state) == 0.0:
if song_monitor_config.mon_cooler_actions == 1 and float(
self.cooler_state) == 0.0:
status_cooler = self.perform_actions.start_cooler(
m_conf.cooling_temp)
if status_cooler != "done" and song_monitor_config.send_notifications == "yes":
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Cooler not on!",
message=
"The cooling unit was not switched on!\n\nSend at: %s\n\n"
% clock.obstimeUT())
self.cooler_state = self.perform_actions.cooler_status()
elif song_monitor_config.mon_cooler_actions == 0 and float(
self.cooler_state) == 0.0:
print clock.timename(
), " The cooler whould have been started now"
self.cooler_state = 1
return 1
except Exception, e:
pass
print("Starting daemon...!")
daemon.start()
elif opt == "-t":
daemon.stop()
print "The daemon is stoped!"
else:
print("Option %s not supported!" % (opt))
if __name__ == '__main__':
try:
main()
except Exception, e:
print e
print "The scheduler has crashed at: ", clock.obstimeUT()
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Scheduler Crash!",
message=
"The scheduler daemon has crashed!\n\nCheck the log file to see why!\n\nMaybe a simple restart helps!!\nLog onto hw as the user obs and type\nor_scheduler -s\nThis should start the scheduler. Check the log file /home/obs/logs/or_scheduler.log if it starts correctly."
)
send_song_mail.send_mail().send_sms(
receiver=m_conf.wakeup_sms_who,
message=
"The scheduler daemon was stopped for some reason. You got mail!")
def check_memory_hw(self):
if song_monitor_config.mon_memory_hw == 1:
try:
import psutil
except Exception, e:
print e
print "psutil was not installed..."
return 0
else:
# psutil.virtual_memory():
# vmem(total=8392966144L, available=7489146880L, percent=10.8, used=7220899840L, free=1172066304L, active=2213416960, inactive=4481515520, buffers=375693312L, cached=5941387264)
mem = psutil.virtual_memory()
swap = psutil.swap_memory()
print clock.timename(
), " HW memory usage: Tot: %iMB, Av: %iMB, Us: %iMB, Free: %iMB, Act: %iMB, InAct: %iMB, Buf: %iMB, Cach: %iMB" % (
int(mem.total / 1.0e6), int(mem.available / 1.0e6),
int(mem.used / 1.0e6), int(mem.free / 1.0e6),
int(mem.active / 1.0e6), int(mem.inactive / 1.0e6),
int(mem.buffers / 1.0e6), int(mem.cached / 1.0e6))
print clock.timename(
), " HW SWAP memory usage: Tot: %iMB, Us: %iMB, Free: %iMB" % (
int(swap.total / 1.0e6), int(
swap.used / 1.0e6), int(swap.free / 1.0e6))
if (
swap.free / 1.0e6
) < song_monitor_config.hw_swap_limit and self.notify_swap_delay < time.time(
):
print clock.timename(
), " Swap memory is in use. Swap memory left on hw: %f MB\n" % (
swap.free / 1.0e6)
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="hw out of swap memory!",
message=
"hw is using swap meory and only %fMB swap memory is left.\n\n Maybe restart some of the python modules to free memory!!\n\n Please be aware!\n\nSend at: %s\n\n"
% ((swap.free / 1.0e6), clock.obstimeUT()))
#value = send_song_mail.send_mail().send_sms(receiver=m_conf.wakeup_sms_who, message="hw is using up the swap memory. Please be aware!!!")
self.notify_swap_delay = time.time() + (
24. * 60. * 60.) # Add a delay of 1 day
elif (
mem.free / 1.0e6
) < song_monitor_config.hw_mem_limit and self.notify_mem_delay < time.time(
):
print clock.timename(
), " There is not much memory left on hw: %f MB\n" % (
mem.free / 1.0e6)
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="hw out of memory!",
message=
"hw has %fMB memory left.\n\n Maybe restart some of the python modules to free memory!!\n\n Please be aware!\n\nSend at: %s\n\n"
% ((mem.free / 1.0e6), clock.obstimeUT()))
self.notify_mem_delay = time.time() + (
24. * 60. * 60.) # Add a delay of 1 day
return 1
or time_stamp["ins_at"] + datetime.timedelta(seconds=300) <
datetime.datetime.utcnow()):
# if float(mean_value) >= float(song_checker_config.spec_box_temp_max) and time.time() > self.delay_time:
if self.verbose == 'yes':
print clock.timename(
), " The temperature inside the spectrograph box was %s and an email was sendt out" % (
str(mean_value))
if self.mon_house_hold_value == 1:
# If the temperature is higher than the allowed limit an email will be sendt out.
if song_monitor_config.send_notifications == "yes":
if time_stamp["ins_at"] + datetime.timedelta(
seconds=300) < datetime.datetime.utcnow():
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Temperatures too old!",
message=
"The temperatures (from our tmon sensor) in the database are no longer up to date.\nLast datapoint was inserted at %s.\nPlease do something!!!\n\nSend at: %s\n\n"
% (str(time_stamp), clock.obstimeUT()))
elif temp_values["temperature_11"] > float(
song_checker_config.spec_box_temp_max):
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Spec box temp too high!",
message=
"The temperature inside the spectrograph box was %s.\nPlease do something!!!\n\nSend at: %s\n\n"
% (str(temp_values["temperature_11"]),
clock.obstimeUT()))
elif temp_values["temperature_11"] < float(
class Check_House_Hold(object):
"""
@brief: This class handles all checks on the time of day.
"""
def __init__(self):
"""
Initialization of the time checks.
"""
self.mon_house_hold_value = song_monitor_config.mon_house_hold_actions # 1 = do things, 0 = do nothing.
self.verbose = song_monitor_config.verbose
self.checker = song_checker.Checker()
self.delay_time = time.time()
self.specbox_delay_time = time.time()
self.email_slony = 0
self.email_ds = 0
self.email_ds_alarm = 0
self.email_ds_scratch = 0
self.email_ds_scratch_alarm = 0
self.slony_delay = time.time()
self.notify_swap_delay = 0
self.notify_mem_delay = 0
self.notify_test_i2_heater_delay = 0
self.notify_i2_heater_delay = 0
def check_container(self):
"""
Checks the temperature inside the container.
"""
try:
temp_values = read_value_db.get_fields_with_id(
"house_hold", [
"temperature_9", "temperature_10", "temperature_13",
"temperature_14"
], "box_id", "2")
time_stamp = read_value_db.get_fields_with_id(
"house_hold", ["ins_at"], "box_id", "2")
except Exception, e:
print clock.timename(
), " Could not connect to the database to read container temperatures"
return 0
temp_arr = []
for temp in temp_values.values():
if float(temp) != float(1.0):
temp_arr.append(float(temp))
### Calculate the mean temperature of the four messured inside the container:
mean_value = numpy.mean(temp_arr)
if self.verbose == 'yes' and float(mean_value) >= float(
song_checker_config.container_temp):
print clock.timename(
), " The temperature inside the container was:", mean_value
if float(mean_value) >= float(song_checker_config.container_temp
) and time.time() > self.delay_time:
if self.verbose == 'yes':
print clock.timename(
), " The temperature inside the container was %s and an email was sendt out" % (
str(mean_value))
# If the temperature is higher than the allowed limit an email will be sendt out.
if self.mon_house_hold_value == 1:
if song_monitor_config.send_sms == "yes":
value = send_song_mail.send_mail().send_sms(
receiver=m_conf.wakeup_sms_who,
message=
"The temperature inside the container is higher than %s. Please do something!!!"
% str(song_checker_config.container_temp))
if song_monitor_config.send_to_support == "yes":
send_song_mail.send_mail().sending_an_email(
reciever=['support'],
sender="SONG_MS",
subject="Container temp too high!",
message=
"The temperature inside the container is higheder than %s.\nPlease contact the IAC night operator to tell him to go to the container. Maybe the air condition is failing.!!!\n\nSend at: %s\n\n"
% (str(
song_checker_config.dome_temp), clock.obstimeUT()))
if song_monitor_config.send_notifications == "yes":
value = send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="Container temp too high!",
message=
"The temperature inside the container is higheder than %s.\nPlease contact the IAC night operator to tell him to go to the container. Maybe the air condition is failing.!!!\n\nSend at: %s\n\n"
% (str(
song_checker_config.dome_temp), clock.obstimeUT()))
# Send message to error daemon
#ticket_container = error_notifier.serror(20,"Monitor: Container temperature is %s" % str(mean_value))
######################
# The monitor should stop the telescope, park it and close the dome and close the side ports.
######################
self.delay_time = time.time(
) + 7200 # this is so that the function will not sendt out an email every 10 second.
return 1
def check_slony(self):
"""
@brief: This function will check if data are copied to the central database.
"""
if song_monitor_config.mon_slony == 1:
slony_val = ""
time_stamp = 0
try:
slony_val, time_stamp = self.checker.check_slony()
except Exception, e:
print clock.timename(), " Could not check slony"
print clock.timename(), e
if slony_val == 0:
print clock.timename(), " Slony is working fine"
if self.email_slony == 0 and slony_val != 0:
if slony_val == 1:
value = send_song_mail.send_mail().sending_an_email(
reciever=["mads", "frank", "eric"],
sender="SONG_MS",
subject="Slony is down!",
message=
"Slony is not replicating the ORs from central to tenerife!!!Last weather data was inserted at: %s\n\nSend at: %s\n\n"
% (time_stamp, clock.obstimeUT()))
print clock.timename(
), " Slony was not replicating the ORs from central to tenerife"
elif slony_val == 2:
value = send_song_mail.send_mail().sending_an_email(
reciever=["mads", "frank", "eric"],
sender="SONG_MS",
subject="Slony is down!",
message=
"Slony is not replicating data from tenerife to central!!!\nLast weather data was inserted at: %s\n\nSend at: %s\n\n"
% (time_stamp, clock.obstimeUT()))
print clock.timename(
), " Slony was not replicating data from tenerife to central"
elif slony_val == 3:
value = send_song_mail.send_mail().sending_an_email(
reciever=["mads", "frank", "eric"],
sender="SONG_MS",
subject="Slony is down!",
message=
"Slony is not replicating at all!!!\n\nSend at: %s\n\n"
% clock.obstimeUT())
print clock.timename(), " Slony was not replicating at all"
elif slony_val == 5:
value = send_song_mail.send_mail().sending_an_email(
reciever=["mads", "frank", "eric"],
sender="SONG_MS",
subject="Database is down!",
message=
"Connection to the central database could not be achieved!!!\n\nSend at: %s\n\n"
% clock.obstimeUT())
print clock.timename(
), " Connection to the central database was not achieved"
elif slony_val == 4:
value = send_song_mail.send_mail().sending_an_email(
reciever=["mads", "frank", "eric"],
sender="SONG_MS",
subject="Database is down!",
message=
"Connection to the tenerife database could not be achieved!!!\n\nSend at: %s\n\n"
% clock.obstimeUT())
print clock.timename(
), " Connection to the tenerife database was not achieved"
self.email_slony = 1
self.slony_delay = time.time()
elif self.email_slony == 1 and time.time(
) >= self.slony_delay + 86400: # Only allow one email per day.
self.email_slony = 0
self.slony_delay = 0
if slony_val == 5:
print clock.timename(
), " Connection to the central database was not achieved"
elif slony_val == 4:
print clock.timename(
), " Connection to the tenerife database was not achieved"
return 1
days=1
) #a date string. Format should be YYYMMDD and it should be the date the observations started!!!
ret_val = ""
try:
print clock.timename(
), "Calling SONGwriter to start the extraction of the spectre of the night: ", obs_night_date.strftime(
"%Y%m%d")
ret_val = songwriter_xmlrpcclient.all_science(
obs_night_date.strftime("%Y%m%d"))
# ret_val = songwriter_MFA_client.all_science(obs_night_date.strftime("%Y%m%d"))
except Exception, e:
print clock.timename(), e
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="SONGwriter problem!",
message=
"The request to start reducing the data from last night failed...\n\nSend at: %s\n\n"
% clock.obstimeUT())
if ret_val != 1:
send_song_mail.send_mail().sending_an_email(
reciever=m_conf.notify_email_who,
sender="SONG_MS",
subject="SONGwriter problem!",
message=
"The request to start reducing the data from last night failed...\n\nSend at: %s\n\n"
% clock.obstimeUT())
##########################################################################################
##########################################################################################
if song_monitor_config.mon_side_ports == 1: # If the config file specifies it side port checks will be carried out. try: song_checker_handle.check_side_ports() except Exception,e: print clock.timename(), " Problem in check_side_ports()" print clock.timename(), e if song_monitor_config.mon_telescope == 1: # If the config file specifies it telescope checks will be carried out. try: song_checker_handle.check_telescope() except Exception,e: print clock.timename(), e print clock.timename(), " Problem in check_telescope()" if song_monitor_config.send_notifications == "yes" and self.telescope_hangs_sms == 0: send_song_mail.send_mail().sending_an_email(reciever=m_conf.notify_email_who,sender="SONG_MS",subject="Telescope Error!",message="The monitor could not connect to the telescope!\n\nPlease check if something is wrong. Check if the TSI is running on the astelco pc. Check if the tcs_daemon is running on hw. \n\nSend at: %s\n\n" % (clock.obstimeUT())) if song_monitor_config.send_sms == "yes": print clock.timename(), " Sending an sms to: %s " % (m_conf.wakeup_sms_who) send_song_mail.send_mail().send_sms(receiver=m_conf.wakeup_sms_who, message="The monitor could not connect to the telescope!") if song_monitor_config.send_to_support == "yes": send_song_mail.send_mail().sending_an_email(reciever=['support'],sender="SONG_MS",subject="Telescope Error!",message="The monitor could not connect to the telescope!\n\nPlease check if something is wrong. Check if the TSI is running on the astelco pc. Check if the tcs_daemon is running on hw. \n\nSend at: %s\n\n" % (clock.obstimeUT())) self.telescope_hangs_sms = 1 if song_monitor_config.mon_network == 1: # If the config file specifies it netowrk checks will be carried out. try: song_checker_handle.check_network() except Exception,e: print clock.timename(), " Problem in check_network()" print clock.timename(), e
