def update_main_html():
[tyear, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime("Local")
today = str(mon) + '/' + str(day) + '/ ' +str(tyear)
syear = tyear
smon = mon -1
if smon < 1:
smon = 12
syear = tyear -1
lyear = str(syear)
lmon = str(smon)
if smon < 10:
lmon = '0' + lmon
line = lmon + '_' + lyear
file = hosue_keeping + 'template'
data = open(file, 'r').read()
data = data.replace('#LATEST#', line)
data = data.replace('#DATE#', today)
file = web_dir + 'exposure.html'
fo = open(file, 'w')
fo.write(data)
fo.close()
def plot_steps(time, set1, set2):
"""
plot tsc and fa movement
input: time --- a list of time in fractional year
set1 --- a list of tsc value
set2 --- a list of fa value
output: monthly_sim.png
"""
#
#--- set plottting range
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
xmin = 2000
xmax = year + 1
if mon > 6:
xmax += 1
ymin = 0
ymax = max(set1)
ymax1 = int(1.1 * ymax) + 10
ymax = max(set2)
ymax2 = int(1.1 * ymax) + 10
#
#-- set a few parameters
#
fsize = 9
lsize = 0
color = 'red'
marker = 'o'
msize = 3
plt.close("all")
mpl.rcParams['font.size'] = fsize
props = font_manager.FontProperties(size=fsize)
#
#--- TSC plot
#
a1 = plt.subplot(121)
plot_sub(a1, time, set1, xmin, xmax, ymin, ymax1, color, lsize, marker, msize, tline='TSC')
a1.set_xlabel('Time (year)', size=fsize)
a1.set_ylabel('TSC Cummulative Moter Dist (x10^4 mm)', size=fsize)
#
#-- FA plot
#
a2 = plt.subplot(122)
plot_sub(a2, time, set2, xmin, xmax, ymin, ymax2, color, lsize, marker, msize, tline='FA')
a2.set_xlabel('Time (year)', size=fsize)
a2.set_ylabel('FA Cummulative Moter Dist (mm)', size=fsize)
#
#--- save the plot
#
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(10.0, 5.0)
outname = 'monthly_sim.png'
plt.savefig(outname, format='png', dpi=100)
def print_month_list():
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
line = 'Sep99 41 71\n'
begin = 71
for iyear in range(1999, year+1):
if tcnv.isLeapYear(iyear) == 1:
month_list = month_list2
else:
month_list = month_list1
lyear = str(iyear)
syear = lyear[2] + lyear[3]
for month in range(1, 13):
if iyear == 1999 and month < 10:
continue
if iyear == year and month > mon:
break
lmon = tcnv.changeMonthFormat(month)
time = lmon + syear
daynum = month_list[month-1]
end = daynum + begin
line = line + time + ' ' + str(begin) + ' ' + str(end) + '\n'
begin = end
fo = open('/data/mta_www/mta_sim/Scripts/SIM_move/Outputs/months', 'w')
fo.write(line)
fo.close()
def find_data_collection_interval():
tlist = tcnv.currentTime(format='UTC')
tyear = tlist[0]
tyday = tlist[7]
tdom = tcnv.YdateToDOM(tyear, tyday)
file = data_dir + 'Disp_dir/hist_ccd3'
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
chk = 0
k = 1
while(chk == 0):
atemp = re.split('<>', data[len(data)-k])
ldom = atemp[0]
if mcf.chkNumeric(ldom) == True:
ldom = int(ldom)
chk = 1
break
else:
k += 1
ldom += 1
return(ldom, tdom)
def acis_dose_control(year = 'NA', month = 'NA', comp_test = 'NA'):
"""
monthly acis dose update control script
input: optional year and month
"""
if year == 'NA' or month == 'NA':
(year, mon, day, hours, min, sec, weekday, yday, dst) = tcnv.currentTime('Local')
month = mon -1
if month < 1:
month = 12
year -= 1
else:
year = int(year)
month = int(month)
syear = str(year)
smon = str(month)
if month < 10:
smon = '0' + smon
#
#--- extract data
#
getd.acis_dose_get_data(year, month, year, month)
def add_date_on_html():
"""
updating the modified date on three html files
Input: None
Outpu: three htmla pages updated
"""
current = tcnv.currentTime(format='Display')
if level == 1:
top_level = '/'
else:
top_level = '/Lev2/'
html_file = 'long_term.html'
plot_out = top_level + 'Plots/Plot_long_term/'
change_date(current, html_file, plot_out)
html_file = 'past_one_year.html'
plot_out = top_level + 'Plots/Plot_past_year/'
change_date(current, html_file, plot_out)
html_file = 'quarter.html'
plot_out = top_level + 'Plots/Plot_quarter/'
change_date(current, html_file, plot_out)
def hrc_create_dose_map(year = 'NA', mon = 'NA'):
'extract data, create a center part map/statistics, driving script. if year and month are given, computer for that year/month. Otherwise one month before the current month'
#
#--- if year and mon are not provided, check a month before the current month
#
if year == 'NA':
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('Local')
lyear = year
lmon = mon -1
if lmon < 1:
lmon = 12
lyear -= 1
else:
lyear = year
lmon = mon
#
#--- ectract data
#
# getdata.hrc_dose_get_data(lyear, lmon, lyear, lmon)
#
#--- compute stat
#
# hstat.hrc_dose_extract_stat_data_month(lyear, lmon, cum_dir, mon_dir, data_out)
#
#--- plot data
#
eplot.hrc_dose_plot_exposure_stat(data_out, plot_dir)
def set_min_max(xdata, ydata, xtime = 0, ybot = -999):
"""
set plotting range
Input: xdata ---- xdata
ydata ---- ydata
xtime ---- if it is >0, it set the plotting range from 1999 to the current in year
ybot ---- if it is == 0, the ymin will be 0, if the ymin computed is smaller than 0
Output: [xmin, xmax, ymin, ymax]
"""
xmin = min(xdata)
xmax = max(xdata)
xdiff = xmax - xmin
xmin -= 0.1 * xdiff
xmax += 0.2 * xdiff
if xtime > 0:
xmin = 1999
tlist = tcnv.currentTime()
xmax = tlist[0] + 1
ymin = min(ydata)
ymax = max(ydata)
ydiff = ymax - ymin
ymin -= 0.1 * ydiff
ymax += 0.2 * ydiff
if ybot == 0:
if ymin < 0:
ymin = 0
return [xmin, xmax, ymin, ymax]
def plot_cum_grating(time, h_in, l_in):
"""
plot cummulative count rates of hetig and letig insertion.
input: time --- fractional year
h_in --- hetig insertion cummulative count rate (month step)
l_in --- letig insertion cummulative count rate
output: monthly_grat.prn
"""
#
#--- set x axis plotting range
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
xmin = 2000
xmax = year + 1
if mon > 6:
xmax += 1
#
#--- set y axis plotting range
#
ymin = 0.0
ymax = max(h_in)
ymax2 = max(l_in)
if ymax2 > ymax:
ymax = ymax2
ymax = int(1.1 * ymax) + 10
#
#--- set a few parameters
#
fsize = 9
lsize = 0
color = 'red'
marker = 'o'
msize = 3
plt.close("all")
mpl.rcParams['font.size'] = fsize
props = font_manager.FontProperties(size=fsize)
plt.subplots_adjust(hspace=0.08, wspace=0.10)
#
#--- HETG Cumulative Count Plots
#
a1 = plt.subplot(121) #--- two panel plot: left
plot_sub(a1, time, h_in, xmin, xmax, ymin, ymax, color, lsize, marker, msize, tline='HETG')
a1.set_xlabel('Time (year)', size=fsize)
a1.set_ylabel('Cumulative Insertion Counts', size=fsize)
#
#--- LETG Cumulative Count Plots
#
a1 = plt.subplot(122) #--- two panel plot: right
plot_sub(a1, time, l_in, xmin, xmax, ymin, ymax, color, lsize, marker, msize, tline='LETG')
a1.set_xlabel('Time (year)', size=fsize)
#
#--- save the plot
#
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(10.0, 5.0)
outname = 'monthly_grat.png'
plt.savefig(outname, format='png', dpi=100)
def update_html():
"""
create/update error html page for a gvien machine and a user
input: error_list_<cpu>_<user>, wheren cpu and user are found from the machine running this and account using.
output: cron_error_<cpu>_<usr>.html in html_dir.
"""
#
#--- find the current time
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('Local')
syear = str(year)
smon = str(mon)
lmon = tcnv.changeMonthFormat(mon)
if mon < 10:
smon = '0' + smon
#
#--- set the file name and a hmtl page name
#
file = error_log_dir + machine + '_' + user + '_' + smon + '_' + syear
html = html_dir + 'cron_error_' + machine + '_' + user + '_' + smon + '_' + syear + '.html'
#
#--- start writing the html page
#
out = open(html, 'w')
line = '<!DOCTYPE html>\n'
line = line + '<html>\n'
line = line + '<head>\n'
line = line + '<title>Cron Error Log for ' + user.upper() + ' on ' + machine.upper() + ': ' + lmon + ' ' + syear + '</title>\n'
line = line + '<link rel="stylesheet" type="text/css" href="/mta/REPORTS/Template/mta_style_short.css" />\n'
line = line + '</head>\n'
line = line + '<body>\n'
line = line + '<h3 style="padding-bottom: 10px"> Cron Error Log for ' + user.upper() + ' on ' + machine.upper() + ': ' + lmon + ' ' + syear + '</h3>\n\n'
line = line + '<hr />\n'
line = line + '<pre style="padding-left: 5px;padding-bottom:10px">\n'
out.write(line)
#
#--- write the content of error_list
#
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
for ent in data:
out.write(ent)
out.write('\n')
line = '</pre>\n'
line = line + '<br /><hr /><br />\n'
line = line + 'Back to <a href="https://cxc.cfa.harvard.edu/mta_days/mta_cron/cron_error_main.html">Top Page</a>\n'
line = line + '\n</body>\n'
line = line + '</html>\n'
out.write(line)
out.close()
def create_this_montcreate_this_month():
"""
create ACA monthly script for the month
input: none, but read from ./Template
outpu: a file named .update_this_month
"""
tlist = tcnv.currentTime()
year = str(tlist[0])
syr = year[2] + year[3]
mon = int(float(tlist[1]))
smon = str(tlist[1])
if mon < 10:
smon = '0' + smon
lmon = tcnv.changeMonthFormat(mon)
#
#--- format of: 2015_05
#
yearmon = year + '_' + smon
#--- format of: MAY15
#
umonyr = lmon + syr
umonyr = umonyr.upper()
#
#--- format of: may15
#
lmonyr = umonyr.lower()
file = dir_path + 'Template/update_temp'
f = open(file, 'r')
data = f.read()
f.close()
data = data.replace('#YEAR_MON#', yearmon)
data = data.replace('#UMONYR#', umonyr)
data = data.replace('#LMONYR#', lmonyr)
ofile = out_path + 'update_this_month'
fo = open(ofile, 'w')
fo.write(data)
fo.close()
cmd = 'chmod 755 ' + ofile
os.system(cmd)
#
#--- make output directory
#
cmd = 'mkdir ' + out_path + '/' + umonyr
os.system(cmd)
cmd = 'mkdir ' + out_path + '/' + umonyr + '/Report'
os.system(cmd)
cmd = 'mkdir ' + out_path + '/' + umonyr + '/Data'
os.system(cmd)
def move_tl_files():
"""
manage # of trace files in the directories
the file is kept in /data/mta/Script/Dumps/ for 3 days
and also gizpped files are save in two different directory
after 3 days, the files are removed from the main directory, but
other copies are kept another 6 days.
input: none but read from directories
output: gzipped files in TLfiles/Dumps_mon/IN directories
"""
#
#--- set directory paths
#
mdir = '/data/mta/Script/Dumps/'
tdir = '/data/mta/Script/Dumps/TLfiles/'
ddir = '/data/mta/Script/Dumps/Dumps_mon/IN/'
sdir = ddir + '/Done/'
#
#--- find today's date in seconds from 1998.1.1
#
today = int(tcnv.currentTime(format='SEC1998'))
#
#--- set boundaries at 3 days ago and 6 days ago
#
day3ago = today - 1 * 86400
day6ago = today - 3 * 86400
#
#--- find trace log file older than 3 days in the main direcotry, remove them
#
flist = get_file_list(mdir)
remove_older_files(flist, day3ago)
#
#--- remove trace logs older than 6 days ago from TLfiles directory
#
flist = get_file_list(tdir)
remove_older_files(flist, day6ago)
#
#--- remove trace logs older than 6 days ago from Dumps_mon/IN/Done directory
#
flist = get_file_list(sdir)
remove_older_files(flist, day6ago)
#
#--- now copy new files to appropriate directory
#
find_new_files(mdir, ddir, '*CCDM*')
find_new_files(mdir, ddir, '*PCAD*')
find_new_files(mdir, ddir, '*ACIS*')
find_new_files(mdir, ddir, '*IRU*')
find_new_files(mdir, ddir, '*MUPS2*')
find_new_files(mdir, tdir, '*EPHIN*', fzip=1)
find_new_files(mdir, tdir, '*ELBILOW*', fzip=1)
find_new_files(mdir, tdir, '*MUPS*', fzip=1)
def plot_dusk_result(diskName, duskName, nameList, pastData):
#
#--- find the disk capacity of the given disk
#
disk_capacity = diskCapacity(diskName)
#
#--- read the output from dusk
#
line = run_dir + duskName
f = open(line, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
capacity = {} #---- make a dictionary
for ent in data:
atemp = re.split('\s+|\t+', ent)
try:
val = 100.0 * float(atemp[0]) /disk_capacity
val = round(val, 2)
capacity[atemp[1]] = val
except:
pass
#
#--- today's date
#
today = tcnv.currentTime('local')
year = today[0]
month = today[1]
day = today[2]
hours = today[3]
minutes = today[4]
seconds = today[5]
#
#--- convert to dom
#
dom = tcnv.findDOM(year, month, day, hours, minutes,seconds)
dom = round(dom, 2)
#
#--- append the new data to the data table
#
f = open(pastData, 'a')
f.write(str(dom))
for dName in nameList:
f.write(':')
string = str(capacity[dName])
f.write(string)
f.write('\n')
f.close()
#
#---- start plotting history data
#
try:
plot_history_trend(diskName, duskName, nameList, pastData)
except:
pass
def run_all_sim_script():
grat.plot_grat_movement()
sim.plot_sim_movement()
[year, month, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
month -= 1
if month < 1:
month = 12
year -= 1
aor.plot_aorwspd(year, month)
def find_time_interval(test='NO', month=4, year=2015):
"""
create start and stop time and output directory name
input: test --- if it is test, yes. default: 'NO'
month --- test month input. default: 4. it is ignored if not test.
year --- test year input. default: 2015. it is ignored if not test.
output: start --- start time in the format of mm/01/yy (e.g. 03/01/15)
stop --- stop time
dir --- output directory name in the format of <Mon><yy> (e.g. Jan15)
"""
#
#--- for the case if you want to run the test, there is option...
#
if test == 'NO':
#
#--- this function gives you the current time. year in the format of 2015, and month in 4
#
[year, month, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
#
#--- we need start and stop times in the formats of 03/01/15, 04/01/15
#
year_start = year
year_end = year
this_month = str(month)
if month < 10:
this_month = '0' + this_month
temp_month = month -1
if temp_month < 1:
temp_month = 12
year_start -= 1
last_month = str(temp_month)
if temp_month < 10:
last_month = '0' + last_month
temp = str(year_start)
year_start = temp[2] + temp[3]
temp = str(year_end)
year_end = temp[2] + temp[3]
start = last_month + '/01/' + year_start
stop = this_month + '/01/' + year_end
#
#--- create the output directory name (format example: Apr15)
#
cmonth = tcnv.changeMonthFormat(temp_month)
dir = cmonth + year_start
return (start, stop, dir)
def find_available_deph_data():
"""
create a list of potential new data file name
input: none, but read from /dsops/GOT/aux/DEPH.dir/
output: cdata --- a list of the data file names
"""
#
#--- find current time
#
ttemp = tcnv.currentTime()
year = int(ttemp[0])
ydate = int(ttemp[7])
syear = str(year)
tyear = syear[2] + syear[3]
#
#--- first 20 days of the year, we also check the last year input data
#
if ydate < 20:
lyear = year -1
slyear = str(lyear)
ltyear = slyear[2] + slyear[3]
cmd = 'ls /dsops/GOT/aux/DEPH.dir/DE' + ltyear + '*.EPH > ' + zspace
os.system(cmd)
cmd = 'ls /dsops/GOT/aux/DEPH.dir/DE' + tyear + '*.EPH >> ' + zspace
try:
os.system(cmd)
except:
pass
else:
cmd = 'ls /dsops/GOT/aux/DEPH.dir/DE' + tyear + '*.EPH > ' + zspace
try:
os.system(cmd)
except:
pass
try:
f = open(zspace, 'r')
cdata = [line.strip() for line in f.readlines()]
f.close()
except:
cdata = []
mcf.rm_file(zspace)
return cdata
def make_data_list(year, mon, day, tail):
"""
make a data list from the last entry date to the most current data
input: year --- the year of the last entry
mon --- the month of the last entry
day --- the day of month of the last entry
tail --- the suffix of the data file
output dlist --- a list of data names
"""
#
#--- convert the date into seconds from 1998.1.1
#
dst = tcnv.convertDateToTime2(year, mon, day)
#
#--- find today's time
#
today = tcnv.currentTime()
cyear = today[0]
cmon = today[1]
cday = today[2]
cdst = tcnv.convertDateToTime2(cyear, cmon, cday)
dlist = []
#
#--- check the current date is larger than the date indicated
#--- if so find how many days between and retrieve data for each day
#
if cdst > dst:
step = int(( cdst - dst)/86400)
if step >= 1:
head = make_header(year, mon, day)
name = head + tail
dlist.append(name)
for i in range(2, step):
sdate = int(cdst - 86400.0 * i)
out = tcnv.axTimeMTA(sdate)
atemp = re.split(':', out)
year = int(atemp[0])
ydate = int(atemp[1])
[mon, day] = tcnv.changeYdateToMonDate(year, ydate)
head = make_header(year, mon, day)
name = head + tail
dlist.append(name)
return dlist
def update_plt_html_date(comp_test = 'NA'):
"""
update html pages for plots; just replacing date
no input, but get the list from plot_dir
"""
(lyear, lmon, lday, lhours, lmin, lsec, lweekday, lyday, dst) = tcnv.currentTime('Local')
date = 'Last Update: ' + str(lmon) + '/' + str(lday) + '/' + str(lyear)
if comp_test == 'test':
cmd = 'ls ' + test_plot_dir + '/*html>./ztemp'
else:
cmd = 'ls ' + plot_dir + '/*html>./ztemp'
os.system(cmd)
f = open('./ztemp', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
os.system('rm ./ztemp')
for ent in data:
f = open(ent, 'r')
hdat = [line.strip() for line in f.readlines()]
f.close()
f = open('./temp', 'w')
for oline in hdat:
m = re.search('Last Update', oline)
if m is not None:
f.write(date)
f.write('\n')
else:
f.write(oline)
f.write('\n')
f.close()
cmd = 'mv ./temp ' + ent
os.system(cmd)
def set_min_max(ydata):
"""
set plotting range
Input: ydata ---- ydata
Output: [xmin, xmax, ymin, ymax]
"""
xmin = 1999
tlist = tcnv.currentTime()
xmax = tlist[0] + 1
ymin = min(ydata)
ymax = max(ydata)
ydiff = ymax - ymin
ymin -= 0.1 * ydiff
ymax += 0.2 * ydiff
if ymin < 0:
ymin = 0
return [xmin, xmax, ymin, ymax]
def ccd_comb_plot(choice):
"""
a control script to create plots
Input: choice --- if check, you will be asked to provide year and month
all oter cases, the script will use the previous month
Output: png formated plotting files
"""
#
#--- find today's date, and set a few thing needed to set output directory and file name
#
if choice == 'check':
year = raw_input("Year: ")
mon = raw_input("Month: ")
year = int(float(year))
mon = int(float(mon))
else:
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
syear = str(year)
smonth = str(mon)
if mon < 10:
smonth = '0'+ smonth
lyear = year
lmon = mon - 1
if lmon < 1:
lmon = 12
lyear -= 1
slyear = str(lyear)
slmonth = str(lmon)
if lmon < 10:
slmonth = '0' + slmonth
#
#--- monthly plot
#
dlist = collect_data_file_names('month')
plot_out = './'
header = 'month_plot_ccd'
plot_data(dlist, plot_out, header, yr=slyear, mo=slmonth, psize=2.5, xunit='ydate')
def update_bias_html():
"""
pdate bias_home.html page
Input: None but read from:
<house_keeping>/bias_home.html
Output: <web_dir>/bias_home.html
"""
#
#--- find today's date
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
lmon = str(mon)
if mon < 10:
lmon = '0' + lmon
lday = str(day)
if day < 10:
lday = '0' + lday
#
#--- line to replace
#
newdate = "Last Upate: " + lmon + '/' + lday + '/' + str(year)
#
#--- read the template
#
line = house_keeping + 'bias_home.html'
data = mcf.readFile(line)
#
#--- print out
#
outfile = web_dir + 'bias_home.html'
fo = open(outfile, 'w')
for ent in data:
m = re.search('Last Update', ent)
if m is not None:
fo.write(newdate)
else:
fo.write(ent)
fo.write('\n')
fo.close()
def update_plt_html_date(comp_test='NA'):
"""
update html pages for plots; just replacing date
no input, but get the list from plot_dir
"""
(lyear, lmon, lday, lhours, lmin, lsec, lweekday, lyday,
dst) = tcnv.currentTime('Local')
date = 'Last Update: ' + str(lmon) + '/' + str(lday) + '/' + str(lyear)
if comp_test == 'test':
cmd = 'ls ' + test_plot_dir + '/*html>./ztemp'
else:
cmd = 'ls ' + plot_dir + '/*html>./ztemp'
os.system(cmd)
f = open('./ztemp', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
os.system('rm ./ztemp')
for ent in data:
f = open(ent, 'r')
hdat = [line.strip() for line in f.readlines()]
f.close()
f = open('./temp', 'w')
for oline in hdat:
m = re.search('Last Update', oline)
if m is not None:
f.write(date)
f.write('\n')
else:
f.write(oline)
f.write('\n')
f.close()
cmd = 'mv ./temp ' + ent
os.system(cmd)
def set_min_max(ydata):
"""
set plotting range
Input: ydata ---- ydata
Output: [xmin, xmax, ymin, ymax]
"""
xmin = 1999
tlist = tcnv.currentTime()
xmax = tlist[0] + 1
ymin = min(ydata)
ymax = max(ydata)
ydiff = ymax - ymin
ymin -= 0.1 * ydiff
ymax += 0.2 * ydiff
if ymin < 0:
ymin = 0
return [xmin, xmax, ymin, ymax]
def dose_map_creation(year='', mon=''):
"""
create acis and hrc dose maps
input: year --- year; if it is "", the year of the last month will be used
mon --- month;if it is "", the month value of the last month is used
output: /data/mta/www/mta_max_exp/Images/*.png, e.g., ACSI_11_2016_i2.png
"""
#
#--- if year and month are not given, use the last month's month and year value
#
if year == '':
#
#--- find today's date
#
ltime = tcnv.currentTime()
year = ltime[0]
lyear = str(year) #--- '2016'
#
#--- set the last month's month and year
#
mon = ltime[1] -1
if mon < 1:
mon = 12
year -= 1
cmon = str(mon)
if mon < 10:
cmon = '0' + cmon #--- e.g. 03 or 11
lmon = tcnv.changeMonthFormat(mon) #--- e.g. Mar or Nov
lmonyr = lmon.lower() + lyear[2] + lyear[3] #--- e.g. jan16
lm_y = cmon + '_' + lyear #--- e.g. 03_2016
#
#--- create clean acis and hrc exposure maps using ds9
#
run_exposure_maps(lyear, cmon)
def dose_map_creation(year='', mon=''):
"""
create acis and hrc dose maps
input: year --- year; if it is "", the year of the last month will be used
mon --- month;if it is "", the month value of the last month is used
output: /data/mta/www/mta_max_exp/Images/*.png, e.g., ACSI_11_2016_i2.png
"""
#
#--- if year and month are not given, use the last month's month and year value
#
if year == '':
#
#--- find today's date
#
ltime = tcnv.currentTime()
year = ltime[0]
lyear = str(year) #--- '2016'
#
#--- set the last month's month and year
#
mon = ltime[1] - 1
if mon < 1:
mon = 12
year -= 1
cmon = str(mon)
if mon < 10:
cmon = '0' + cmon #--- e.g. 03 or 11
lmon = tcnv.changeMonthFormat(mon) #--- e.g. Mar or Nov
lmonyr = lmon.lower() + lyear[2] + lyear[3] #--- e.g. jan16
lm_y = cmon + '_' + lyear #--- e.g. 03_2016
#
#--- create clean acis and hrc exposure maps using ds9
#
run_exposure_maps(lyear, cmon)
def add_html_page(ptype, plot_out, yr, mo):
"""
update/add html page to Plot directory
Input: ptype --- indiecator of which html page to be updated
plot_out --- a directory where the html page is updated/created
yr --- a year of the file
mo --- a month of the file
Output: either month.html or year.hmtl in an appropriate directory
"""
current = tcnv.currentTime(format='Display')
lmon = ''
if ptype == 'month':
ofile = plot_out + 'month.html'
lmon = tcnv.changeMonthFormat(int(mo))
if level == 2:
file = house_keeping + 'month2.html'
else:
file = house_keeping + 'month.html'
elif ptype == 'year':
ofile = plot_out + 'year.html'
if level == 2:
file = house_keeping + 'year2.html'
else:
file = house_keeping + 'year.html'
text = open(file, 'r').read()
text = text.replace('#YEAR#', yr)
text = text.replace('#MONTH#', lmon)
text = text.replace('#DATE#', current)
f = open(ofile, 'w')
f.write(text)
f.close()
def add_date_on_html():
"""
updating the modified date on three html files
Input: None
Outpu: three htmla pages updated
"""
current = tcnv.currentTime(format='Display')
if level == 1:
top_level = '/'
else:
top_level = '/Lev2/'
html_file = 'long_term.html'
plot_out = top_level + 'Plots/Plot_long_term/'
change_date(current, html_file, plot_out)
html_file = 'past_one_year.html'
plot_out = top_level + 'Plots/Plot_past_year/'
change_date(current, html_file, plot_out)
html_file = 'quarter.html'
plot_out = top_level + 'Plots/Plot_quarter/'
change_date(current, html_file, plot_out)
def update_datatable(per0, per1, per2, per3, per4, per5, per6, per7, per8):
"""
this function appends the newest data to the disk space data table
Input: per0 ... per5: new measures for each disk. currently per3 is empty
Output: <data_out>/disk_space_data (updated)
"""
#
#--- find out today's date in Local time frame
#
today = tcnv.currentTime('local')
year = today[0]
month = today[1]
day = today[2]
hours = today[3]
minutes = today[4]
seconds = today[5]
#
#--- convert to dom
#
dom = tcnv.findDOM(year, month, day, hours, minutes, seconds)
dom = round(dom, 3)
#
#--- today's data
#
line = str(dom) + '\t' + str(per0) + '\t' + str(per1) + '\t' + str(
per2) + '\t' + str(per4) + '\t' + str(per5) + '\t' + str(
per6) + '\t' + str(per7) + '\t' + str(per8) + '\t'
#
#--- append to the data table
#
file = data_out + 'disk_space_data'
f = open(file, 'a')
f.write(line)
f.write("\n")
f.close()
def check_obs_status(obsid):
"""
check obsid is eligible to be added to the curren list.
input: obsid. if it is either 'scheduled', 'unobserved', or 'observed' status and if it is obsverd
less than 30 day ago, it is eliible.
"""
#
#--- set limit to the last 30 days
#
[year, mon, day, hours, min, sec, weekday, yday,
dst] = tcnv.currentTime('LOCAL')
tdom = tcnv.findDOM(year, mon, day, 0, 0, 0)
dom_limit = tdom - 30
#
#--- extract basic information
#
monitor = []
groupid = []
try:
(group_id, pre_id, pre_min_lead, pre_max_lead, grating, type, instrument, obs_ao_str, status, \
seq_nbr, ocat_propid, soe_st_sched_date, lts_lt_plan,targname) = sql.get_target_info(int(obsid), monitor,groupid)
if soe_st_sched_date is not None:
date = soe_st_sched_date
chk = 1
elif lts_lt_plan is not None:
date = lts_lt_plan
chk = 1
else:
date = 'NA'
chk = 0
except:
date = 'NA'
chk = 0
#
#--- check status change
#
if chk == 1:
if status == 'scheduled' or status == 'unobserved' or status == 'observed':
#
#--- recompute with updated date
#
if date == 'NA':
dom = 'NA'
else:
temp = re.split('\s+', str(date))
omon = tcnv.changeMonthFormat(temp[0])
oday = int(temp[1])
oyear = int(temp[2])
dom = tcnv.findDOM(oyear, omon, oday, 0, 0, 0)
if dom > dom_limst:
chk = 1
else:
chk = 0
else:
chk = 0
return chk
def collect_data_file_names(period, syear=2000, smonth=1, eyear=2000, emonth=12):
"""
for a given period, create a list of directory names
Input: period --- indicator of which peirod, "month", "quarter", "year", "lyear", "full", and "check'"
if period == 'check', then you need to give a period in year and month
syear --- year of the starting date
smonth --- month of the starting date
eyear --- year of the ending date
emonth --- month of the ending date
Output data_lst --- a list of the directory names
"""
#
#--- find today's date
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
data_list = []
#
#--- find the last month
#
if period == 'month':
mon -= 1
if mon < 1:
mon = 12
year -= 1
if mon < 10:
cmon = '0' + str(mon)
else:
cmon = str(mon)
dfile = data_dir + 'Data_' + str(year) + '_' + cmon
data_list.append(dfile)
#
#--- find the last three months
#
if period == 'quarter':
for i in range(1, 4):
lyear = year
month = mon -i
if month < 1:
month = 12 + month
lyear = year -1
if month < 10:
cmon = '0' + str(month)
else:
cmon = str(month)
dfile = data_dir + 'Data_' + str(lyear) + '_' + cmon
data_list.append(dfile)
#
#--- find data for the last one year (ending the last month)
#
elif period == 'year':
cnt = 0
if mon > 1:
for i in range(1, mon):
if i < 10:
cmon = '0' + str(i)
else:
cmon = str(i)
dfile = data_dir + 'Data_' + str(year) + '_' + cmon
data_list.append(dfile)
cnt += 1
if cnt < 11:
year -= 1
for i in range(mon, 13):
if i < 10:
cmon = '0' + str(i)
else:
cmon = str(i)
dfile = data_dir + 'Data_' + str(year) + '_' + cmon
data_list.append(dfile)
#
#--- fill the list with the past year's data
#
elif period == 'lyear':
year -= 1
for i in range(1, 13):
if i < 10:
cmon = '0' + str(i)
else:
cmon = str(i)
dfile = data_dir + 'Data_' + str(year) + '_' + cmon
data_list.append(dfile)
#
#--- fill the list with the entire data
#
elif period == 'full':
for iyear in range(2000, year+1):
for i in range (1, 13):
if i < 10:
cmon = '0' + str(i)
else:
cmon = str(i)
dfile = data_dir + 'Data_' + str(iyear) + '_' + cmon
data_list.append(dfile)
#
#--- if the period is given, use them
#
elif period == 'check':
syear = int(syear)
eyear = int(eyear)
smonth = int(smonth)
emonth = int(emonth)
if syear == eyear:
for i in range(smonth, emonth+1):
if i < 10:
cmon = '0' + str(i)
else:
cmon = str(i)
dfile = data_dir + 'Data_' + str(syear) + '_' + cmon
data_list.append(dfile)
elif syear < eyear:
for iyear in range(syear, eyear+1):
if iyear == syear:
for month in range(smonth, 13):
if i < 10:
cmon = '0' + str(i)
else:
cmon = str(i)
dfile = data_dir + 'Data_' + str(iyear) + '_' + cmon
data_list.append(dfile)
elif iyear == eyear:
for month in range(1, emonth+1):
if i < 10:
cmon = '0' + str(i)
else:
cmon = str(i)
dfile = data_dir + 'Data_' + str(iyear) + '_' + cmon
data_list.append(dfile)
else:
for month in range(1, 13):
if i < 10:
cmon = '0' + str(i)
else:
cmon = str(i)
dfile = data_dir + 'Data_' + str(iyear) + '_' + cmon
data_list.append(dfile)
return data_list
def printHtml(indir,outdir, hrc, date, year,month,mean_acc,std_acc,min_acc,min_apos,max_acc,max_apos, asig1, asig2, asig3,mean_dff, \
std_dff,min_dff,min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3):
'create HTML page to display HRC historical data.'
[tyear, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime("Local")
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(day)
if day < 10:
sday = '0' + sday
outdir = outdir + '/' + hrc + '.html'
f = open(outdir, 'w')
#
#--- this is a html 5 document
#
f.write('<!DOCTYPE html>\n')
f.write('<html>\n')
f.write('<head>\n')
f.write("<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />\n")
f.write("<style type='text/css'>\n")
f.write("table{text-align:center;margin-left:auto;margin-right:auto;border-style:solid;border-spacing:8px;border-width:2px;border-collapse:separate}\n")
f.write("a:link {color:#00CCFF;}\n")
f.write("a:visited {color:yellow;}\n")
f.write("td{text-align:center;padding:8px}\n")
f.write("</style>\n")
if hrc == 'hrci':
hname = 'HRC I'
wname = 'HRCI'
else:
hname = 'HRC S'
wname = 'HRCS'
line = '<title>' + hname + ' History Data</title>\n'
f.write(line)
f.write("</head>\n")
f.write('<body style="color:white;background-color:black">\n')
line = '<h2 style="text-align:center">Data: ' + hname + '</h2>\n'
f.write(line)
f.write("<div style='padding-bottom:30px'>\n")
f.write('<table border=1>\n')
f.write('<tr><th> </th><th> </th><th colspan=11>Monlthy</th><th colspan=11>Cumulative</th></tr>\n')
f.write('<tr style="color:yellow"><th>Year</th><th>Month</th>\n')
f.write('<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th><th>Max</th><th>Max Position</th><th>68% Level</th><th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n')
f.write('<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th><th>Max</th><th>Max Position</th><th>68% Level</th><th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n')
for i in range(0, len(date)):
smonth = str(month[i])
if month[i] < 10:
smonth = '0' + smonth
cmonth = tcnv.changeMonthFormat(month[i]) #---- converting digit to letters, i.e. 1 to Jan
#
#--- monthly HRC dose data
#
if mean_dff[i] == 0 and std_dff[i] == 0:
line = '<tr><td>%d</td><td>%d</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td>\n' % (year[i], month[i])
f.write(line)
f.write('<td>No Data</td><td>No Image</td>\n')
else:
line = '<tr><td>%d</td><td>%d</td><td>%4.4f</td><td>%4.4f</td><td>%4.1f</td><td>%s</td><td>%4.1f</td><td>%s</td><td>%4.1f</td><td>%s</td><td>%4.1f</td>\n' \
% (year[i], month[i], mean_dff[i], std_dff[i], min_dff[i],min_dpos[i], max_dff[i], max_dpos[i],dsig1[i], dsig2[i], dsig3[i])
f.write(line)
fname = wname + '_' + smonth + '_' + str(year[i]) + '.fits.gz'
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month_hrc/' + fname +'">fits</a></td>\n'
f.write(line)
fname = wname + '_' + smonth + '_' + str(year[i]) + '.png'
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + fname + '">map</a></td>\n'
f.write(line)
#
#---- cumulative HRC dose data
#
line = '<td>%4.4f</td><td>%4.4f</td><td>%4.1f</td><td>%s</td><td>%4.1f</td><td>%s</td><td>%4.1f</td><td>%s</td><td>%4.1f</td>\n' \
% (mean_acc[i], std_acc[i], min_acc[i], min_apos[i], max_acc[i], max_apos[i], asig1[i], asig2[i], asig3[i])
f.write(line)
fname = wname + '_08_1999_' + smonth + '_' + str(year[i]) + '.fits.gz'
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative_hrc/' + fname +'">fits</a></td>\n'
f.write(line)
fname = wname + '_08_1999_' + smonth + '_' + str(year[i]) + '.png'
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + fname + '">map</a></td>\n\n'
f.write(line)
#
#--- put header every new year so that we can read data easier
#
if month[i] % 12 == 0 and i != (len(date)-1):
f.write('\n<tr style="color:yellow"><th>Year</th><th>Month</th><th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th><th>Max</th><th>Max Position</th><th>68% Level</th><th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n')
f.write('<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th><th>Max</th><th>Max Position</th><th>68% Level</th><th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n\n')
f.write('</table>\n\n')
f.write("</div>\n")
f.write('<hr />\n')
line = '<p style="padding-top:10px;padding-bottom:10px"><strong style="font-size:105%;float:right">Last Update: ' + smon + '/' + sday + '/' + str(tyear) + '</strong></p>\n'
f.write(line)
line = '<p>If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected].</a></p>\n'
f.write(line)
f.write('</body>\n')
f.write('</html>\n')
f.close()
def check_cron():
"""
find new error messages from cron log files for a given machine and a given user.
this script send out email if it finds new error message. The log files are clean up 1 of every month
and the same message could be send out if there are still the same error messages occur.
"""
#
#--- error_log name
#
error_logs = error_log_dir + 'error_list_' + machine + '_' + user
#
#--- find cron file names for this machine for this user
#
cron_file_name = extract_cron_file_name()
#
#--- check today's date. if it is 1st of the month, move the old error_list to archive form.
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('Local')
if day == 1:
lyear = year
lmon = mon - 1
if lmon < 1:
lmon = 12
lyear -= 1
error_logs_old = error_logs + '_' + str(lmon) + '_' + str(lyear)
cmd = 'mv ' + error_logs + ' ' + error_logs_old
os.system(cmd)
cmd = 'mv '+ error_logs_old + ' ' + error_log_dir + 'Past_logs/.'
os.system(cmd)
error_dict = {}
else:
#
#--- read existing error list
#
error_dict = {}
try:
f = open(error_logs, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
for ent in data:
atemp = re.split('\<:\>', ent)
content = []
for i in range(1, len(atemp)):
content.append(atemp[i])
error_dict[atemp[0]] = content
except:
pass
#
#--- set which Log location to check (depends on user.)
#
dir_loc = '/home/' + user + '/Logs/'
#
#--- find names of files and directories in the Logs directory
#
cmd = 'ls -lrtd ' + dir_loc + '/* >' + tempout
os.system(cmd)
f = open(tempout, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + tempout
os.system(cmd)
cron_list =[]
for ent in data:
atemp = re.split('\s+|\t+', ent)
m1 = re.search('d', atemp[0])
m2 = re.search('Past_logs', ent)
#
#--- if it is a directory other than Past_logs, find file names in that directory
#
if (m1 is not None) and (m2 is None):
cmd = 'ls ' + atemp[8] + '/* > ' + tempout
os.system(cmd)
f = open(tempout, 'r')
data2 = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + tempout
os.system(cmd)
for ent2 in data2:
cron_list.append(ent2)
#
#--- files in Logs directory level
#
elif m2 is None:
cron_list.append(atemp[8])
new_error_dict = {}
for file in cron_list:
#
#--- check whether this error message belongs to this machine (and the user)
#
mchk = 0
for comp in cron_file_name:
m = re.search(comp, file)
if m is not None:
mchk = 1
if mchk > 0:
#
#--- check whether the file has any error messages
#
error_list = find_error(file)
if len(error_list) > 0:
#
#--- if there are error messages, compare them to the previous record, and if it is new append to the record.
#
try:
prev_list = error_dict[file]
new_error = []
for ent in error_list:
sent = "".join(ent.split()) #---- removing all white spaces
chk = 0
for comp in prev_list:
scomp = "".join(comp.split())
if sent == scomp:
chk = 1
if chk == 0:
prev_list.append(ent)
new_error.append(ent)
if len(new_error) > 0:
error_dict[file] = prev_list
new_error_dict[file] = new_error
except:
#
#--- there is no previous error_list entry: so all error messages are new and log them
#
error_dict[file] = error_list
new_error_dict[file] = error_list
#
#--- update error logs
#
old_log = error_logs + '~'
cmd = 'mv ' + error_logs + ' ' + old_log
os.system(cmd)
f = open(error_logs, 'w')
for key in error_dict:
line = key
for e_ent in error_dict[key]:
line = line + '<:>' + e_ent
line = line + '\n'
f.write(line)
f.close()
#
#---if new error messages are found; notify to a list of users
#
chk = 0
f = open(tempout, 'w')
for key in new_error_dict:
chk += 1
line = key + '\n'
f.write(line)
for ent in new_error_dict[key]:
line = '\t' + ent + '\n'
f.write(line)
f.write('\n')
f.close()
if chk > 0:
cmd = 'cat ' + tempout + ' | mailx -s "Subject: Cron Error : ' + user + ' on ' + machine + '" ' + email_list
os.system(cmd)
#
#--- add the error message to a recored
#
add_to_log()
#
#--- update html page
#
update_html()
update_main_html()
else:
#
#--- if there is no error, notify that fact to admin
#
f = open(tempout, 'w')
line = '\nNo error is found today on ' + machine + ' by a user ' + user + '.\n'
f.write(line)
f.close()
cmd = 'cat ' + tempout + ' | mailx -s "Subject: No Cron Error : ' + user + ' on ' + machine + '" ' + admin
os.system(cmd)
cmd = 'rm ' + tempout
os.system(cmd)
def acis_dose_control(year = 'NA', month = 'NA', comp_test = 'NA'):
"""
monthly acis dose update control script
input: optional year and month
"""
if year == 'NA' or month == 'NA':
(year, mon, day, hours, min, sec, weekday, yday, dst) = tcnv.currentTime('Local')
month = mon -1
if month < 1:
month = 12
year -= 1
else:
year = int(year)
month = int(month)
syear = str(year)
smon = str(month)
if month < 10:
smon = '0' + smon
#
#--- extract data
#
getd.acis_dose_get_data(year, month, year, month)
#
#--- create cumulative data and sectioned data for bot month and cumulative data
#
file = 'ACIS_' + smon + '_' + syear + '.fits'
file2= 'ACIS_' + smon + '_' + syear + '.fits.gz'
chk = 0
for test in os.listdir('./'):
if fnmatch.fnmatch(test, file2):
chk = 1
break
if chk == 0:
cuml.acis_create_cumulative(file, comp_test)
pass
else:
pass
cuml.acis_create_cumulative(file2, comp_test)
#
#--- compute statistics
#
astat.acis_dose_extract_stat_data_month(year, month, comp_test)
#
#--- plot data
#
aplot.acis_dose_plot_exposure_stat(clean='Yes', comp_test=comp_test)
#
#--- create images (you need to use ds9 to create a better image)
#
aimg.create_acis_maps(year, month, comp_test)
#
#--- update html pages
#
ahtml.acis_dose_make_data_html(comp_test = comp_test)
#
#--- print monthly output
#
arport.acis_dose_monthly_report()
def get_data(ccd_list, out):
"""
read out data from the full cti data table and creates monthly report data table
Input: ccd_list --- a list of ccds which you want to read the data
out --- a type of the data, "image", "spec", or "back"
data are read from "/data/mta/Script/ACIS/CTI/DATA/..."
Output: xSets --- a list of lists of x values of each ccd
ySets --- a list of lists of y values of each ccd
eSets --- a list of lists of y error of each ccd
./Data/ccd<ccd>_data: monthly averaged cti for monthly report
"""
#
#--- read intercept adjusting table
#
al_factors = read_correction_factor('al')
mn_factors = read_correction_factor('mn')
ti_factors = read_correction_factor('ti')
#
#--- find today's date; ctime[0] is year and ctime[1] is month
#
ctime = tcnv.currentTime()
lyear = int(ctime[0])
lmonth= int(ctime[1])
#
#--- set dimension of the array
#
c_cnt = 12 * (int(ctime[0]) - 2000) + int(ctime[1])
d_cnt = len(ccd_list)
#
#--- for none backside CCDs, we use detrended data sets
#
if out == 'back':
dir = '/data/mta/Script/ACIS/CTI/Data/Data_adjust/'
#
#--- vadd to adjust the mean position of CTI
#
else:
dir = '/data/mta/Script/ACIS/CTI/Data/Det_Data_adjust/'
xSets = []
ySets = []
eSets = []
#
#-- go around each ccds
#
for i in range(0, d_cnt):
ccd = ccd_list[i]
#
#--- set cti data array and error array
#
avals = [0 for x in range(0, c_cnt)]
sum = [0 for x in range(0, c_cnt)]
sum2 = [0 for x in range(0, c_cnt)]
#
#--- go around all lines
#
for elm in ('mn'):
vadd = 0
if elm == 'al':
corrections = al_factors
vadd = 0
for k in range(0, 4):
vadd += corrections[k][ccd]
vadd /= 4.0
elif elm == 'ti':
correcitons = ti_factors
else:
corrections = mn_factors
for k in range(0, 4):
vadd += corrections[k][ccd]
vadd /= 4.0
file = dir + 'mn_ccd' + str(ccd)
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
for ent in data:
atemp = re.split('\t+|\s+', ent)
btemp = re.split('-', atemp[0])
#
#--- find the row that you want to add this data
#
pos = 12 * (int(btemp[0]) - 2000) + int(btemp[1]) - 1
for k in range(1, 5):
ctemp = re.split('\+\-', atemp[k])
val = float(ctemp[0])
if val > 0 and val < yupper:
#
#--- correct the value so that all data points have about the same base line
#
val -= (corrections[k-1][ccd])
val += vadd
err = float(ctemp[1])
if err > 0:
avals[pos] += val
sum[pos] += 1.0
sum2[pos] += val * val
#
#--- open file for print out
#
file = './Data/cti_data/ccd' + str(ccd) + '_data'
fo = open(file, 'w')
line = '#\n#date cti errer\n#\n'
fo.write(line)
chk = 0
xvals = []
yvals = []
evals = []
for k in range(2000, lyear+1):
for m in range(0, 13):
if (k == lyear) and (m > lmonth):
chk = 1
break
pos = 12 * (k - 2000) + m -1
#
#--- set time in fractional year. adding 0.04 to set time to the mid month
#
date = k + float(m) / 12.0 + 0.04
date = round(date, 3)
#
#--- compute average and erorr
#
if avals[pos] > 0:
avg = avals[pos] / sum[pos]
err = math.sqrt(sum2[pos] /sum[pos] - avg * avg)
avg = round(avg, 3)
err = round(err, 3)
if len(str(date)) == 7:
line = str(date) + ' \t' + str(avg) + '\t' + str(err) + '\n'
else:
line = str(date) + '\t' + str(avg) + '\t' + str(err) + '\n'
fo.write(line)
xvals.append(date)
yvals.append(avg)
evals.append(err)
if chk > 0:
break
fo.close()
#
#--- create a lists of lists
#
xSets.append(xvals)
ySets.append(yvals)
eSets.append(evals)
return [xSets, ySets, eSets]
def write_html(ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos, asig1, asig2, asig3, \
mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3):
"""
write a html page:
input: ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,m10_acc
m10_apos,mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,m10_dff,m10_dpos
"""
(lyear, lmon, lday, lhours, lmin, lsec, lweekday, lyday,
dst) = tcnv.currentTime('Local')
f = open('acis.html', 'w')
#
#--- this is a html 5 document
#
f.write('<!DOCTYPE html>\n')
f.write('<html>\n')
f.write('<head>\n')
line = '<title>ACIS ' + ccd.upper() + ' Section ' + str(
sec) + ' History Data</title>\n'
f.write(line)
f.write(
"<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />"
)
#
#--- css style sheet
#
f.write('<style type="text/css">\n')
f.write(
'table{text-align:center;margin-left:auto;margin-right:auto;border-style:solid;border-spacing:8px;border-width:2px;border-collapse:separate}\n'
)
f.write('td {text-align:center}\n')
f.write('a:link {color:aqua;}\n')
f.write('a:visited {color:green;}\n')
f.write('</style>\n')
f.write('</head>\n')
f.write('<body style="color:#FFFFFF;background-color:black"> \n')
line = '<br /><h3> Last Update: ' + str(lmon) + '/' + str(
lday) + ' / ' + str(lyear) + '</h3>\n'
f.write(line)
f.write('<table border=1>\n')
header_write(f)
for i in range(0, len(year)):
f.write('<tr>\n')
line = '<td>' + str(year[i]) + '</td>\t'
line = line + '<td>' + str(month[i]) + '</td>\t'
line = line + '<td>' + str(mean_dff[i]) + '</td>\t'
line = line + '<td>' + str(std_dff[i]) + '</td>\t'
line = line + '<td>' + str(min_dff[i]) + '</td>\t'
line = line + '<td>' + str(min_dpos[i]) + '</td>\t'
line = line + '<td>' + str(max_dff[i]) + '</td>\t'
line = line + '<td>' + str(max_dpos[i]) + '</td>\t'
line = line + '<td>' + str(dsig1[i]) + '</td>\t'
line = line + '<td>' + str(dsig2[i]) + '</td>\t'
line = line + '<td>' + str(dsig3[i]) + '</td>\t'
f.write(line)
syear = str(year[i])
smon = str(month[i])
if month[i] < 10:
smon = '0' + smon
lccd = ccd.replace('_', '')
file = 'ACIS_' + smon + '_' + syear + '_' + lccd
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month/' + file + '.fits.gz">fits</a></td>\n'
f.write(line)
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + file + '.png">map</a></td>\n\n'
f.write(line)
line = '<td>' + str(mean_acc[i]) + '</td>\t'
line = line + '<td>' + str(std_acc[i]) + '</td>\t'
line = line + '<td>' + str(min_acc[i]) + '</td>\t'
line = line + '<td>' + str(min_apos[i]) + '</td>\t'
line = line + '<td>' + str(max_acc[i]) + '</td>\t'
line = line + '<td>' + str(max_apos[i]) + '</td>\t'
line = line + '<td>' + str(asig1[i]) + '</td>\t'
line = line + '<td>' + str(asig2[i]) + '</td>\t'
line = line + '<td>' + str(asig3[i]) + '</td>\n'
f.write(line)
file = 'ACIS_07_1999_' + smon + '_' + syear + '_' + lccd
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative/' + file + '.fits.gz">fits</a></td>\n'
f.write(line)
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + file + '.png">map</a></td></tr>\n\n'
f.write(line)
#
#--- put header every new year so that we can read data easier
#
if month[i] % 12 == 0 and i != (len(year) - 1):
header_write(f)
f.write('\n')
f.write('</table>\n\n')
f.write('<br /><br /><hr /><br />\n')
line = '<br /><strong style="font-size:105%;float:right">Last Update: ' + smon + '/' + str(
lday) + '/' + str(lyear) + '</strong>\n'
f.write(line)
line = '<p>If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected].</a></p>\n'
f.write(line)
f.write('</body>\n')
f.write('</html>\n')
f.close()
def update_html():
"""
create/update error html page for a gvien machine and a user
input: error_list_<cpu>_<user>, wheren cpu and user are found from the machine running this and account using.
output: cron_error_<cpu>_<usr>.html in html_dir.
"""
#
#--- find the current time
#
[year, mon, day, hours, min, sec, weekday, yday,
dst] = tcnv.currentTime('Local')
syear = str(year)
smon = str(mon)
lmon = tcnv.changeMonthFormat(mon)
if mon < 10:
smon = '0' + smon
#
#--- set the file name and a hmtl page name
#
file = error_log_dir + machine + '_' + user + '_' + smon + '_' + syear
html = html_dir + 'cron_error_' + machine + '_' + user + '_' + smon + '_' + syear + '.html'
#
#--- start writing the html page
#
out = open(html, 'w')
line = '<!DOCTYPE html>\n'
line = line + '<html>\n'
line = line + '<head>\n'
line = line + '<title>Cron Error Log for ' + user.upper(
) + ' on ' + machine.upper() + ': ' + lmon + ' ' + syear + '</title>\n'
line = line + '<link rel="stylesheet" type="text/css" href="/mta/REPORTS/Template/mta_style_short.css" />\n'
line = line + '</head>\n'
line = line + '<body>\n'
line = line + '<h3 style="padding-bottom: 10px"> Cron Error Log for ' + user.upper(
) + ' on ' + machine.upper() + ': ' + lmon + ' ' + syear + '</h3>\n\n'
line = line + '<hr />\n'
line = line + '<pre style="padding-left: 5px;padding-bottom:10px">\n'
out.write(line)
#
#--- write the content of error_list
#
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
for ent in data:
out.write(ent)
out.write('\n')
line = '</pre>\n'
line = line + '<br /><hr /><br />\n'
line = line + 'Back to <a href="https://cxc.cfa.harvard.edu/mta_days/mta_cron/cron_error_main.html">Top Page</a>\n'
line = line + '\n</body>\n'
line = line + '</html>\n'
out.write(line)
out.close()
def update_html():
"""
update the main html page (sib_main.html)
Input: none, but read a part from <house_keeping>/sim_head_part
Output: <web_dir>/sib_main.html
"""
#
#--- find today's date, and set a few thing needed to set output directory and file name
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
syear = str(year)
smonth = str(mon)
if mon < 10:
smonth = '0' + smonth
lyear = year
lmon = mon - 1
if lmon < 1:
lmon = 12
lyear -= 1
slyear = str(lyear)
slmonth = str(lmon)
if lmon < 10:
slmonth = '0' + slmonth
#
#--- read the head part from the house_keeping
#
if level == 1:
cmd = 'cp ' + house_keeping + '/sib_head_part ' + web_dir + '/sib_main.html'
else:
cmd = 'cp ' + house_keeping + '/sib_head_part2 ' + web_dir + '/sib_main_lev2.html'
os.system(cmd)
#
#--- add the rest
#
if level == 1:
line = web_dir + '/sib_main.html'
else:
line = web_dir + '/sib_main_lev2.html'
fo = open(line, 'a')
for iyear in range(year, 1999, -1):
fo.write("<tr>\n")
if iyear == year:
line = '<th>' + syear + '</th><td>---</td>'
else:
if level == 1:
line = '<th>' + str(
iyear) + '</th><td><a href="./Plots/Plot_' + str(
iyear) + '/year.html">' + str(iyear) + '</a></td>'
else:
line = '<th>' + str(
iyear) + '</th><td><a href="./Lev2/Plots/Plot_' + str(
iyear) + '/year.html">' + str(iyear) + '</a></td>'
fo.write(line)
fo.write("\n")
for imonth in range(1, 13):
simonth = str(imonth)
if imonth < 10:
simonth = '0' + simonth
if (iyear == year) and (imonth >= mon):
line = '<td>' + simonth + '</td>'
else:
if level == 1:
line = '<td><a href="./Plots/Plot_' + str(
iyear
) + '_' + simonth + '/month.html">' + simonth + '</a></td>'
else:
line = '<td><a href="./Lev2/Plots/Plot_' + str(
iyear
) + '_' + simonth + '/month.html">' + simonth + '</a></td>'
fo.write(line)
fo.write("\n")
fo.write("</tr>\n")
fo.write("</table>\n")
line = '<p style="padding-top:40px;padding-bottom:20px"> \n <hr /> \n </p> \n'
fo.write(line)
#
#--- add updated date
#
date = tcnv.currentTime(format='Display')
line = '<p style="padding-top:10px">Last Updated: ' + str(
date) + '<br />' + "\n"
fo.write(line)
line = '<em style="padding-top:10px">If you have any questions, contact: <a href="mailto:[email protected]">[email protected]</a></p>'
fo.write(line)
fo.write("\n")
fo.write("</body>\n")
fo.write("</html>\n")
fo.close()
def ccd_comb_plot(choice, syear = 2000, smonth = 1, eyear = 2000, emonth = 1, header = 'plot_ccd'):
"""
a control script to create plots
Input: choice --- if normal, monthly updates of plots are created.
if check, plots for a given period are created
syear --- starting year of the period, choice must be 'check'
smonth --- starting month of the period, choice must be 'check'
eyear --- ending year of the period, choice must be 'check'
emonth --- ending month of the period, choice must be 'check'
header --- a header of the plot file choice must be 'check'
Output: png formated plotting files
"""
#
#--- find today's date, and set a few thing needed to set output directory and file name
#
if choice != 'check':
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
syear = str(year)
smonth = str(mon)
if mon < 10:
smonth = '0'+ smonth
lyear = year
lmon = mon - 1
if lmon < 1:
lmon = 12
lyear -= 1
slyear = str(lyear)
slmonth = str(lmon)
if lmon < 10:
slmonth = '0' + slmonth
#
#--- normal monthly operation
#
if choice == 'normal':
#
#--- monthly plot
#
dlist = collect_data_file_names('month')
plot_out = web_dir + '/Plots/Plot_' + slyear + '_' + slmonth + '/'
check_and_create_dir(plot_out)
header = 'month_plot_ccd'
plot_data(dlist, plot_out, header, yr=slyear, mo=slmonth, psize=2.5)
#
#--- quarterly plot
#
dlist = collect_data_file_names('quarter')
plot_out = web_dir + '/Plots/Plot_quarter/'
check_and_create_dir(plot_out)
header = 'quarter_plot_ccd'
plot_data(dlist, plot_out, header)
dlist = collect_data_file_names('year')
plot_out = web_dir + '/Plots/Plot_past_year/'
check_and_create_dir(plot_out)
header = 'one_year_plot_ccd'
plot_data(dlist, plot_out, header)
#
#--- full previous year's plot. only updated in Jan of new year
#
if mon == 1:
dlist = collect_data_file_names('lyear')
lyear = year -1
plot_out = web_dir + '/Plots/Plot_' + str(lyear) + '/'
check_and_create_dir(plot_out)
header = 'year_plot_ccd'
plot_data(dlist, plot_out, header, yr=slyear)
#
#--- entire trend plot
#
dlist = collect_data_file_names('full')
plot_out = web_dir + '/Plots/Plot_long_term/'
check_and_create_dir(plot_out)
header = 'full_plot_ccd'
plot_data(dlist, plot_out, header, xunit='year')
#
#--- special case which we need to specify periods
#
elif choice == 'check':
dlist = collect_data_file_names('check', syear, smonth, eyear, emonth)
plot_out = web_dir + '/Plot/'
check_and_create_dir(plot_out)
# header = 'plot_special_ccd'
plot_data(dlist, plot_out, header, yr=syear, mo=smonth)
#
#--- extra...
#
else:
for year in range(2000, 2014):
dlist = collect_data_file_names('check', year, 1, year, 12)
plot_out = web_dir + '/Plots/Plot_' + str(year) + '/'
check_and_create_dir(plot_out)
header = 'one_year_plot_ccd'
plot_data(dlist, plot_out, header, yr = str(year))
for month in range(1, 13):
print " Processing: " + str(year) + ' / ' + str(month)
smonth = str(month)
if month < 10:
smonth = '0' + smonth
dlist = collect_data_file_names('check', year, month, year, month)
plot_out = web_dir + '/Plots/Plot_' + str(year) + '_' + smonth + '/'
check_and_create_dir(plot_out)
header = 'month_plot_ccd'
plot_data(dlist, plot_out, header, yr=str(year), mo=smonth, psize=2.5)
def print_html(year, mon):
"""
create and/or update radiation related html page
"""
#
#--- find today's date
#
if year == '':
[year, mon, day, hours, min, sec, weekday, yday,
dst] = tcnv.currentTime()
#
#--- find out the last month
#
# cyear = year
# lmon = mon -1
# if lmon < 1:
# lmon = 12
# cyear = year -1
#
#--- for the case year and mon are given
#
cyear = year
lmon = mon
#
#--- choose a correct month list depending on whether this is the leap year
#
if tcnv.isLeapYear(cyear) == 1:
mon_list = mon_list1
else:
mon_list = mon_list2
last_day = mon_list[lmon - 1]
#
#--- convert the month from a numeric to letter
#
umon = tcnv.changeMonthFormat(lmon)
smon = umon.lower()
lmon_year = str(cyear)
syear = lmon_year[2] + lmon_year[3]
last_year = str(year - 1)
syear2 = last_year[2] + last_year[3]
monyear = smon + syear
#
#--- set output html page names
#
year_html = 'all' + syear + '.html'
mon_html = monyear + '.html'
rad_html = 'rad_time_' + monyear + '.html'
#
#--- read yearly html page template
#
data = open('./Template/yearly_template', 'r').read()
data = data.replace('$#FYEAR#$', str(year))
data = data.replace('$#SYEAR#$', syear)
fo = open(year_html, 'w')
fo.write(data)
fo.close()
#
#--- read monthly html page template
#
data = open('./Template/monthly_template', 'r').read()
data = data.replace('$#FYEAR#$', str(year))
data = data.replace('$#UMON#$', umon)
data = data.replace('$#MONYEAR#$', monyear)
fo = open(mon_html, 'w')
fo.write(data)
fo.close()
#
#--- read rad_time html page template
#
data = open('./Template/rad_time_template', 'r').read()
data = data.replace('$#LMONTH#$', fmon_list[mon - 2])
data = data.replace('$#FYEAR#$', str(year))
data = data.replace('$#MONYEAR#$', monyear)
fo = open(rad_html, 'w')
fo.write(data)
fo.close()
def plot_cum_grating(time, h_in, l_in):
"""
plot cummulative count rates of hetig and letig insertion.
input: time --- fractional year
h_in --- hetig insertion cummulative count rate (month step)
l_in --- letig insertion cummulative count rate
output: monthly_grat.prn
"""
#
#--- set x axis plotting range
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
xmin = 2000
xmax = year + 1
if mon > 6:
xmax += 1
#
#--- set y axis plotting range
#
ymin = 0.0
ymax = max(h_in)
ymax2 = max(l_in)
if ymax2 > ymax:
ymax = ymax2
ymax = int(1.1 * ymax) + 10
#
#--- set a few parameters
#
fsize = 9
lsize = 0
color = 'red'
marker = 'o'
msize = 3
plt.close("all")
mpl.rcParams['font.size'] = fsize
props = font_manager.FontProperties(size=fsize)
plt.subplots_adjust(hspace=0.08, wspace=0.10)
#
#--- HETG Cumulative Count Plots
#
a1 = plt.subplot(121) #--- two panel plot: left
plot_sub(a1,
time,
h_in,
xmin,
xmax,
ymin,
ymax,
color,
lsize,
marker,
msize,
tline='HETG')
a1.set_xlabel('Time (year)', size=fsize)
a1.set_ylabel('Cumulative Insertion Counts', size=fsize)
#
#--- LETG Cumulative Count Plots
#
a1 = plt.subplot(122) #--- two panel plot: right
plot_sub(a1,
time,
l_in,
xmin,
xmax,
ymin,
ymax,
color,
lsize,
marker,
msize,
tline='LETG')
a1.set_xlabel('Time (year)', size=fsize)
#
#--- save the plot
#
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(10.0, 7.5)
outname = 'monthly_grat.png'
plt.savefig(outname, format='png', dpi=100)
def get_grat_data():
"""
read database and extract needed information, then create data
input: none but read from the database: "/data/mta/www/mta_otg/OTG_sorted.rdb"
output: [time, h_in_ang, h_out_ang, l_in_ang, l_out_ang, h_in, h_out, l_in, l_out]
where: time --- time in fractional year
h_in_ang --- hetig insertion angle
h_out_ang --- hetig retraction angle
l_in_ang --- letig insertion angle
l_out_ang --- letig retraction angle
h_in --- hetig insertion cumm count
h_out --- hetig retraction cumm count
l_in --- letig insertion cumm count
l_out --- hetig retraction cumm count
"""
#
#--- read data
#
f = open(datafile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- find the current year. this will be used to remove iregular data
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
direct = []
grating = []
start = []
stop = []
hposa = []
hposb = []
fposa = []
fposb = []
for i in range(1, len(data)):
ent = data[i]
atemp = re.split('\s+', ent)
try:
test = float(atemp[2])
test2 = float(atemp[4])
except:
continue
direct.append(atemp[0].strip())
grating.append(atemp[1].strip())
val = convert_time(atemp[2])
start.append(val)
stop.append(convert_time(atemp[4]))
hposa.append(float(atemp[18]))
hposb.append(float(atemp[19]))
fposa.append(float(atemp[20]))
fposb.append(float(atemp[21]))
#
#--- create start and stop lists of data bin. the width is a month
#
[blist, elist] = create_monthly_bins(2000, year, mon)
blen = len(blist)
time = [0 for x in range(0, blen)]
h_in = [0 for x in range(0, blen)] #--- hetg insertion cumm count rate
h_in_ang = [0 for x in range(0, blen)] #--- hetg insertion angle
h_out = [0 for x in range(0, blen)] #--- hetg retraction cumm count rate
h_out_ang = [0 for x in range(0, blen)] #--- hetg retraction angle
l_in = [0 for x in range(0, blen)]
l_in_ang = [0 for x in range(0, blen)]
l_out = [0 for x in range(0, blen)]
l_out_ang = [0 for x in range(0, blen)]
for j in range(1, blen):
time[j] = 0.5 * (blist[j] + elist[j]) #--- take a mid point for the bin's time
#
#-- creating cummulative count; the current bin should have, at least, as the same as the
#-- previous bin
#
h_in[j] = h_in[j-1]
h_out[j] = h_out[j-1]
l_in[j] = l_in[j-1]
l_out[j] = l_out[j-1]
h_in_ang_cnt = 0
h_out_ang_cnt = 0
l_in_ang_cnt = 0
l_out_ang_cnt = 0
#
#--- since the data are not ordered by date, go through begining to the end
#--- every bin cycle
#
for i in range(0, len(start)):
if start[i] >= blist[j] and start[i] < elist[j]:
if direct[i] == 'INSR':
if grating[i] == 'HETG':
h_in_ang[j] += fposa[i]
h_in_ang_cnt += 1
h_out_ang[j] += hposa[i]
h_out_ang_cnt += 1
if grating[i] == 'LETG':
l_in_ang[j] += fposa[i]
l_in_ang_cnt += 1
l_out_ang[j] += hposa[i]
l_out_ang_cnt += 1
#
#--- taking monthly average
#
if h_in_ang_cnt> 0:
h_in_ang[j] /= h_in_ang_cnt
if h_out_ang_cnt> 0:
h_out_ang[j] /= h_out_ang_cnt
if l_in_ang_cnt> 0:
l_in_ang[j] /= l_in_ang_cnt
if l_out_ang_cnt> 0:
l_out_ang[j] /= l_out_ang_cnt
#
#--- adding in/out count for the month to appropriate bins
#
h_in[j] += h_in_ang_cnt
h_out[j] += h_out_ang_cnt
l_in[j] += l_in_ang_cnt
l_out[j] += l_out_ang_cnt
return [time, h_in_ang, h_out_ang, l_in_ang, l_out_ang, h_in, h_out, l_in, l_out]
def clean_data(dir,
startYear='NA',
startMonth='NA',
stopYear='NA',
stopMonth='NA'):
"""
clean up and correct ACIS/HRC data. if there is duplicated line, remove it. if there are missing line add one (with NA)
input dir, startYear = 1999, startMonth = 9, stopYear, stopMonth
"""
#
#--- if range is not defined, give them
#
if startYear == 'NA':
startYear = 1999
if startMonth == 'NA':
startMonth = 9
(cyear, cmon, day, hours, min, sec, weekday, yday,
dst) = tcnv.currentTime('Local')
if stopYear == 'NA':
stopYear = cyear
if stopMonth == 'NA':
stopMonth = cmon - 1
if stopMonth < 1:
stopMonth = 12
stopYear -= 1
for type in ('*_acc*', '*_dff*'):
#
#--- find file names
#
for fout in os.listdir(dir):
if fnmatch.fnmatch(fout, type):
ent = dir + fout
f = open(ent, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
f = open('zout', 'w')
cyear = startYear
cmonth = startMonth
pyear = 0
pmonth = 0
for aent in data:
atemp = re.split('\s+|\t+', aent)
year = int(atemp[0])
month = int(atemp[1])
#
#--- if entry is duplicated, remove
#
if year == pyear and month == pmonth:
pass
elif year == cyear and month == cmonth:
f.write(aent)
f.write('\n')
pyear = year
pmonth = month
cmonth += 1
if cmonth > 12:
cmonth = 1
cyear += 1
if cyear == stopYear and cmonth > stopMonth:
break
#
#--- if entries are missing, add "NA"
#
elif year == cyear and month > cmonth:
for i in range(cmonth, month):
smon = str(i)
if i < 10:
smon = '0' + smon
line = str(
year
) + '\t' + smon + '\tNA NA NA NA NA NA NA NA\n'
f.write(line)
f.write(aent)
f.write('\n')
pyear = year
pmonth = month
cmonth += 1
if cmonth > 12:
cmonth = 1
cyear += 1
if cyear == stopYear and cmonth > stopMonth:
break
f.close()
cmd = 'mv zout ' + ent
os.system(cmd)
def update_main_html():
"""
update the main cron error log html page
input: get from the list from indivisula html page, e.g., cron_error_rhodes_mta.html
output: cron_error_main.html
"""
#
#--- create a list of file name (header)
#
# file_list = []
# for cpu in cpu_list:
# for name in usr_list:
# filename = cpu + '_' + name
# file_list.append(filename)
file_list = cpu_usr_list #-- we may go back to above scheme in future, but this si fine for now
#
#--- find current time
#
[year, mon, day, hours, min, sec, weekday, yday,
dst] = tcnv.currentTime('Local')
syear = str(year)
smon = str(mon)
lmon = tcnv.changeMonthFormat(mon)
if mon < 10:
smon = '0' + smon
#
#--- start writing the html page
#
html = html_dir + 'cron_error_main.html'
out = open(html, 'w')
line = '<!DOCTYPE html>\n'
line = line + '<html>\n'
line = line + '<head>\n'
line = line + '<title>Cron Error Main page</title>\n'
line = line + '<link rel="stylesheet" type="text/css" href="/mta/REPORTS/Template/mta_style_short.css" />\n'
line = line + '</head>\n'
line = line + '<body>\n'
line = line + '<h2 style="padding-bottom: 10px"> Cron Error Log</h2>\n\n'
line = line + '<pre style="padding-left: 5px;padding-bottom:10px">\n'
line = line + '<hr />\n'
line = line + '<table border=2 cellpadding = 5 cellspacing =5>\n'
line = line + '<tr><th>Period</th>'
out.write(line)
for ent in file_list:
atemp = re.split('_', ent)
line = '<th>' + atemp[1] + ' on ' + atemp[0] + '</th>'
out.write(line)
out.write('</tr>\n')
#
#--- find the names of each html file (e.g. cron_error_rhodes_mta_06_2012.html)
#
templist = tempdir + 'zlist'
cmd = 'ls ' + html_dir + '> ' + templist
os.system(cmd)
f = open(templist, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + templist
os.system(cmd)
error_file_list = []
for ent in data:
m1 = re.search('.html', ent)
m2 = re.search('cron_error_main.html', ent)
if (m1 is not None) and (m2 is None):
error_file_list.append(ent)
#
#--- start printing each row; column is ordered newest to oldeest
#
year_list = range(2012, year + 1)
year_list.reverse()
month_list = range(1, 13)
month_list.reverse()
for dyear in year_list:
for dmonth in month_list:
if dyear == 2012 and dmonth < 6: #---- this is the year/month the script was started
break
if (dyear < year) or (dyear == year and dmonth <= mon):
lmon = tcnv.changeMonthFormat(
dmonth) #--- convert month in digit to letters
line = '<tr><th>' + lmon + ' ' + str(dyear) + '</th>'
out.write(line)
#
#--- check which file (e.g. cron_error_rhodes_mta_06_2012.html) actually exists
#
for fent in file_list:
smon = str(dmonth)
if dmonth < 10:
smon = '0' + smon
fname = 'cron_error_' + fent + '_' + smon + '_' + str(
dyear) + '.html'
chk = 0
for comp in error_file_list:
if fname == comp:
chk = 1
break
if chk > 0:
#
#--- if exist, create a link
#
line = '<td style="color:red;text-align:center"><a href="' + fname + '">Error List</a></td>'
else:
line = '<td style="text-align:center">No Error</td>'
out.write(line)
out.write('</tr>\n')
out.write('</table>\n\n')
out.write('<br /> <hr />\n')
tdate = current_time_from_machine()
line = '<pstyle="font-size:95%"><em>Last Update: ' + tdate + '</em><br />\n'
line = line + 'If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected]</a>.</p>\n'
out.write(line)
line = '\n</body>\n'
line = line + '</html>\n'
out.write(line)
out.close()
def acis_dose_make_data_html(indir='NA', outdir='NA', comp_test='NA'):
"""
read data and create html pages
input: indir, outdir, both are optional
"""
#
#--- setting indir and outdir if not given
#
if indir == 'NA':
if comp_test == 'test':
indir = test_data_out
else:
indir = data_out
if outdir == 'NA':
if comp_test == 'test':
outdir = test_data_out
else:
outdir = data_out
#
#--- read data
#
for ccd in ('i_2', 'i_3', 's_2', 's_3'):
for sec in range(0, 4):
inst = ccd + '_n_' + str(sec)
date = []
year = []
month = []
mean_acc = []
std_acc = []
min_acc = []
min_apos = []
max_acc = []
max_apos = []
asig1 = []
asig2 = []
asig3 = []
mean_dff = []
std_dff = []
min_dff = []
min_dpos = []
max_dff = []
max_dpos = []
dsig1 = []
dsig2 = []
dsig3 = []
[date,year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, mean_dff,std_dff, \
min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3] = expf.readExpData(indir, inst)
#
#--- write html page
#
write_html(ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, \
mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3)
out_name = outdir + inst + '.html'
cmd = 'mv acis.html ' + out_name
os.system(cmd)
#
#--- update top html page
#
line = house_keeping + 'exposure.html'
f = open(line, 'r')
data = f.read()
f.close()
now = tcnv.currentTime("Display")
ndate = 'Last Update: ' + now
ndata = data.replace("Last Update:", ndate)
line = web_dir + 'exposure.html'
f = open(line, 'w')
f.write(ndata)
f.close()
#
#--- update plot page htmls
#
update_plt_html_date(comp_test)
def sci_run_get_radiation_data():
'extract needed radiation data from /data/mta4/www/DAILY/mta_rad/ACE/, and put in rada_data<YYYYY>, where YYYY is the year'
#
#--- find out today's date in Local time frame
#
if comp_test == 'test':
year = 2012
month = 1
day = 2
else:
today = tcnv.currentTime('local')
year = today[0]
month = today[1]
day = today[2]
if month == 1 and day == 1:
#
#--- this is a new year... complete the last year
#
year -= 1
#
#--- extract data form ACE data files
#
line = '/data/mta4/www/DAILY/mta_rad/ACE/' + str(
year) + '*_ace_epam_5m.txt'
cmd = 'cat ' + line + ' > /tmp/mta/ztemp'
cmd = 'cat ' + line + ' > ./ztemp'
os.system(cmd)
#> f = open('/tmp/mta/ztemp', 'r')
f = open('./ztemp', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#> system('rm /tmp/mta/ztemp')
os.system('rm ./ztemp')
#
#--- move the old file to "~" to prepare for the new data
#
name = 'rad_data' + str(year)
if comp_test == 'test':
crtFile = test_web_dir + name
else:
oldFile = house_keeping + name + '~'
crtFile = house_keeping + name
cmd = 'chmod 775 ' + crtFile + ' ' + oldFile
os.system(cmd)
cmd = 'mv ' + crtFile + ' ' + oldFile
os.system(cmd)
cmd = 'chmod 644 ' + oldFile
os.system(cmd)
f = open(crtFile, 'w')
for ent in data:
#
#--- remove comments and headers
#
m = re.search('^#', str(ent))
n = re.search('^:', str(ent))
if (m is None) and (n is None):
line = ent + '\n'
f.write(line)
f.close()
def check_cron():
"""
find new error messages from cron log files for a given machine and a given user.
this script send out email if it finds new error message. The log files are clean up 1 of every month
and the same message could be send out if there are still the same error messages occur.
"""
#
#--- error_log name
#
error_logs = error_log_dir + 'error_list_' + machine + '_' + user
#
#--- find cron file names for this machine for this user
#
cron_file_name = extract_cron_file_name()
#
#--- check today's date. if it is 1st of the month, move the old error_list to archive form.
#
[year, mon, day, hours, min, sec, weekday, yday,
dst] = tcnv.currentTime('Local')
if day == 1:
lyear = year
lmon = mon - 1
if lmon < 1:
lmon = 12
lyear -= 1
error_logs_old = error_logs + '_' + str(lmon) + '_' + str(lyear)
cmd = 'mv ' + error_logs + ' ' + error_logs_old
os.system(cmd)
cmd = 'mv ' + error_logs_old + ' ' + error_log_dir + 'Past_logs/.'
os.system(cmd)
error_dict = {}
else:
#
#--- read existing error list
#
error_dict = {}
try:
f = open(error_logs, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
for ent in data:
atemp = re.split('\<:\>', ent)
content = []
for i in range(1, len(atemp)):
content.append(atemp[i])
error_dict[atemp[0]] = content
except:
pass
#
#--- set which Log location to check (depends on user.)
#
dir_loc = '/home/' + user + '/Logs/'
#
#--- find names of files and directories in the Logs directory
#
cmd = 'ls -lrtd ' + dir_loc + '/* >' + tempout
os.system(cmd)
f = open(tempout, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + tempout
os.system(cmd)
cron_list = []
for ent in data:
atemp = re.split('\s+|\t+', ent)
m1 = re.search('d', atemp[0])
m2 = re.search('Past_logs', ent)
#
#--- if it is a directory other than Past_logs, find file names in that directory
#
if (m1 is not None) and (m2 is None):
cmd = 'ls ' + atemp[8] + '/* > ' + tempout
os.system(cmd)
f = open(tempout, 'r')
data2 = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + tempout
os.system(cmd)
for ent2 in data2:
cron_list.append(ent2)
#
#--- files in Logs directory level
#
elif m2 is None:
cron_list.append(atemp[8])
new_error_dict = {}
for file in cron_list:
#
#--- check whether this error message belongs to this machine (and the user)
#
mchk = 0
for comp in cron_file_name:
m = re.search(comp, file)
if m is not None:
mchk = 1
if mchk > 0:
#
#--- check whether the file has any error messages
#
error_list = find_error(file)
if len(error_list) > 0:
#
#--- if there are error messages, compare them to the previous record, and if it is new append to the record.
#
try:
prev_list = error_dict[file]
new_error = []
for ent in error_list:
sent = "".join(
ent.split()) #---- removing all white spaces
chk = 0
for comp in prev_list:
scomp = "".join(comp.split())
if sent == scomp:
chk = 1
if chk == 0:
prev_list.append(ent)
new_error.append(ent)
if len(new_error) > 0:
error_dict[file] = prev_list
new_error_dict[file] = new_error
except:
#
#--- there is no previous error_list entry: so all error messages are new and log them
#
error_dict[file] = error_list
new_error_dict[file] = error_list
#
#--- update error logs
#
old_log = error_logs + '~'
cmd = 'mv ' + error_logs + ' ' + old_log
os.system(cmd)
f = open(error_logs, 'w')
for key in error_dict:
line = key
for e_ent in error_dict[key]:
line = line + '<:>' + e_ent
line = line + '\n'
f.write(line)
f.close()
#
#---if new error messages are found; notify to a list of users
#
chk = 0
f = open(tempout, 'w')
for key in new_error_dict:
chk += 1
line = key + '\n'
f.write(line)
for ent in new_error_dict[key]:
line = '\t' + ent + '\n'
f.write(line)
f.write('\n')
f.close()
if chk > 0:
cmd = 'cat ' + tempout + ' | mailx -s "Subject: Cron Error : ' + user + ' on ' + machine + '" ' + email_list
os.system(cmd)
#
#--- add the error message to a recored
#
add_to_log()
#
#--- update html page
#
update_html()
update_main_html()
else:
#
#--- if there is no error, notify that fact to admin
#
f = open(tempout, 'w')
line = '\nNo error is found today on ' + machine + ' by a user ' + user + '.\n'
f.write(line)
f.close()
cmd = 'cat ' + tempout + ' | mailx -s "Subject: No Cron Error : ' + user + ' on ' + machine + '" ' + admin
os.system(cmd)
cmd = 'rm ' + tempout
os.system(cmd)
def print_main_head_part(fo, new_bad_pix_list, new_hot_pix_list, new_col_pix_list):
localt = tcnv.currentTime('UTC')
tyear = int(localt[0])
tyday = int(localt[7])
dom = tcnv.YdateToDOM(tyear, tyday)
dispt = tcnv.currentTime('Display')
atemp = re.split('\s+', dispt)
dtime = atemp[0] + ' ' + atemp[1] + ' ' + atemp[2] +', ' + atemp[4]
hfile = house_keeping + 'head'
head_part = open(hfile, 'r').read()
head_part = head_part.replace('#DISPLAY# ', dtime)
head_part = head_part.replace('#YDATE#', str(tyday))
head_part = head_part.replace('#DOM#', str(dom))
line = ''
chk = 0
for i in range(0, 10):
if new_bad_pix_list[i][0] != '':
chk += 1
line = line + 'CCD' + str(i) + ': '
for ent in new_bad_pix_list[i]:
line = line + ent
line = line + '<br />\n'
if chk == 0:
line = 'No New Warm Pixel'
head_part = head_part.replace('#WARMPIX#', line)
line = ''
chk = 0
for i in range(0, 10):
if new_hot_pix_list[i][0] != '':
chk += 1
line = line + 'CCD' + str(i) + ': '
for ent in new_hot_pix_list[i]:
line = line + ent
line = line + '<br />\n'
if chk == 0:
line = 'No New Hot Pixel'
head_part = head_part.replace('#HOTPIX#', line)
line = ''
chk = 0
for i in range(0, 10):
if new_col_pix_list[i][0] != '':
chk += 1
line = line + 'CCD' + str(i) + ': '
for ent in new_col_pix_list[i]:
line = line + ent
line = line + '<br />\n'
if chk == 0:
line = 'No New Warm Column'
head_part = head_part.replace('#WARMCOL#', line)
fo.write(head_part)
def print_index_html():
"""
create and/or update radiation related html page
"""
#
#--- find today's date
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
#
#--- read header part and the first part of the index page
#
line = open('./Template/index_top', 'r').read()
#
#--- start creating a link table
#
line = line + '<table border=1 cellpadding=10 cellspacing=2>\n'
line = line + '<tr>\n'
line = line + '<td colspan=13>\n'
line = line + '<table border=1 width=100%>\n'
line = line + '<tr><th> <a href="all.html" style="font-size:120%">Mission since JAN2010</a></th></tr>\n'
line = line + '</table>\n'
line = line + '</td>\n'
line = line + '</tr>\n'
line = line + '<tr>'
line = line + '<th>Year</th><th>Jan</th><th>Feb</th><th>Mar</th><th>Apr</th><th>May</th><th>Jun</th>\n'
line = line + '<th>Jul</th><th>Aug</th><th>Sep</th><th>Oct</th><th>Nov</th><th>Dec</th>\n'
line = line + '</tr>\n'
for eyear in range(year, 1999, -1):
line = line + '<tr>\n'
line = line + '<th>' + str(eyear) + '</th>\n'
lyear = str(eyear)
syear = lyear[2] + lyear[3]
for emon in range(1, 13):
lmon = tcnv.changeMonthFormat(emon)
monyear = lmon.lower() + syear
cmonyear = monyear.upper()
if eyear == year and emon > mon:
line = line + '<td>' + cmonyear + '</td>\n'
else:
line = line + '<td><a href="./' + monyear + '.html">' + cmonyear + '</a></td>\n'
line = line + '</tr>\n\n'
line = line + '</table>\n'
#
#--- table finished. add a closing part
#
line = line + '<div style="padding-top: 15px"></div>\n'
line = line + '<hr />'
line = line + '<div style="padding-top: 15px"></div>\n'
line = line + '<p>This page is maintained by B. Spitzbart (<a href="*****@*****.**">[email protected]</a>).\n'
line = line + '</body</html>\n'
#
#--- now write out the page
#
fo = open('/data/mta4/www/DAILY/mta_rad/index.html', 'w')
fo.write(line)
fo.close()
def update_main_html():
"""
update the main cron error log html page
input: get from the list from indivisula html page, e.g., cron_error_rhodes_mta.html
output: cron_error_main.html
"""
#
#--- create a list of file name (header)
#
# file_list = []
# for cpu in cpu_list:
# for name in usr_list:
# filename = cpu + '_' + name
# file_list.append(filename)
file_list = cpu_usr_list #-- we may go back to above scheme in future, but this si fine for now
#
#--- find current time
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('Local')
syear = str(year)
smon = str(mon)
lmon = tcnv.changeMonthFormat(mon)
if mon < 10:
smon = '0' + smon
#
#--- start writing the html page
#
html = html_dir + 'cron_error_main.html'
out = open(html, 'w')
line = '<!DOCTYPE html>\n'
line = line + '<html>\n'
line = line + '<head>\n'
line = line + '<title>Cron Error Main page</title>\n'
line = line + '<link rel="stylesheet" type="text/css" href="/mta/REPORTS/Template/mta_style_short.css" />\n'
line = line + '</head>\n'
line = line + '<body>\n'
line = line + '<h2 style="padding-bottom: 10px"> Cron Error Log</h2>\n\n'
line = line + '<pre style="padding-left: 5px;padding-bottom:10px">\n'
line = line + '<hr />\n'
line = line + '<table border=2 cellpadding = 5 cellspacing =5>\n'
line = line + '<tr><th>Period</th>'
out.write(line)
for ent in file_list:
atemp = re.split('_', ent)
line = '<th>' + atemp[1] + ' on ' + atemp[0] + '</th>'
out.write(line)
out.write('</tr>\n')
#
#--- find the names of each html file (e.g. cron_error_rhodes_mta_06_2012.html)
#
templist = tempdir + 'zlist'
cmd = 'ls ' + html_dir + '> ' + templist
os.system(cmd)
f = open(templist, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + templist
os.system(cmd)
error_file_list = []
for ent in data:
m1 = re.search('.html', ent)
m2 = re.search('cron_error_main.html', ent)
if (m1 is not None) and (m2 is None):
error_file_list.append(ent)
#
#--- start printing each row; column is ordered newest to oldeest
#
year_list = range(2012, year + 1)
year_list.reverse()
month_list = range(1,13)
month_list.reverse()
for dyear in year_list:
for dmonth in month_list:
if dyear == 2012 and dmonth < 6: #---- this is the year/month the script was started
break
if (dyear < year) or (dyear == year and dmonth <= mon):
lmon = tcnv.changeMonthFormat(dmonth) #--- convert month in digit to letters
line = '<tr><th>' + lmon + ' ' + str(dyear) + '</th>'
out.write(line)
#
#--- check which file (e.g. cron_error_rhodes_mta_06_2012.html) actually exists
#
for fent in file_list:
smon = str(dmonth)
if dmonth < 10:
smon = '0' + smon
fname = 'cron_error_' + fent + '_' + smon + '_' + str(dyear) + '.html'
chk = 0
for comp in error_file_list:
if fname == comp:
chk = 1
break
if chk > 0:
#
#--- if exist, create a link
#
line = '<td style="color:red;text-align:center"><a href="' + fname + '">Error List</a></td>'
else:
line = '<td style="text-align:center">No Error</td>'
out.write(line)
out.write('</tr>\n')
out.write('</table>\n\n')
out.write('<br /> <hr />\n')
tdate = current_time_from_machine()
line = '<pstyle="font-size:95%"><em>Last Update: ' + tdate + '</em><br />\n'
line = line + 'If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected]</a>.</p>\n'
out.write(line)
line = '\n</body>\n'
line = line + '</html>\n'
out.write(line)
out.close()
year = 2012
month = 4
day = 22
#
#--- create test directories
#
asrf.prep_for_test(web_dir, comp_test)
elif comp_test == 'test2': #---- this is another test case, but go over two years 2011 and 2012
year = 2012
month = 2
day = 2
asrf.prep_for_test(web_dir, comp_test)
else:
today = tcnv.currentTime('local')
year = today[0]
month = today[1]
day = today[2]
#
#--- setting the current output directory
#
current_dir = 'Year' + str(year) + '/'
#-----------------------------------------------------------------------------------------------
#---acis_sci_run_get_data: extracts mit data and updates acis science run data and plots ---
#-----------------------------------------------------------------------------------------------
def acis_sci_run_get_data():
"""
def extractLimit(datloc):
"""
read average and std values of each msid, compute lower and upper limits, and create limit data table
Input: datloc: the name of the data directory under data_dir
Output: data_dir/os_limit_table
"""
#
#--- read data set names from data directory
#
datpass = data_dir + datloc + '/'
cmd = 'ls ' + datpass + '* >' + ztemp
os.system(cmd)
f = open(ztemp, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + ztemp
# os.system(cmd)
#
#--- print header
#
outfile = data_dir + 'os_limit_table'
fw = open(outfile, 'w')
dtime = tcnv.currentTime('Display')
line = '#Last Update: ' + dtime + '\n'
fw.write(line)
fw.write('#\t\t\tLower Limit\t\tUpper Limit\t\tGroup\n')
fw.write('#msid\t\t Red\tYellow\tYellow\tRed\n')
fw.write('#-------------------------------------------------------\n')
#
#---- start printing limit values
#
for ent in data:
print ent
temp = re.split(datpass, ent)
group = temp[1] #---- group ID
if ent == outfile: #---os_limt_table is in the data directory, too; so ignore it
pass
else:
f = open(ent, 'r')
msdat = [line.strip() for line in f.readlines()]
f.close()
colEnt = re.split('\s+|\t+', msdat[0])
colLen = len(colEnt)
colName = []
#
#--- read msid names
#
for ent in colEnt:
if ent != 'std' and ent != '#time':
colName.append(ent)
#
#--- read average and std of each msid
#
lastp = int(colLen / 2)
for k in range(0, lastp + 1):
exec("avg%d = []" % (k))
exec("sig%d = []" % (k))
total = 0
for line in msdat:
if total == 0:
total += 1
else:
atemp = re.split('\s+|\t+', line)
for k in range(1, colLen):
k2 = int(k / 2)
if k % 2 == 0:
k2 -= 1
#
#--- the data is occasionally na; if that is the case, use the previous value. if it is the first one, set it to "0"
#
try:
exec("sig%d.append(float(atemp[%d]))" %
(k2, k))
except:
try:
exec("sig%d.append(sig%d[len(sig%d) -1])" %
(k2, k2, k2))
except:
exec("sig%d.append(0)" % (k2))
else:
try:
exec("avg%d.append(float(atemp[%d]))" %
(k2, k))
except:
try:
exec("avg%d.append(avg%d[len(avg%d) -1])" %
(k2, k2, k2))
except:
exec("avg%d.append(0)" % (k2))
total += 1
total -= 2
if total > 2: #---- we need at least 4 data points
fw.write('#\n')
for k in range(0, len(colName)):
#
#--- compute limits; as for a deviation, use the average of the past two years
#
exec(
"madd = sig%d[%d] + sig%d[%d] + sig%d[%d] + sig%d[%d]"
% (k, total - 3, k, total - 2, k, total - 1, k, total))
madd /= 4.0
exec("lavg = avg%d[%d]" % (k, total))
upperYellow = lavg + 4.0 * madd
upperRed = lavg + 5.0 * madd
lowerYellow = lavg - 4.0 * madd
lowerRed = lavg - 5.0 * madd
if upperYellow == lowerYellow:
if lowerYellow == 0:
lowerYellow -= 0.01
upperYellow += 0.01
lowerRed -= 0.01
upperRed += 0.01
else:
diff = 0.01 * upperRed
lowerYellow -= diff
upperYellow += diff
lowerRed -= diff
upperRed += diff
if len(colName[k]) < 8:
line = '%s\t\t%5.3f\t%5.3f\t%5.3f\t%5.3f\t%s\n' % (
colName[k], lowerRed, lowerYellow, upperYellow,
upperRed, group)
else:
line = '%s\t%5.3f\t%5.3f\t%5.3f\t%5.3f\t%s\n' % (
colName[k], lowerRed, lowerYellow, upperYellow,
upperRed, group)
fw.write(line)
fw.close()
def hrc_gain_trend_plot():
"""
create time trend of Gaussian profiles fit on HRC PHA data. It also create trend along radial distnace for each year
Input: none but the data is read from <house_keeping>/fitting_results
Outut: time trend plots in <plot_dir> / hrc_i_time_trend.png hrc_s_time_trend.png
radial_distance plots hrc_i_radial_dist_year<year>.png
"""
file = house_keeping + 'fitting_results'
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- set a few things such as labeling
#
yMinSets = [0, 0, 20]
yMaxSets = [250, 250, 100]
yname = 'PHA'
entLabels = ["PHA Median", "PHA Voigt Peak Position", "PHA FWHM"]
#
#--- time trend plots
#
xmin = 1999
ctemp = tcnv.currentTime() #--- finding this year
xmax = end_year = int(ctemp[0]) + 1
xname = 'Time (Year)'
#
#--- HRC I
#
time_i = []
dist_i = []
med_i = []
center_i = []
width_i = []
time_s = []
dist_s = []
med_s = []
center_s = []
width_s = []
for ent in data:
if ent[0] == '#': #--- avoid comments
continue
atemp = re.split('\t+', ent)
#
#--- if the width is < 1 or amp is smaller than 5, probably it did not find a correct source position
#
if float(atemp[12]) < 1:
continue
if float(atemp[11]) < 5:
continue
fyear = tcnv.sectoFracYear(float(atemp[2]))
if atemp[3] == 'HRC-I':
time_i.append(fyear)
dist_i.append(float(atemp[8]))
med_i.append(float(atemp[9]))
center_i.append(float(atemp[10]))
width_i.append(float(atemp[12]))
else:
time_s.append(fyear)
dist_s.append(float(atemp[8]))
med_s.append(float(atemp[9]))
center_s.append(float(atemp[10]))
width_s.append(float(atemp[12]))
xSets = [time_i, time_i, time_i]
ySets = [med_i, center_i, width_i]
dist = dist_indexing(dist_i)
plotPanel(xmin, xmax, yMinSets, yMaxSets, xSets, ySets, dist, xname, yname,
entLabels)
cmd = 'mv out.png ' + web_dir + 'Trend_plots/hrc_i_time_trend.png'
os.system(cmd)
#
#--- HRC S
#
xSets = [time_s, time_s, time_s]
ySets = [med_s, center_s, width_s]
dist = dist_indexing(dist_s)
plotPanel(xmin, xmax, yMinSets, yMaxSets, xSets, ySets, dist, xname, yname,
entLabels)
cmd = 'mv out.png ' + web_dir + 'Trend_plots/hrc_s_time_trend.png'
os.system(cmd)
#
#--- trend along radial distance
#
xmin = 0
xmax = 16
xname = 'Radial Distance(Arcsec)'
#
#--- HRC I
#
for pyear in range(1999, end_year):
dist = []
med = []
center = []
width = []
mark = []
pend = pyear + 1
for j in range(0, len(time_i)):
if (time_i[j] >= pyear) and (time_i[j] < pend):
dist.append(dist_i[j])
med.append(med_i[j])
center.append(center_i[j])
width.append(width_i[j])
xSets = [dist, dist, dist]
ySets = [med, center, width]
plotPanel(xmin,
xmax,
yMinSets,
yMaxSets,
xSets,
ySets,
mark,
xname,
yname,
entLabels,
extraNote=pyear)
outname = web_dir + '/Trend_plots/hrc_i_radial_dist_year' + str(
pyear) + '.png'
cmd = 'mv out.png ' + outname
os.system(cmd)
#
#--- HRC S
#
for pyear in range(1999, end_year):
dist = []
med = []
center = []
width = []
mark = []
pend = pyear + 1
for j in range(0, len(time_s)):
if (time_s[j] >= pyear) and (time_s[j] < pend):
dist.append(dist_s[j])
med.append(med_s[j])
center.append(center_s[j])
width.append(width_s[j])
xSets = [dist, dist, dist]
ySets = [med, center, width]
plotPanel(xmin,
xmax,
yMinSets,
yMaxSets,
xSets,
ySets,
mark,
xname,
yname,
entLabels,
extraNote=pyear)
outname = web_dir + '/Trend_plots/hrc_s_radial_dist_year' + str(
pyear) + '.png'
cmd = 'mv out.png ' + outname
os.system(cmd)
def write_html(ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos, asig1, asig2, asig3, \
mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3):
"""
write a html page:
input: ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,m10_acc
m10_apos,mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,m10_dff,m10_dpos
"""
(lyear, lmon, lday, lhours, lmin, lsec, lweekday, lyday, dst) = tcnv.currentTime('Local')
f = open('acis.html', 'w')
#
#--- this is a html 5 document
#
f.write('<!DOCTYPE html>\n')
f.write('<html>\n')
f.write('<head>\n')
line = '<title>ACIS ' + ccd.upper() + ' Section ' + str(sec) + ' History Data</title>\n'
f.write(line)
f.write("<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />")
#
#--- css style sheet
#
f.write('<style type="text/css">\n')
f.write('table{text-align:center;margin-left:auto;margin-right:auto;border-style:solid;border-spacing:8px;border-width:2px;border-collapse:separate}\n')
f.write('td {text-align:center}\n')
f.write('a:link {color:aqua;}\n')
f.write('a:visited {color:green;}\n')
f.write('</style>\n')
f.write('</head>\n')
f.write('<body style="color:#FFFFFF;background-color:black"> \n')
line = '<br /><h3> Last Update: ' + str(lmon) + '/' + str(lday) + ' / ' + str(lyear) + '</h3>\n'
f.write(line)
f.write('<table border=1>\n')
header_write(f)
for i in range(0, len(year)):
f.write('<tr>\n')
line = '<td>' + str(year[i]) + '</td>\t'
line = line + '<td>' + str(month[i]) + '</td>\t'
line = line + '<td>' + str(mean_dff[i]) + '</td>\t'
line = line + '<td>' + str(std_dff[i]) + '</td>\t'
line = line + '<td>' + str(min_dff[i]) + '</td>\t'
line = line + '<td>' + str(min_dpos[i]) + '</td>\t'
line = line + '<td>' + str(max_dff[i]) + '</td>\t'
line = line + '<td>' + str(max_dpos[i]) + '</td>\t'
line = line + '<td>' + str(dsig1[i]) + '</td>\t'
line = line + '<td>' + str(dsig2[i]) + '</td>\t'
line = line + '<td>' + str(dsig3[i]) + '</td>\t'
f.write(line)
syear = str(year[i])
smon = str(month[i])
if month[i] < 10:
smon = '0' + smon
lccd = ccd.replace('_', '')
file = 'ACIS_' + smon + '_' + syear + '_' + lccd
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month/' + file + '.fits.gz">fits</a></td>\n'
f.write(line)
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + file + '.png">map</a></td>\n\n'
f.write(line)
line = '<td>' + str(mean_acc[i]) + '</td>\t'
line = line + '<td>' + str(std_acc[i]) + '</td>\t'
line = line + '<td>' + str(min_acc[i]) + '</td>\t'
line = line + '<td>' + str(min_apos[i]) + '</td>\t'
line = line + '<td>' + str(max_acc[i]) + '</td>\t'
line = line + '<td>' + str(max_apos[i]) + '</td>\t'
line = line + '<td>' + str(asig1[i]) + '</td>\t'
line = line + '<td>' + str(asig2[i]) + '</td>\t'
line = line + '<td>' + str(asig3[i]) + '</td>\n'
f.write(line)
file = 'ACIS_07_1999_' + smon + '_' + syear + '_' + lccd
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative/' + file + '.fits.gz">fits</a></td>\n'
f.write(line)
line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + file + '.png">map</a></td></tr>\n\n'
f.write(line)
#
#--- put header every new year so that we can read data easier
#
if month[i] % 12 == 0 and i != (len(year)-1):
header_write(f)
f.write('\n')
f.write('</table>\n\n')
f.write('<br /><br /><hr /><br />\n')
line = '<br /><strong style="font-size:105%;float:right">Last Update: ' + smon + '/' + str(lday) + '/' + str(lyear) + '</strong>\n'
f.write(line)
line = '<p>If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected].</a></p>\n'
f.write(line)
f.write('</body>\n')
f.write('</html>\n')
f.close()
def update_new_obs_list():
"""
update new_obs_lsit, too_list, ddt_list, and obs_in_30days
no input but data are taken from sot_ocat.out.
"""
#
#--- set limit to the last 30 days
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('LOCAL')
tdom = tcnv.findDOM(year, mon, day, 0, 0, 0)
dom_limit = tdom - 30
dom_limit = tdom - 100
dom30days = tdom + 30 #---- use to find observations will happen in 30 days
#
#--- read special obsid --- poc list
#
line = too_dir + 'special_obsid_poc_list'
f = open(line, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
sp_obsid = []
sp_user = []
for ent in data:
atemp = re.split('\s+', ent)
sp_obsid.append(atemp[0])
sp_user.append(atemp[1])
#
#--- read database
#
line = obs_ss + 'sot_ocat.out'
f = open(line, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
out_list = []
#
#--- read too_list and ddt_list
#
line = too_dir + 'too_list'
fo = open(line, 'r')
too = [line.strip() for line in fo.readlines()]
fo.close()
too_dict = {}
for ent in too:
atemp = re.split('\s+', ent)
too_dict[atemp[2]] = ent
line = too_dir + 'ddt_list'
fo = open(line, 'r')
ddt = [line.strip() for line in fo.readlines()]
fo.close()
ddt_dict = {}
for ent in ddt:
atemp = re.split('\s+', ent)
ddt_dict[atemp[2]] = ent
#
#--- open temporary writing files
#
newf = temp_dir + 'new_obs_list'
out1 = open(newf, 'w')
obsf = temp_dir + 'obs_in_30days'
out2 = open(obsf, 'w')
#
#--- start itelations
#
for ent in data:
atemp = re.split('\^', ent)
try:
status = atemp[16].strip().lower()
date = atemp[13].strip()
except:
continue
#
#--- limit data re-checking to the last 30 days
#
if str(date).lower() == 'null' or str(date).lower() == 'none':
dom = 'NA'
else:
temp = re.split('\s+', str(date))
omon = tcnv.changeMonthFormat(temp[0])
oday = int(temp[1])
oyear = int(temp[2])
dom = tcnv.findDOM(oyear, omon, oday, 0, 0, 0)
ochk = 0
if str(date).lower() == 'null' or str(date).lower() == 'none':
ochk = 1
else:
if dom > dom_limit:
ochk = 1
if ochk == 1 and (status == 'scheduled' or status == 'unobserved' or status == 'observed'):
chk = 0
obsid = atemp[1].strip()
#
#--- check this obsid is listed on a special_obsid_poc_list
#
sp_poc = 'na'
for sval in range(0, len(sp_obsid)):
if obsid == sp_obsid[sval]:
sp_poc = sp_user[sval]
break
#
#--- extract basic information
#
monitor = []
groupid = []
try:
(group_id, pre_id, pre_min_lead, pre_max_lead, grating, type, instrument, obs_ao_str, status, \
seq_nbr, ocat_propid, soe_st_sched_date, lts_lt_plan,targname, object) = sql.get_target_info(int(obsid), monitor,groupid)
if soe_st_sched_date is not None:
date = soe_st_sched_date
chk = 1
elif lts_lt_plan is not None:
date = lts_lt_plan
chk = 1
else:
date = 'NA'
chk = 0
except:
date = 'NA'
chk = 0
#
#--- check status change
#
if status == 'scheduled' or status == 'unobserved' or status == 'observed':
#
#--- recompute with updated date
#
if date == 'NA':
dom = 'NA'
else:
temp = re.split('\s+', str(date))
omon = tcnv.changeMonthFormat(temp[0])
oday = int(temp[1])
oyear = int(temp[2])
dom = tcnv.findDOM(oyear, omon, oday, 0, 0, 0)
#
#--- if it is ddt or too, add to the list anyway
#
if type.lower() == 'ddt' or type.lower() == 'too':
chk = 1
if date == 'NA':
dom = tdom + 1000
#
#--- the observation is cleared all criteria; prepare to print them out to files
#
if chk == 1:
pchk = 1
else:
pchk = 0
continue
if sp_poc != 'na':
#
#-- for the case the obsid is given a specific poc
#
person = sp_poc
elif type.lower() in ('ddt', 'too'):
test = pre_assigned_pos(object, grating)
if test != 0:
person = test
else:
#
#-- too/ddt case: assign poc
#
if date == 'NA':
person = 'TBD'
try:
[person, chk] = tdfnc.find_too_ddt_poc(obsid)
except:
person = 'TBD'
else:
try:
[person, chk] = tdfnc.find_too_ddt_poc(obsid)
except:
person = 'TBD'
#
#--- observed obsid but no poc name -- record mistake; so drop
#
if person == 'TBD':
if status == 'observed':
pchk = 0
else:
#
#--- none too/ddt observations
#
person = tdfnc.match_usint_person(type,grating,int(seq_nbr),instrument, targname)
#
#--- print out to files
#
if pchk == 1:
line = type.lower() + '\t' + str(seq_nbr) + '\t' + obsid + '\t' + status + '\t' \
+ person + '\t' + str(obs_ao_str) + '\t' + str(date) + '\n'
#
#--- if it is ddt or too observation, replace the line in ddt_list or too_list
#
if type.lower() == 'ddt':
try:
line = ddt_dict[obsid]
line = line + '\n'
except:
pass
if type.lower() == 'too':
try:
line = too_dict[obsid]
line = line + '\n'
except:
pass
out1.write(line)
if status != 'observed' and dom < dom30days:
out2.write(line)
out1.close()
out2.close()
completeTask(temp_dir, too_dir, 'new_obs_list')
completeTask(temp_dir, too_dir, 'obs_in_30days')
#
#---- create new_obs_list.txt
#
ofile = too_dir + 'new_obs_list.txt'
out = open(ofile, 'w')
out.write('Type Seq # ObsId Status TOO AO Observation Date\n')
out.write('--------------------------------------------------------------------------\n\n')
out.close()
cmd = 'cat ' + too_dir + 'new_obs_list >> ' + ofile
os.system(cmd)
def acis_dose_make_data_html(indir = 'NA', outdir = 'NA', comp_test = 'NA'):
"""
read data and create html pages
input: indir, outdir, both are optional
"""
#
#--- setting indir and outdir if not given
#
if indir == 'NA':
if comp_test == 'test':
indir = test_data_out
else:
indir = data_out
if outdir == 'NA':
if comp_test == 'test':
outdir = test_data_out
else:
outdir = data_out
#
#--- read data
#
for ccd in ('i_2', 'i_3', 's_2', 's_3'):
for sec in range(0, 4):
inst = ccd + '_n_' + str(sec)
date = []
year = []
month = []
mean_acc = []
std_acc = []
min_acc = []
min_apos = []
max_acc = []
max_apos = []
asig1 = []
asig2 = []
asig3 = []
mean_dff = []
std_dff = []
min_dff = []
min_dpos = []
max_dff = []
max_dpos = []
dsig1 = []
dsig2 = []
dsig3 = []
[date,year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, mean_dff,std_dff, \
min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3] = expf.readExpData(indir, inst)
#
#--- write html page
#
write_html(ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, \
mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3)
out_name = outdir + inst + '.html'
cmd = 'mv acis.html ' + out_name
os.system(cmd)
#
#--- update top html page
#
line = house_keeping + 'exposure.html'
f = open(line, 'r')
data = f.read()
f.close()
now = tcnv.currentTime("Display")
ndate = 'Last Update: ' + now
ndata = data.replace("Last Update:", ndate)
line = web_dir + 'exposure.html'
f = open(line, 'w')
f.write(ndata)
f.close()
#
#--- update plot page htmls
#
update_plt_html_date(comp_test)
def create_monthly(year='', mon=''):
"""
create monthly report
input: year --- year; if it is "", the year of the last month will be used
mon --- month;if it is "", the month value of the last month is used
output: monthly report in /data/mta4/www/REPORTS/MONTLHY/<yyyy><MMM>/
"""
#
#--- if year and month are not given, use the last month's month and year value
#
if year == '':
#
#--- find today's date
#
ltime = tcnv.currentTime()
year = ltime[0]
lyear = str(year) #--- '2016'
#
#--- set the last month's month and year
#
mon = ltime[1] - 1
if mon < 1:
mon = 12
year -= 1
lyear = str(year) #--- '2016'
cmon = str(mon)
if mon < 10:
cmon = '0' + cmon #--- e.g. 03 or 11
lmon = tcnv.changeMonthFormat(mon) #--- e.g. Mar or Nov
lmonyr = lmon.lower() + lyear[2] + lyear[3] #--- e.g. jan16
lm_y = cmon + '_' + lyear #--- e.g. 03_2016
#
#--- set output directory
#
odir = '/data/mta4/www/REPORTS/MONTHLY/' + str(year) + lmon.upper()
cmd = 'mkdir ' + odir
os.system(cmd)
odir = odir + '/'
#
#--- set month interval depending on leap year or not
#
if tcnv.isLeapYear(year) == 0:
sdate = [
'001', '032', '060', '091', '121', '152', '182', '213', '244',
'274', '305', '335'
]
edate = [
'032', '060', '091', '121', '152', '182', '213', '244', '274',
'305', '335', '001'
]
else:
sdate = [
'001', '032', '061', '092', '122', '153', '183', '214', '245',
'275', '306', '336'
]
edate = [
'032', '061', '092', '122', '153', '183', '214', '245', '275',
'306', '336', '001'
]
#
#--- set start and stop time of the month in <yyyy>:<ddd>:00:00:00 format
#
tstart = str(year) + ':' + sdate[mon - 1] + ':00:00:00'
eyear = year
if mon == 12:
eyear = year + 1
tstop = str(eyear) + ':' + edate[mon - 1] + ':00:00:00'
#
#--- create configulation plot
#
cmd = "cd /data/mta/Script/Month/Config/; create_config_plot.py " + tstart + ' ' + tstop
os.system(cmd)
#
#--- create CTI plots
#
os.system(
"cd /data/mta/Script/Month/CTI/; monthly_report_cti_avg_plots.py")
os.system(
"cd /data/mta/Script/Month/CTI/; monthly_report_cti_avg_plots_two_section.py"
)
#
#--- create Focal Plane temperature plots
#
os.system("cd /data/mta/Script/Month/FOCAL/; run_all_focal_scripts")
#
#--- create ACIS SIB plots
#
os.system("cd /data/mta/Script/Month/SIB/; sib_monthly_report_plot.py")
#
#--- create SIM movement plots
#
os.system("cd /data/mta/Script/Month/SIM/; create_monthly_sim_plots")
#
#--- copy the created plots to the report directory
#
cmd = "cp /data/mta/Script/Month/Config/rad_use_*.png " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/CTI/Plots/*png " + odir
os.system(cmd)
cmd = "cp -rf /data/mta/Script/Month/CTI/Data " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/FOCAL/*png " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/FOCAL/Plots/*png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_bad_pixel/Plots/hist_ccd_plot_front_side.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_bad_pixel/Plots/hist_plot_ccd5.png " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/SIB/*png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_max_exp/Images/hrc_max_exp.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_grat/Focus/Plots/acis_hetg_streak_lrf_focus.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_grat/Focus/Plots/acis_hetg_ax_lrf_focus.png " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/SIM/*.png " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_rad/mon_per_diff_last_one_year.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_rad/rad_cnts_" + lmonyr + ".gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/IRU/Plots_new/" + lyear + '/' + lmonyr + "_bias.png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_max_exp/Images/hrc_max_exp.gif " + odir
os.system(cmd)
#
#--- move the plots to past plot saving directories
#
os.system(
"mv -f /data/mta/Script/Month/Config/*.png /data/mta/Script/Month/Config/Plots/."
)
os.system(
"mv -f /data/mta/Script/Month/FOCAL/Plots/*gif /data/mta/Script/Month/FOCAL/Plots/Past/."
)
os.system(
"mv -f /data/mta/Script/Month/SIB/*png /data/mta/Script/Month/SIB/Plots/."
)
#
#--- sun spot cycle download
#
cmd = 'wget -q -O' + odir + '/solar-cycle-sunspot-number.gif http://services.swpc.noaa.gov/images/solar-cycle-sunspot-number.gif'
os.system(cmd)
#
#--- now copy month depend plot files and set the template file for the month
#
if mon == 1 or mon == 7:
cmd = "cp /data/mta_www/mta_acis_sci_run/Corner_pix/Trend_Plots/I3cp.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_sci_run/Corner_pix/Trend_Plots/S3cp.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_bias_bkg/Plots/Overclock/ccd2.png " + odir + 'ccd2_oc.png'
os.system(cmd)
cmd = "cp /data/mta_www/mta_bias_bkg/Plots/Sub/ccd2.png " + odir + 'ccd2_sub.png'
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_2.png " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_6.png " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_7.png " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_11.png " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY1.html' #--- template file name
elif mon == 2 or mon == 8:
cmd = "cp /data/mta4/www/DAILY/mta_src/ACIS-I_d_tyear10_psf.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_xy.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_rnd.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_rnd.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_ravg.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_ravg.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_snr.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_snr.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_rot.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_rot.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_aiming/Fig_save/acis_point_err.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_aiming/Fig_save/hrc_i_point_err.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_aiming/Fig_save/hrc_s_point_err.gif " + odir
os.system(cmd)
cmd = "cp -r /data/mta4/www/DAILY/mta_src/Plots " + odir + 'HRMA_Plots'
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY2.html'
elif mon == 3 or mon == 9:
cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD3_rej_cti.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD3_rej_obs.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD7_rej_cti.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD7_rej_obs.gif " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_acis_gain/Plots/gain_plot_ccd3.png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_acis_gain/Plots/offset_plot_ccd3.png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_acis_gain/Plots/gain_plot_ccd5.png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_acis_gain/Plots/offset_plot_ccd5.png " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY3.html'
elif mon == 4 or mon == 10:
#cmd = "cp /data/mta_www/mta_grat/EdE/heg_all.gif " + odir
#os.system(cmd)
#cmd = "cp /data/mta_www/mta_grat/EdE/meg_all.gif " + odir
#os.system(cmd)
#cmd = "cp /data/mta_www/mta_grat/EdE/leg_all.gif " + odir
#os.system(cmd)
cmd = "cp /data/mta_www/mta_grat/EdE/*_2019.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sim_twist/Plots/twist_plot.png " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY4.html'
elif mon == 5 or mon == 11:
cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd3.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd5.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd7.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_pos.html " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_width.html " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_cnt.html " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY5.html'
elif mon == 6 or mon == 12:
cmd = "cp /data/mta_www/mta_sim_twist/Plots/I-1.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sim_twist/Plots/S-2.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sim_twist/Plots/H-I-2.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sim_twist/Plots/H-S-2.png " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY6.html'
#
#--- create clean acis and hrc exposure maps using ds9
#
run_exposure_maps(lyear, cmon)
#--------------------------------------------------
#--- read the template and substitute the contents
#--------------------------------------------------
fx = open(file, 'r')
text = fx.read()
fx.close()
#
#--- substitute values
#
fmonth_list = [
'January', 'February', 'March', 'April', 'May', 'June', 'July',
'August', 'September', 'October', 'November', 'December'
]
fmon = fmonth_list[mon - 1]
text = text.replace('#Month#', fmon) #--- full month name e.g. May
text = text.replace('#YEAR#', lyear) #--- full year nane e.g. 2016
text = text.replace('#SMON#', lmon) #--- short month, e.g. Jan
text = text.replace('#LSMON#',
lmon.lower()) #--- short lower month, e.g. jan
text = text.replace('#USMON#',
lmon.upper()) #--- short upper month, e.g. JAN
line = 'rad_use_' + lmonyr + '.png'
text = text.replace('#RADUSE#', line) #--- e.g., rad_use_oct16.png
line = cmon + '_' + lyear
text = text.replace('#LMONYR#', line) #--- mon_year e.g. 07_2016
#
#--- acis exposure tables
#
os.system(
"cd /data/mta/Script/Exposure/Exc/; /data/mta/Script/Exposure/Scripts/ACIS_Scripts/acis_dose_monthly_report.py"
)
line = '/data/mta/Script/Exposure/Exc/monthly_diff_' + cmon + '_' + lyear
f = open(line, 'r')
dout = f.read()
f.close()
text = text.replace('#ACISMON#', dout) #--- acis monthly dose
line = '/data/mta/Script/Exposure/Exc/monthly_acc_' + cmon + '_' + lyear
f = open(line, 'r')
dout = f.read()
f.close()
text = text.replace('#ACISCMON#', dout) #--- acis cumulative dose
#
#--- last 12 months exposure maps
#
byear = year - 1
text = past_data_entry(1, 0, mon, byear, text)
text = past_data_entry(2, 3, mon, byear, text)
text = past_data_entry(3, 6, mon, byear, text)
text = past_data_entry(4, 9, mon, byear, text)
#
#---- acis focal
#
fx = open('/data/mta/Script/Month/FOCAL/Plots/month_avg', 'r')
data = [line.strip() for line in fx.readlines()]
fx.close()
atemp = re.split(':', data[0])
btemp = re.split('\+\/\-', atemp[1])
ft = round(float(btemp[0]), 2)
ferr = round(float(btemp[1]), 2)
text = text.replace('#FT#', str(ft))
text = text.replace('#FERR#', str(ferr))
atemp = re.split(':', data[1])
btemp = re.split('\+\/\-', atemp[1])
fw = round(float(btemp[0]), 2)
werr = round(float(btemp[1]), 2)
text = text.replace('#FW#', str(fw))
text = text.replace('#WERR#', str(werr))
line = 'erad_' + lmonyr + '.gif' #--- erand_nov16.gif
text = text.replace('#ERAND#', line)
#
#--- acis sib
#
yearmon = str(year) + '_' + cmon
text = text.replace('#YEARMON#', yearmon) #--- e.g. 2015_03
#
#--- hrc i monthly dose
#
line = hrc_monthly_report('HRCI', cmon, lmon, lyear)
text = text.replace('#HRCIDOSE#', line) #--- HRC I DIFF
#
#--- hrc cumulative
#
line = hrc_cumulative_report('HRCI', cmon, lmon, lyear)
text = text.replace('#CHRCIDOSE#', line) #--- HRC I Cumulative
#
#--- hrc s monthly dose
#
line = hrc_monthly_report('HRCS', cmon, lmon, lyear)
text = text.replace('#HRCSDOSE#', line) #--- HRC S DIFF
#
#--- hrc cumulative
#
line = hrc_cumulative_report('HRCS', cmon, lmon, lyear)
text = text.replace('#CHRCSDOSE#', line) #--- HRC S Cumulative
#
#--- IRU
#
text = text.replace("#SMONYR#", lmonyr) #--- e.g., nov16
#
#--- envelope trending
#
line = get_envelope_trending(mon, odir)
text = text.replace("#ENVTREND#", line)
#
#--- critical trends which are reported on Mar, Jun, Sep, and Dec
#
if mon in [3, 6, 9, 12]:
text = run_critical_trend(text)
#
#--- Monthly Trend Reports
#
text = run_month_trend(mon, text)
#
#--- last index
#
line = create_index(year, mon)
text = text.replace('#YINDEX#', line)
#
#--- finally print out the report
#
ofile = odir + "MONTHLY.html"
fo = open(ofile, 'w')
fo.write(text)
fo.close()
#
#--- chnage permission and owners
#
cmd = 'chmod 755 ' + odir + '/*'
os.system(cmd)
cmd = 'chgrp -R mtagroup ' + odir + '/*'
os.system(cmd)
def report_error():
"""
read errors from <cup_usr_list>_error_list, sort it out, clean, and send out email
Input: none but read from <cup_usr_list>_error_list
Output: email sent out
"""
#
#--- find the current time
#
[year, mon, day, hours, min, sec, weekday, yday,
dst] = tcnv.currentTime("Local")
#
#--- create surfix for files which will be saved in Past_errors directory
#
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(day)
if day < 10:
sday = '0' + sday
tail = str(year) + smon + sday
for tag in cpu_usr_list:
efile = house_keeping + 'Records/' + tag + '_error_list'
pfile = house_keeping + 'Records/Past_errors/' + tag + '_error_list_' + tail
prev_line = ''
chk = mcf.chkFile(efile)
if chk > 0:
#
#--- read error messages from the file
#
f = open(efile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- sort the data so that we can correct messages to each cron job together
#
data.sort()
task_list = []
time_list = []
mssg_list = []
for ent in data:
atemp = re.split(' : ', ent)
task_list.append(atemp[0])
stime = int(atemp[1])
dtime = tcnv.axTimeMTA(stime)
time_list.append(dtime)
mssg_list.append(atemp[2])
#
#--- write out cron job name
#
fo = open(zspace, 'w')
cname = task_list[0]
line = '\n\n' + cname + '\n____________________\n\n'
fo.write(line)
for i in range(0, len(mssg_list)):
if task_list[i] != cname:
cname = task_list[i]
line = '\n\n' + cname + '\n____________________\n\n'
fo.write(line)
#
#--- create each line. if it is exactly same as one line before, skip it
#
line = time_list[i] + ' : ' + mssg_list[i] + '\n'
if line != prev_line:
fo.write(line)
prev_line = line
fo.close()
#
#--- send email out
#
# cmd = 'cp ' + zspace + ' ' + '/data/mta/Script/Cron_check/Scripts/' + tag
# os.system(cmd)
send_mail(tag, email_list)
#
#--- move the error list to Past_errors directory
#
cmd = 'mv ' + efile + ' ' + pfile
os.system(cmd)
def clean_table():
"""
sort and clean the table data in <data_dir>/Results
Input: none but read from <data_dir>/Results/<elm>_ccd<ccd#>
Output: cleaned up <data_dir>/Results/<elm>_ccd<ccd#>
"""
#
#--- make a backup
#
atime = tcnv.currentTime(format='UTC')
tyear = str(atime[0])
syear = tyear[2] + tyear[3]
tmon = atime[1]
smon = str(tmon)
if int(tmon) < 10:
smon = '0' + smon
tday = atime[2]
sday = str(tday)
if int(tday) < 10:
sday = '0' + sday
cout = data_dir + '/Results/Save_' + smon + sday + syear
mcf.mk_empty_dir(cout)
cmd = 'cp -f ' + data_dir + '/Results/*_ccd* ' + cout + '/.'
os.system(cmd)
#
#--- now clean up the data
#
cmd = 'ls ' + data_dir + '/Results/*_ccd* > ' + zspace
os.system(cmd)
f = open(zspace, 'r')
flist = [line.strip() for line in f.readlines()]
f.close()
mcf.rm_file(zspace)
for file in flist:
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
data.sort()
prev = data[0]
cleaned = [prev]
for ent in data:
if ent == prev:
prev = ent
else:
cleaned.append(ent)
prev = ent
fo = open(file, 'w')
for ent in cleaned:
line = ent + '\n'
fo.write(line)
fo.close()