def mv_old_file(dom):
dom -= 30
if dom > 0:
[year, ydate] = tcnv.DOMtoYdate(dom)
sydate = str(ydate)
if ydate < 10:
sydate = '00' + sydate
elif ydate < 100:
sydate = '0' + sydate
atime = str(year) + ':' + sydate + ':00:00:00'
stime = tcnv.axTimeMTA(atime)
cmd = 'ls ' + house_keeping + '/Defect/CCD*/* > ' + zspace
os.system(cmd)
fs = open(zspace, 'r')
ldata = [line.strip() for line in fs.readlines()]
fs.close()
mcf.rm_file(zspace)
for ent in ldata:
atemp = re.split('\/acis', ent)
btemp = re.split('_', atemp[1])
if int(btemp[0]) < stime:
out = ent
out = out.replace('Defect', 'Defect/Save')
cmd = 'mv ' + ent + ' ' + out
os.system(cmd)
def stime_to_ddate(stime):
"""
change data in second from 1998.1.1 to mm/dd/yy format
input: stime --- time in seconds from 1998.1.1
output: dtime --- date in the form of mm/dd/yy (e.g. 08/19/15)
"""
tlist = tcnv.axTimeMTA(stime)
atemp = re.split(':', tlist)
year = int(float(atemp[0]))
ydate = int(float(atemp[1]))
[mon, date] = tcnv.changeYdateToMonDate(year, ydate)
lyear = str(atemp[0])
syr = lyear[2] + lyear[3]
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(date)
if date < 10:
sday = '0' + sday
dtime = smon + '/' + sday + '/' + syr
return dtime
def stime_to_ddate2(stime):
"""
change data in second from 1998.1.1 to yyyymmdd format
input: stime --- time in seconds from 1998.1.1
output: dtime --- date in the form of yyyymmdd (e.g. 20150819)
"""
tlist = tcnv.axTimeMTA(stime)
atemp = re.split(':', tlist)
year = int(float(atemp[0]))
lyear = str(atemp[0])
ydate = int(float(atemp[1]))
[mon, date] = tcnv.changeYdateToMonDate(year, ydate)
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(date)
if date < 10:
sday = '0' + sday
dtime = lyear + smon + sday
return dtime
def get_data_period():
"""
set start and stop time of the data period
input: none
output: [start, stop] --- in format of mm/dd/yy
"""
#
#--- find the last date of the data update
#
maxt = 0
for dname in ('I2cp.dat', 'I3cp.dat', 'S2cp.dat', 'S3cp.dat'):
file = web_dir + dname
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
atemp = re.split('\s+', data[-1])
val = float(atemp[0])
if val > maxt:
maxt = int(val)
atemp = tcnv.axTimeMTA(str(maxt))
ttemp = re.split(':', atemp)
year = int(float(ttemp[0]))
ydate = int(float(ttemp[1]))
(month, mdate) = tcnv.changeYdateToMonDate(year, ydate)
smon = str(month)
if month < 10:
smon = '0' + smon
sday = str(mdate)
if mdate < 10:
sday = '0' + sday
start = smon + '/' + sday + '/' + ttemp[0][2] + ttemp[0][3]
#
#--- find today's date; it will be the last date of the period
#
ctime = datetime.datetime.now()
syear = str(ctime.year)
month = ctime.month
cmon = str(month)
if month < 10:
cmon = '0' + cmon
date = ctime.day
cday = str(date)
if date < 10:
cday = '0' + cday
stop = cmon + '/' + cday + '/' + syear[2] + syear[3]
return [start, stop]
def read_rad_zone_data(start, stop):
"""
extract radiation zone information for a given time span
input: start --- starting time
stop --- stopping time
data are read from /data/mta/Script/Interrupt/house_keeping/rad_zone_info
output: rad_start --- a list of radiation zone starting time
rad_stop --- a list of radiation zone stopping time
"""
f = open('/data/mta/Script/Interrupt/house_keeping/rad_zone_info', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
rad_start = []
rad_stop = []
rad_in = 0
for ent in data:
atemp = re.split('\s+', ent)
try:
val = float(atemp[1]) #---- check time is in digit (dom)
except:
continue
[year, ydate] = tcnv.DOMtoYdate(float(atemp[1]))
#
#--- convert time to sec from 1998.1.1
#
day_part = int(ydate)
rest = ydate - day_part
hour = int(24 * rest)
rest = 24 * rest - hour
minutes = int(60 * rest)
ltime = str(year) + ':' + str(day_part) + ':' + str(hour) + ':' + str(minutes) + ':00'
stime = tcnv.axTimeMTA(ltime)
if stime >= start and stime < stop:
if atemp[0] == 'ENTRY' and rad_in == 0:
rad_start.append(stime)
rad_in = 1
elif atemp[0] == 'EXIT' and rad_in == 1:
rad_stop.append(stime)
rad_in = 0
elif stime >=stop:
break
else:
continue
if len(rad_stop) < len(rad_start):
rad_stop.append(stop)
return [rad_start, rad_stop]
def stime_to_ydate(stime):
"""
convert time in sec from 1998.1.1 to ydate. no year info
input: stime ---- time in sec from 1998.1.1
output: ydate ---- ydate
"""
time = tcnv.axTimeMTA(int(stime))
atemp = re.split(':', time)
ydate = float(atemp[1]) + float(atemp[2])/24.0 + float(atemp[3])/1440.0 + float(atemp[4]) / 86400.0
return ydate
def sec1998tofracday(stime):
"""
convert time from seconds from 1998.1.1 to fractional yday
input: stime --- time in seconds from 1998.1.1
output: lday --- fractional year date. igore year
"""
ptime = tcnv.axTimeMTA(stime)
atemp = re.split(':', ptime)
day = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
dtime = day + hh/24.0 + mm/ 1440.0
lday = "%.2f" % round(dtime, 2)
return lday
def current_time():
"""
THIS FUNCTION IS NOT USED IN THIS SCRIPT ANYMORE
find current time in seconds from 1998.1.1
input: none
output: dtime --- the current time in seconds from 1998.1.1
"""
stime = time.time()
out = time.gmtime()
stime -= tcorrect
ct = str(out[0]) + ':' + str(out[7]) + ':' + str(out[3]) + ':' + str(out[4]) + ':' + str(out[5])
dtime = tcnv.axTimeMTA(ct)
return dtime
def make_data_list(year, mon, day, tail):
"""
make a data list from the last entry date to the most current data
input: year --- the year of the last entry
mon --- the month of the last entry
day --- the day of month of the last entry
tail --- the suffix of the data file
output dlist --- a list of data names
"""
#
#--- convert the date into seconds from 1998.1.1
#
dst = tcnv.convertDateToTime2(year, mon, day)
#
#--- find today's time
#
today = tcnv.currentTime()
cyear = today[0]
cmon = today[1]
cday = today[2]
cdst = tcnv.convertDateToTime2(cyear, cmon, cday)
dlist = []
#
#--- check the current date is larger than the date indicated
#--- if so find how many days between and retrieve data for each day
#
if cdst > dst:
step = int(( cdst - dst)/86400)
if step >= 1:
head = make_header(year, mon, day)
name = head + tail
dlist.append(name)
for i in range(2, step):
sdate = int(cdst - 86400.0 * i)
out = tcnv.axTimeMTA(sdate)
atemp = re.split(':', out)
year = int(atemp[0])
ydate = int(atemp[1])
[mon, day] = tcnv.changeYdateToMonDate(year, ydate)
head = make_header(year, mon, day)
name = head + tail
dlist.append(name)
return dlist
def create_display_data_table():
"""
create a readable data table for html page
Input: none, but read from <data_dir>/ccd<ccd>_<node>
Output: <web_dir>/ccd<ccd>_<node>
"""
for ccd in range(0, 10):
for node in range(0, 4):
file = 'ccd' + str(ccd) + '_' + str(node)
infile = data_dir + file
outfile = web_dir + 'Data/' + file
f = open(infile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
fo = open(outfile, 'w')
#
#--- adding heading
#
line = "#\n#Date Mn K alpha Al K alpha Ti K alpha Slope Sigma Int Sigma\n#\n"
fo.write(line)
for ent in data:
atemp = re.split('\s+', ent)
stime = int(atemp[0])
#
#--- converting the date into <mon> <year> form (e.g. May 2013)
#
ltime = tcnv.axTimeMTA(stime)
btemp = re.split(':', ltime)
year = btemp[0]
[mon, mdate] = tcnv.changeYdateToMonDate(int(year), int(btemp[1]))
lmon = tcnv.changeMonthFormat(mon)
line = lmon + ' ' + year
for j in range(1, len(atemp)):
line = line + '\t' + atemp[j]
line = line + '\n'
fo.write(line)
fo.close()
def extract_sim_data():
"""
extract sim data from PRIMARYCCDM_*.*.tl
input: none but read from <dumpdir>/PRIMARYCCDM_*.*.tl
output: <outdir>sim_data.out
"""
#
#--- find the time of the last entry from the sim_data.out
#
sfile = outdir + 'sim_data.out'
f = open(sfile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- cleaning up the data; drop the data which the date starts from ":" e.g. :2014
#
pdata = []
for ent in data:
if re.search('^:', ent):
continue
else:
pdata.append(ent)
#
#--- the last entiry values
#
if len(pdata) > 0:
atemp = re.split('\s+', pdata[len(pdata)-1])
ltime = tcnv.axTimeMTA(atemp[0]) #--- converting time to sec from 1998.1.1
time_2 = atemp[0]
col1_2 = atemp[1]
col2_2 = atemp[2]
col3_2 = atemp[3]
else:
ltime = 0
time_2 = 0
col1_2 = ''
col2_2 = ''
col3_2 = ''
#
#--- check whether input files exists
#
cmd = 'ls -rt ' + dumpdir + 'PRIMARYCCDM_*.*.tl >' + zspace
os.system(cmd)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + zspace
os.system(cmd)
dlen = len(data)
if dlen < 1:
exit(1)
#
#--- files exist. read the data from the last 10 files
#
tlist = data[dlen-40:]
for ent in tlist:
cmd = 'cat ' +ent + ' >> ' + zspace
os.system(cmd)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + zspace
os.system(cmd)
prev = ''
fo = open('./temp_save', 'w')
#
#--- go though each data line
#
for ent in data:
try:
#
#--- expect the first letter of the data line is numeric (e.g. 2014).
#
val = float(ent[0])
except:
continue
#
#--- only data with "FMT" format will be used
#
mc = re.search('FMT', ent)
if mc is None:
continue
atemp = re.split('\t+', ent)
#
#--- if there are less than 20 entries, something wrong; skip it
#
if len(atemp) < 20:
continue
#
#--- convert time format
#
time = atemp[0]
time = time.strip();
time = time.replace(' ', ':')
time = time.replace(':::', ':00')
time = time.replace('::', ':0')
#
#--- if the time is exactly same as one before, skip it
#
if time == time_2:
continue
#
#--- if the time is already in the database, keip it
#
stime = tcnv.axTimeMTA(time)
if stime <= ltime:
continue
#
#--- use only data which tscpos and fapos have numeric values
#
tscpos = atemp[4].strip()
fapos = atemp[5].strip()
if tscpos == "" or fapos == "":
continue
else:
tscpos = int(float(tscpos))
fapos = int(float(fapos))
# aopc = atemp[11].strip()
# if aopc == '':
# aopc = '0'
mpwm = atemp[12].strip()
if mcf.chkNumeric(mpwm):
mpwm = int(float(mpwm))
mpwm = str(mpwm)
else:
mpwm = '0'
#
#--- we want to print only beginning and ending of the same data entries.
#--- skip the line if all three entiries are same as one before, except the last one
#
if col1_2 == tscpos and col2_2 == fapos and col3_2 == mpwm:
time_2 = time
continue
line = time + '\t' + str(tscpos) + '\t' + str(fapos) + '\t' + mpwm + '\n'
if line == prev:
continue
else:
pline = time_2 + '\t' + str(col1_2) + '\t' + str(col2_2) + '\t' + str(col3_2) + '\n'
fo.write(pline)
fo.write(line)
prev = line
time_2 = time
col1_2 = tscpos
col2_2 = fapos
col3_2 = mpwm
fo.close()
sfile2 = sfile + '~'
cmd = 'cp ' + sfile + ' ' + sfile2
os.system(cmd)
cmd = 'cat ./temp_save >> ' + sfile
os.system(cmd)
def acis_gain_plot_trend():
"""
plotting trends of gain and offset
Input: none, but read from <data_dir>
Output: <web_dir>/Plots/gain_plot_ccd<ccd>.png
<web_dir>/Plots/offset_plot_ccd<ccd>.png
"""
for ccd in range(0, 10):
#
#--- plotting 4 nodes on one panel, but gain and offset separately
#
Xset_gain = []
Yset_gain = []
Eset_gain = []
yMin_gain = []
yMax_gain = []
Label_gain = []
Xset_offset = []
Yset_offset = []
Eset_offset = []
yMin_offset = []
yMax_offset = []
Label_offset = []
for node in range(0, 4):
#
#--- read data for given CCD and Node #
#
file = data_dir + 'ccd' + str(ccd) + '_' + str(node)
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
time = []
gain = []
gerr = [] #--- error for gain
offset = []
oerr = [] #--- error for offset
#
#--- setting lower and upper limits to remove outlyers
#
sum = 0
sum2 = 0
for ent in data:
atemp = re.split('\s+', ent)
gval = float(atemp[4])
sum += gval
sum2 += gval * gval
avg = sum / len(data)
sig = math.sqrt(sum2 / len(data) - avg * avg)
blim = avg - 3.0 * sig
tlim = avg + 3.0 * sig
for ent in data:
atemp = re.split('\s+', ent)
#
#--- convert time into year date (e.g.2012.14)
#
gval = float(atemp[4])
if (gval <= blim) or (gval >= tlim):
continue
stime = tcnv.axTimeMTA(int(atemp[0]))
btemp = re.split(':', stime)
year = int(btemp[0])
ydate = int(btemp[1])
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366.0
else:
base = 365.0
ytime = float(year) + float(ydate) / base
time.append(ytime)
gain.append(float(atemp[4]))
gerr.append(float(atemp[5]))
offset.append(float(atemp[6]))
oerr.append(float(atemp[7]))
xmax = max(time)
Xset_gain.append(time)
Yset_gain.append(gain)
Eset_gain.append(gerr)
#
#--- set plotting range
#
avg = mean(gain)
ymin = avg - 0.002
ymin = round(ymin, 3) - 0.001
ymax = avg + 0.002
ymax = round(ymax, 3) + 0.001
yMin_gain.append(ymin)
yMax_gain.append(ymax)
name = 'Gain (ADU/eV) Node' + str(node)
Label_gain.append(name)
Xset_offset.append(time)
Yset_offset.append(offset)
Eset_offset.append(oerr)
avg = mean(offset)
ymin = avg - 8.0
ymin = int(ymin)
ymax = avg + 8.0
ymax = int(ymax)
yMin_offset.append(ymin)
yMax_offset.append(ymax)
name = 'Offset (ADU) Node' + str(node)
Label_offset.append(name)
xmin = int(2000)
xmax = int(xmax) + 1
xname = 'Time (year)'
#
#--- actual plotting starts here
#
yname = 'Gain'
plotPanel(xmin, xmax, yMin_gain, yMax_gain, Xset_gain, Yset_gain,
Eset_gain, xname, yname, Label_gain)
outname = web_dir + "/Plots/gain_plot_ccd" + str(ccd) + '.png'
cmd = 'mv out.png ' + outname
os.system(cmd)
yname = 'Offset'
plotPanel(xmin, xmax, yMin_offset, yMax_offset, Xset_offset,
Yset_offset, Eset_offset, xname, yname, Label_offset)
outname = web_dir + "/Plots/offset_plot_ccd" + str(ccd) + '.png'
cmd = 'mv out.png ' + outname
os.system(cmd)
def plot_data(start, stop, mag_plot=1):
"""
create a configulation display panel for a give time period
input: start --- starting time in sec from 1998.1.1
stop --- stopping time in sec from 1998.1.1
"""
#
#---- set a few parameters
#
if mag_plot == 0:
pnum = 5
else:
pnum = 6
mpl.rcParams['font.size'] = 11
mpl.rcParams['font.weight'] = 'strong'
props = font_manager.FontProperties(size=6)
plt.subplots_adjust(hspace=0.05)
plt.subplots_adjust(wspace=0.12)
#
#--- set a few others
#
xpos = stime_to_ydate(stop) + 0.1
ystep = 0.2
#
#--- read sim information
#
[acis_i_start, acis_i_stop, acis_s_start, \
acis_s_stop, hrc_i_start, hrc_i_stop, \
hrc_s_start, hrc_s_stop, hetg_start, \
hetg_stop, letg_start, letg_stop, \
radmon_start, radmon_stop, fmt, time] = erd.find_sim_position(start, stop)
#
#--- hetg /letg information plot
#
start_set = [hetg_start, letg_start]
stop_set = [hetg_stop, letg_stop]
ax1 = plt.subplot(pnum, 1, 1)
plot_strip_box(ax1,start, stop, start_set, stop_set, color1)
plt.text(xpos, 0.8, "HETG", color=color1[0])
plt.text(xpos, 0.6, "LETG", color=color1[1])
#
#--- acis /hrc information plot
#
start_set = [acis_i_start, acis_s_start, hrc_i_start, hrc_s_start]
stop_set = [acis_i_stop, acis_s_stop, hrc_i_stop, hrc_s_stop]
ax2 = plt.subplot(pnum, 1, 2)
plot_strip_box(ax2,start, stop, start_set, stop_set, color1)
plt.text(xpos, 0.9, "ACIS I ", color=color1[0])
plt.text(xpos, 0.7, "ACIS S ", color=color1[1])
plt.text(xpos, 0.5, "HRC I ", color=color1[2])
plt.text(xpos, 0.3, "HRC S ", color=color1[3])
#
#--- cti information plot
#
[cti_start, cti_stop] = erd.read_ccd_data(start, stop)
start_set = [cti_start]
stop_set = [cti_stop]
ax3 = plt.subplot(pnum, 1, 3)
plot_strip_box(ax3, start, stop, start_set, stop_set, color1)
#
#--- altitude information plot
#
[atime, alt, magx, magy, magz, crm] = erd.read_orbit_data(start, stop)
plot_line(ax3, start, stop, atime, alt)
plt.text(xpos, 0.8, "CTI ", color=color1[0])
plt.text(xpos, 0.7, "Check ", color=color1[0])
plt.text(xpos, 0.4, "Altitude ", color="green")
#
#--- radmon information plot
#
start_set = [radmon_start]
stop_set = [radmon_stop]
ax4 = plt.subplot(pnum, 1, 4)
plot_strip_box(ax4, start, stop, start_set, stop_set, color1)
#
#--- hrc sheild rate plot
#
[htime, rate] = erd.read_hrc_data(start, stop)
plot_line(ax4, start, stop, htime, rate)
#
#--- goes p3 rate plot
#
# [gtime, p1, p2, p3] = erd.read_goes_data(start, stop)
# plot_points(ax4, start, stop, gtime, p3, color='lime', pts=0.5,lw=0)
plt.text(xpos, 0.8, "Radmon", color=color1[0])
plt.text(xpos, 0.6, "HRC", color="green")
plt.text(xpos, 0.5, "Shield", color="green")
plt.text(xpos, 0.4, "Rate", color="green")
# plt.text(xpos, 0.2, "GOES P3", color="green")
#
#--- magnetsphere plot
#
if mag_plot != 0:
[start_set, stop_set] = find_mag_region(start, stop)
axm = plt.subplot(6, 1, 5)
plot_strip_box(axm, start, stop, start_set, stop_set, color1)
# plt.text(xpos, 0.9, "Solar", color=color1[0])
# plt.text(xpos, 0.8, "Wind", color=color1[0])
plt.text(xpos, 0.6, "Magneto-", color=color1[1])
plt.text(xpos, 0.5, "sheath", color=color1[1])
plt.text(xpos, 0.3, "Magneto-", color=color1[2])
plt.text(xpos, 0.2, "sphere", color=color1[2])
#
#--- often the data are not available; so make a note on the plot
#
# diff1 = stop - start
# tlast = time[len(time)-1]
# diff2 = tlast - start
# ratio = diff2 / diff1
# if ratio < 0.7:
# xnote = 0.5 * (stop - tlast)
# plt.text(xnote, 0.5, "No Data", color='maroon')
#
#--- FMT format information plot
#
[start_set, stop_set] = find_fmt_region(start, stop, fmt, time)
ax5 = plt.subplot(pnum, 1, pnum)
plot_strip_box(ax5, start, stop, start_set, stop_set,color4)
plt.text(xpos, 0.9, "FMT1", color=color4[0])
plt.text(xpos, 0.7, "FMT2", color=color4[1])
plt.text(xpos, 0.5, "FMT3", color=color4[2])
plt.text(xpos, 0.3, "FMT4", color=color4[3])
plt.text(xpos, 0.1, "FMT5", color=color4[4])
#
#--- plot x axis tick label only at the bottom ones
#
if mag_plot == 0:
ax_list = [ax1, ax2, ax3, ax4, ax5]
else:
ax_list = [ax1, ax2, ax3, ax4, ax5, axm]
for ax in ax_list:
for label in ax.get_yticklabels():
label.set_visible(False)
if ax != ax5:
for label in ax.get_xticklabels():
label.set_visible(False)
else:
pass
#
#--- x axis label
#
mid = int(0.5 * (start + stop))
ltime = tcnv.axTimeMTA(mid)
atemp = re.split(':', ltime)
year = int(atemp[0])
ydate = int(atemp[1])
[mon, day] = tcnv.changeYdateToMonDate(year,ydate)
xlabel = "Time (DOY) " + str(atemp[0])
ax5.set_xlabel(xlabel)
#
#--- set the size of the plotting area in inch (width: 10.0in, height 5.0in)
#
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(12.0, 10.0)
#
#--- save the plot in png format
#
syear = str(year)
smon = tcnv.changeMonthFormat(mon)
smon = smon.lower()
outname = 'rad_use_' + smon + syear[2] + syear[3] + '.png'
plt.savefig(outname, format='png', dpi=100)
plt.close('all')
def report_error():
"""
read errors from <cup_usr_list>_error_list, sort it out, clean, and send out email
Input: none but read from <cup_usr_list>_error_list
Output: email sent out
"""
#
#--- find the current time
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime("Local")
#
#--- create surfix for files which will be saved in Past_errors directory
#
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(day)
if day < 10:
sday = '0' + sday
tail = str(year) + smon + sday
for tag in cpu_usr_list:
efile = house_keeping + 'Records/' + tag + '_error_list'
pfile = house_keeping + 'Records/Past_errors/' + tag + '_error_list_' + tail
prev_line = ''
chk = mcf.chkFile(efile)
if chk > 0:
#
#--- read error messages from the file
#
f = open(efile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- sort the data so that we can correct messages to each cron job together
#
data.sort()
task_list = []
time_list = []
mssg_list = []
for ent in data:
atemp = re.split(' : ' , ent)
task_list.append(atemp[0])
stime = int(atemp[1])
dtime = tcnv.axTimeMTA(stime)
time_list.append(dtime)
mssg_list.append(atemp[2])
#
#--- write out cron job name
#
fo = open(zspace, 'w')
cname = task_list[0]
line = '\n\n' + cname + '\n____________________\n\n'
fo.write(line)
for i in range(0, len(mssg_list)):
if task_list[i] != cname:
cname = task_list[i]
line = '\n\n' + cname + '\n____________________\n\n'
fo.write(line)
#
#--- create each line. if it is exactly same as one line before, skip it
#
line = time_list[i] + ' : ' + mssg_list[i] + '\n'
if line != prev_line:
fo.write(line)
prev_line = line
fo.close()
#
#--- send email out
#
# cmd = 'cp ' + zspace + ' ' + '/data/mta/Script/Cron_check/Scripts/' + tag
# os.system(cmd)
send_mail(tag, email_list)
#
#--- move the error list to Past_errors directory
#
cmd = 'mv ' + efile + ' ' + pfile
os.system(cmd)
#
#--- pylab plotting routine related modules
#
from pylab import *
import matplotlib.pyplot as plt
import matplotlib.font_manager as font_manager
import matplotlib.lines as lines
from matplotlib.transforms import Bbox
mag_plot = 1
if len(sys.argv) == 3:
start = sys.argv[1]
stop = sys.argv[2]
else:
print "Input: start stop in the format of 2014:204:00:00:00 or in seconds from 1998.1.1"
exit(1)
if __name__ == "__main__":
try:
start = int(float(start))
stop = int(float(stop))
except:
start = tcnv.axTimeMTA(start)
stop = tcnv.axTimeMTA(stop)
plot_data(start, stop, mag_plot)
def sci_run_add_to_rad_zone_list(file='NA'):
"""
adding radiation zone list to rad_zone_list.
input: file --- the name of inputfile containing:
e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto
output <house_keeping>/rad_zone_list
"""
#
#--- check whether the list alread exists; if it does, read which ones are already in the list
#
cmd = 'ls ' + house_keeping + '* > ./ztemp'
os.system(cmd)
test = open('./ztemp').read()
os.system('rm -f ./ztemp')
m1 = re.search('rad_zone_list', test)
m2 = re.search('rad_zone_list~', test)
eventList = []
echk = 0
if m1 is not None:
line = house_keeping + 'rad_zone_list'
f = open(line, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
for ent in data:
atemp = re.split('\s+|\t+', ent)
eventList.append(atemp[0])
echk = 1
#
#--- if file is not given (if it is NA), ask the file input
#
if file == 'NA':
file = raw_input('Please put the intrrupt timing list: ')
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- put the list in the reverse order
#
data.reverse()
for ent in data:
if not ent:
break
#
#--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30)
#--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list
#--- to each date
#
etemp = re.split('\s+', ent)
echk = 0
for comp in eventList:
if comp == etemp[0]:
echk = 1
break
if echk == 0:
atemp = re.split(':', etemp[1])
year = atemp[0]
month = atemp[1]
date = atemp[2]
hour = atemp[3]
mins = atemp[4]
#
#--- convert to dom/sec1998
#
ydate = tcnv.findYearDate(int(year), int(month), int(date))
dom = tcnv.findDOM(int(year), int(ydate))
line = year + ':' + str(int(ydate)) + ':' + hour + ':' + mins + ':00'
csec = tcnv.axTimeMTA(line)
#
#--- end date
#
atemp = re.split(':', etemp[2])
eyear = atemp[0]
emonth = atemp[1]
edate = atemp[2]
ehour = atemp[3]
emins = atemp[4]
ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate))
line = eyear + ':' + str(int(ydate)) + ':' + ehour + ':' + emins + ':00'
csec2 = tcnv.axTimeMTA(line)
#
#--- date stamp for the list
#
list_date = str(year) + str(month) + str(date)
#
#--- check radiation zones for 3 days before to 5 days after from the interruptiondate
#--- if the interruption lasted longer than 5 days, extend the range 7 more days
#
begin = dom - 3
end = dom + 5
diff = csec2 - csec
if diff > 432000:
end += 7
#
#--- read radiation zone infornation
#
infile = house_keeping + '/rad_zone_info'
f = open(infile, 'r')
rdata = [line.strip() for line in f.readlines()]
f.close()
status = []
rdate = []
chk = 0
last_st= ''
cnt = 0
for line in rdata:
atemp = re.split('\s+', line)
dtime = float(atemp[1]) #--- dom of the entry or exit
if chk == 0 and atemp[0] == 'ENTRY' and dtime >= begin:
status.append(atemp[0])
rdate.append(dtime)
chk += 1
last_st = atemp[0]
cnt += 1
elif chk > 0 and dtime >= begin and dtime <= end:
status.append(atemp[0])
rdate.append(dtime)
last_st = atemp[0]
cnt += 1
elif atemp[1] > end and last_st == 'EXIT':
break
elif atemp[1] > end and last_st == 'ENTRY':
status.append(atemp[0])
rdate.append(dtime)
cnt += 1
break
f = open('./temp_zone', 'w')
#
#--- a format of the output is, e.g.: '20120313 (4614.2141112963,4614.67081268519):...'
#
line = list_date + '\t'
f.write(line)
upper = cnt -1
i = 0;
while i < cnt:
line = '(' + str(rdate[i]) + ','
f.write(line)
i += 1
if i < upper:
line = str(rdate[i]) + '):'
f.write(line)
else:
line = str(rdate[i]) + ')\n'
f.write(line)
i += 1
f.close()
#
#--- append the past rad zone list
#
oldFile = house_keeping + '/rad_zone_list~'
crtFile = house_keeping + '/rad_zone_list'
if m1 is not None:
cmd = 'cat '+ './temp_zone ' + crtFile + ' > ./temp_comb'
os.system(cmd)
else:
os.system('mv .temp_zone ./temp_comb')
os.system('rm ./temp_zone')
#
#--- save the old file and move the update file to rad_zone_list
#
if m2 is not None:
cmd = 'chmod 775 ' + crtFile + ' ' + oldFile
os.system(cmd)
if m1 is not None:
cmd = 'mv ' + crtFile + ' ' + oldFile
os.system(cmd)
cmd = 'chmod 644 ' + oldFile
os.system(cmd)
cmd = 'mv ' + './temp_comb ' + crtFile
os.system(cmd)
def sci_run_compute_gap(file = 'NA'):
'for a given file name which contains a list like: "20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto", recompute the lost science time (excluding radiation zone) '
#
#--- if file is not given (if it is NA), ask the file input
#
if file == 'NA':
file = raw_input('Please put the intrrupt timing list: ')
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30)
#--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list
#--- to each date
#
update = []
for ent in data:
if not ent: #--- if it is a blank line end the operation
break
etemp = re.split('\s+', ent)
atemp = re.split(':', etemp[1])
year = atemp[0]
month = atemp[1]
date = atemp[2]
hour = atemp[3]
mins = atemp[4]
#
#--- convert to dom/sec1998
#
ydate = tcnv.findYearDate(int(year), int(month), int(date)) #--- a function from convertTimeFormat
dom = tcnv.findDOM(int(year), int(ydate), int(hour), int(mins), 0) #--- a function from convertTimeFormat
line = year + ':' + str(ydate) + ':' + hour + ':' + mins + ':00'
csec = tcnv.axTimeMTA(line) #--- a function from convertTimeFormat
#
#--- end date
#
atemp = re.split(':', etemp[2])
eyear = atemp[0]
emonth = atemp[1]
edate = atemp[2]
ehour = atemp[3]
emins = atemp[4]
ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate))
dom2 = tcnv.findDOM(int(eyear), int(ydate), int(ehour), int(emins), 0)
line = eyear + ':' + str(ydate) + ':' + ehour + ':' + emins + ':00'
csec2 = tcnv.axTimeMTA(line)
#
#--- date stamp for the list
#
list_date = str(year) + str(month) + str(date)
#
#--- read radiation zone information from "rad_zone_list" and add up the time overlap with
#--- radiatio zones with the interruption time period
#
line = house_keeping + '/rad_zone_list'
f = open(line, 'r')
rlist = [line.strip() for line in f.readlines()]
f.close()
sum = 0
for record in rlist:
atemp = re.split('\s+', record)
if list_date == atemp[0]:
btemp = re.split(':', atemp[1])
for period in btemp:
t1 = re.split('\(', period)
t2 = re.split('\)', t1[1])
t3 = re.split('\,', t2[0])
pstart = float(t3[0])
pend = float(t3[1])
if pstart >= dom and pstart < dom2:
if pend <= dom2:
diff = pend - pstart
sum += diff
else:
diff = dom2 - pstart
sum += diff
elif pstart < dom2 and pend > dom:
if pend <= dom2:
diff = pend - dom
sum += diff
else:
diff = dom2 - dom
sum += diff
break
sum *= 86400 #--- change unit from day to sec
sciLost = (csec2 - csec - sum) / 1000 #--- total science time lost excluding radiation zone passes
line = etemp[0] + '\t' + etemp[1] + '\t' + etemp[2] + '\t' + "%.1f" % sciLost + '\t' + etemp[4]
update.append(line)
#
#--- update the file
#
oldfile = file + '~'
cmd = 'mv ' + file + ' ' + oldfile
os.system(cmd)
f = open(file, 'w')
for ent in update:
f.write(ent)
f.write('\n')
f.close()
def acis_gain_plot_trend():
"""
plotting trends of gain and offset
Input: none, but read from <data_dir>
Output: <web_dir>/Plots/gain_plot_ccd<ccd>.png
<web_dir>/Plots/offset_plot_ccd<ccd>.png
"""
for ccd in range(0, 10):
#
#--- plotting 4 nodes on one panel, but gain and offset separately
#
Xset_gain = []
Yset_gain = []
Eset_gain = []
yMin_gain = []
yMax_gain = []
Label_gain = []
Xset_offset = []
Yset_offset = []
Eset_offset = []
yMin_offset = []
yMax_offset = []
Label_offset= []
for node in range(0, 4):
#
#--- read data for given CCD and Node #
#
file = data_dir + 'ccd' + str(ccd) + '_' + str(node)
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
time = []
gain = []
gerr = [] #--- error for gain
offset = []
oerr = [] #--- error for offset
#
#--- setting lower and upper limits to remove outlyers
#
sum = 0
sum2 = 0
for ent in data:
atemp = re.split('\s+', ent)
gval = float(atemp[4])
sum += gval
sum2 += gval * gval
avg = sum / len(data)
sig = math.sqrt(sum2/len(data) - avg * avg)
blim = avg - 3.0 * sig;
tlim = avg + 3.0 * sig;
for ent in data:
atemp = re.split('\s+', ent)
#
#--- convert time into year date (e.g.2012.14)
#
gval = float(atemp[4])
if (gval <= blim) or (gval >= tlim):
continue
stime = tcnv.axTimeMTA(int(atemp[0]))
btemp = re.split(':', stime)
year = int(btemp[0])
ydate = int(btemp[1])
chk = 4.0 * int (0.25 * year)
if chk == year:
base = 366.0
else:
base = 365.0
ytime = float(year) + float(ydate) / base
time.append(ytime)
gain.append(float(atemp[4]))
gerr.append(float(atemp[5]))
offset.append(float(atemp[6]))
oerr.append(float(atemp[7]))
xmax = max(time)
Xset_gain.append(time)
Yset_gain.append(gain)
Eset_gain.append(gerr)
#
#--- set plotting range
#
avg = mean(gain)
ymin = avg - 0.002
ymin = round(ymin, 3) -0.001
ymax = avg + 0.002
ymax = round(ymax, 3) +0.001
yMin_gain.append(ymin)
yMax_gain.append(ymax)
name = 'Gain (ADU/eV) Node' + str(node)
Label_gain.append(name)
Xset_offset.append(time)
Yset_offset.append(offset)
Eset_offset.append(oerr)
avg = mean(offset)
ymin = avg - 8.0
ymin = int(ymin)
ymax = avg + 8.0
ymax = int(ymax)
yMin_offset.append(ymin)
yMax_offset.append(ymax)
name = 'Offset (ADU) Node' + str(node)
Label_offset.append(name)
xmin = int(2000)
xmax = int(xmax) + 1
xname = 'Time (year)'
#
#--- actual plotting starts here
#
yname = 'Gain'
plotPanel(xmin, xmax, yMin_gain, yMax_gain, Xset_gain, Yset_gain, Eset_gain, xname, yname, Label_gain)
outname = web_dir + "/Plots/gain_plot_ccd" + str(ccd) + '.png'
cmd = 'mv out.png ' + outname
os.system(cmd)
yname = 'Offset'
plotPanel(xmin, xmax, yMin_offset, yMax_offset, Xset_offset, Yset_offset, Eset_offset, xname, yname, Label_offset)
outname = web_dir + "/Plots/offset_plot_ccd" + str(ccd) + '.png'
cmd = 'mv out.png ' + outname
os.system(cmd)
def create_weekly_report(date, year, debug = 0):
"""
main script to set up the weekly report template for the week
input: date --- date in the format of mmdd (e.g. 0910)
year --- year in the format of yyyy (e.g. 2015)
debug --- if it is other than 0, print out some output
output: a direcotry containing templete (e.g. Sep10)
"""
#
#--- if the test is requested, create Test directory
#
if debug != 0:
os.system('mkdir /data/mta/Script/Weekly/Test/')
odir = '/data/mta/Script/Weekly/Test/'
else:
odir = '/data/mta4/www/REPORTS/'
oned = 86400
syear = str(year) #--- 4 digit year
yrd2 = year[2] + year[3] #--- 2 digit year
year = int(float(year)) #--- integer year
date = str(date)
smon = date[0] + date[1] #--- two digit month
mon = int(float(smon)) #--- integer month
lmon = tcnv.changeMonthFormat(mon) #--- month in letter (e.g.Mar)
sday = date[2] + date[3] #--- two digit mday
day = int(float(sday)) #--- integer mday
stop = tcnv.convertDateToCTime(year, mon, day, 0, 0, 0)
day_n = stop - 7 * oned
# tout = tcnv.axTimeMTA(day_n)
# ttemp = re.split(':', tout)
# iru_start = str(ttemp[0]) + '_' + str(ttemp[1])
#day01 = stop - 6 * oned
#day0 = stop - 7 * oned
day01 = stop - 5 * oned
day0 = stop - 6 * oned
lday0 = stime_to_ddate(day0)
sday0 = sdate_to_ldate(lday0)
start = day0
lday1 = stime_to_ddate(day01)
tout = tcnv.axTimeMTA(day0)
ttemp = re.split(':', tout)
iru_start = str(ttemp[0]) + '_' + str(ttemp[1])
#
#--- year of the beginning of the period; could be different from that of the end
#
byear = ttemp[0]
lday0 = stime_to_ddate(day0)
day1 = stop - 5 * oned
lday1 = stime_to_ddate(day1)
day2 = stop - 4 * oned
lday2 = stime_to_ddate(day2)
sday2 = sdate_to_ldate(lday2)
day3 = stop - 3 * oned
lday3 = stime_to_ddate(day3)
day4 = stop - 2 * oned
lday4 = stime_to_ddate(day4)
sday4 = sdate_to_ldate(lday4)
day5 = stop - 1 * oned
lday5 = stime_to_ddate(day5)
day6 = stop
lday6 = stime_to_ddate(day6)
sday6 = sdate_to_ldate(lday6)
#tout = tcnv.axTimeMTA(day5)
tout = tcnv.axTimeMTA(day6)
ttemp = re.split(':', tout)
iru_stop = '_' + str(ttemp[1])
day7 = stop + 1 * oned
lday7 = stime_to_ddate(day7)
#
#---- setting file name
#
atemp = re.split('\/', lday6)
file_date = atemp[0] + atemp[1]
file_date2 = atemp[0] + '/' + atemp[1]
file_name = file_date + '.html'
#
#--- title
#
titledate = lday0 + ' - ' + lday6
ldate = sdate_to_ldate(lday6)
ldate_sp = sdate_to_ldate_with_space(lday6)
#
#--- focal temp file name
#
fptemp = file_date + '_fptemp.gif'
fpext_range = str(start)+' '+ str(stop)
fpstart = str(start)
fplsub = '"'+ sday0 + '", "' + sday2 + '", "' + sday4 + '", "' + sday6 + '"'
fpdsub = str(day0) + ', ' + str(day2) + ', ' + str(day4) + ', ' + str(day6)
#
#--- IRU span
#
irudate = iru_start + iru_stop
#
#--- telemetry idl command
#
telmstart = stime_to_ddate2(start)
telmstop = stime_to_ddate2(stop)
telem_command = 'weekly_telem,' + telmstart + ',' + telmstop
#
#--- telemetry header line
#
daylist = '|' + lday0 +'|' + lday1 + '|' + lday2 +'|' + lday3 +'|' + lday4 +'|' + lday5 +'|' + lday6
#
#--- find trending dates/title
#
tfile = tdir + 'trending_order'
f = open(tfile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
data.reverse()
chk = 0
for ent in data:
if ent == '\s+' or ent == '':
continue
atemp = re.split(' : ', ent)
adate = atemp[0].strip()
aname = atemp[1].strip()
if chk == 0:
if adate == file_date2:
title = aname
chk = 1
else:
if aname == title:
last_trend_date = adate
break
#
#--- index.html input
#
s1 = sday0[0:3] + ' ' + sday0[3:5]
s2 = sday6[0:3] + ' ' + sday6[3:5]
index = '<td> <a href="./' + str(year) + '/' + file_date + '.html">' + s1 + ' - ' + s2 + '</a>'
#
#--- debugging output
#
if debug != 0:
print "file_name; " + file_name
print "title date: " + titledate
print "ldate: " + ldate
print "fptemp: " + fptemp
print "fpext_range: " + fpext_range
print "fpstart: " + fpstart
print " fplsub: " + fplsub
print " fpdsub: " + fpdsub
print "irudate: " + irudate
print "telmstart: " + telmstart
print "telmstop: " + telmstop
print "telmcommand: " + telem_command
print "daylist: " + daylist
print "title: " + title
print "last_trend_date: " + last_trend_date
print "index: " + index
#
#--- create a work directory
#
cmd = 'mkdir ' + wdir + ldate
os.system(cmd)
outdir = wdir + ldate + '/'
cmd = 'cp ' + tdir + 'get_ftemp_data.perl' + ' ' + outdir
os.system(cmd)
cmd = 'cp ' + tdir + 'subst3.perl' + ' ' + outdir
os.system(cmd)
cmd = 'cp ' + tdir + 'test' + ' ' + outdir
os.system(cmd)
cmd = 'mkdir ' + outdir + '/param'
os.system(cmd)
#
#--- create instruction page
#
tfile = tdir + 'instruction'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#LDATE_S#', ldate_sp)
input = input.replace('#LDATE#', ldate)
input = input.replace('#DDATE#', file_date)
input = input.replace('#YEAR#', syear)
input = input.replace('#DAYLIST#', daylist)
input = input.replace('#TELM_CMD#',telem_command)
input = input.replace('#INDEX#', index)
ofile = outdir + ldate.lower()
fo = open(ofile, 'w')
fo.write(input)
fo.close()
#
#--- focal temp related files
#
tfile = tdir + 'get_ftemp_wrap_script'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#MANDD#', ldate)
ofile = outdir + 'get_ftemp_wrap_script'
fo = open(ofile, 'w')
fo.write(input)
fo.close()
cmd = 'chmod 755 ' + outdir + 'get_ftemp_wrap_script'
os.system(cmd)
tfile = tdir + 'get_ftemp_main_script'
f = open(tfile, 'r')
input = f.read()
f.close()
#input = input.replace('#START#', str(start))
#input = input.replace('#STOP#', str(stop))
input = input.replace('#START#', str(start + 86400))
input = input.replace('#STOP#', str(stop + 86400))
ofile = outdir + 'get_ftemp_main_script'
fo = open(ofile, 'w')
fo.write(input)
fo.close()
cmd = 'chmod 755 ' + outdir + 'get_ftemp_main_script'
os.system(cmd)
tfile = tdir + 'plot_erad_time.pro'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#START#', str(start))
input = input.replace('#SDATELIST#', fpdsub)
input = input.replace('#LDATELIST#', fplsub)
ofile = outdir + 'plot_erad_time.pro'
fo = open(ofile, 'w')
fo.write(input)
fo.close()
cmd = 'cp -f ' + outdir + 'plot_erad_time.pro ./Focal/'
os.system(cmd)
#
#--- this_week file
#
tfile = tdir + 'this_week'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#DDATE#', file_date)
input = input.replace('#IRUSPAN1#', irudate)
input = input.replace('#IRUSPAN2#', irudate)
input = input.replace('#TITLE#', title)
input = input.replace('#TITLEDATE#',titledate)
atemp = last_trend_date
atemp = atemp.replace('/', '')
input = input.replace('#PREVREPORT#', atemp)
atemp = re.split('/', last_trend_date)
pmon = int(float(atemp[0]))
lmon = tcnv.changeMonthFormat(pmon)
line = lmon + ' ' + atemp[1]
#
#--- the previous report could be from the last year
#
ryear = syear
if mon < pmon:
ryear = year -1
ryear = str(ryear)
input = input.replace('#RYEAR#', ryear)
input = input.replace('#PREVDATE#', line)
atitle = str(title)
atitle = atitle.replace(' ', '_')
#file = tdir + 'Headers/' + atitle
#fs = open(file, 'r')
#trend = fs.read()
#fs.close()
#input = input.replace("#TREND#", trend)
[temp1, temp2, temp3, temp4] = read_cti_values()
input = input.replace('#ATEMP#', temp1)
input = input.replace('#ATEMP2#', temp2)
input = input.replace('#DTEMP#', temp3)
input = input.replace('#DTEMP2#', temp4)
[val, step] = read_sim()
input = input.replace('#WMOVE#', val)
input = input.replace('#WSTEP#', step)
#
#--- make photon and bad pixel output
#
run_bad_pix_and_photon(outdir)
#
#--- run to get focal temp fits files
#
tstop = stop + 86400
[fcnt, fdata] = run_focal_temp_data(outdir, start, stop, fptemp)
[fcnt, fdata] = run_focal_temp_data_new()
input = input.replace('#TEMPPEAK#', str(fcnt))
input = input.replace('#TEMPLIST#', fdata)
#
#--- bad pixel
#
file = outdir + 'bad_pix_list'
fs = open(file, 'r')
bdata = fs.read()
fs.close()
input = input.replace('BAD_PIXEL_TABLE', bdata)
#
#--- photon
#
file = outdir + 'photons'
fs = open(file, 'r')
pdata = fs.read()
fs.close()
input = input.replace('PHOTON_TABLE', pdata)
#
#--- telem data
#
update_weekly_telem(year, byear, mon)
tdata = run_telem_data(telem_command, daylist, outdir)
input = input.replace('TELEM_TABLE', tdata)
#
#--- trend data
#
trend = set_trend_data_input(str(date))
input = input.replace('#TREND#', trend)
#
#--- write out the weekly report
#
ofile = outdir + file_name
fo = open(ofile, 'w')
fo.write(input)
fo.close()
#
#--- clean up
#
cmd = 'rm ./out ./out2'
os.system(cmd)
#
#--- move files
#
move_files(date, year, outdir, file_name, fptemp, odir)
#
#--- send out email to admin; notify the job complete
#
send_email_to_admin(date, year)
def extract_sim_data():
"""
extract sim data from PRIMARYCCDM_*.*.tl
input: none but read from <dumpdir>/PRIMARYCCDM_*.*.tl
output: <outdir>sim_data.out
"""
#
#--- find the time of the last entry from the sim_data.out
#
sfile = outdir + 'sim_data.out'
f = open(sfile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- cleaning up the data; drop the data which the date starts from ":" e.g. :2014
#
pdata = []
for ent in data:
if re.search('^:', ent):
continue
else:
pdata.append(ent)
#
#--- the last entiry values
#
if len(pdata) > 0:
atemp = re.split('\s+', pdata[len(pdata) - 1])
ltime = tcnv.axTimeMTA(
atemp[0]) #--- converting time to sec from 1998.1.1
time_2 = atemp[0]
col1_2 = atemp[1]
col2_2 = atemp[2]
col3_2 = atemp[3]
else:
ltime = 0
time_2 = 0
col1_2 = ''
col2_2 = ''
col3_2 = ''
#
#--- check whether input files exists
#
cmd = 'ls -rt ' + dumpdir + 'PRIMARYCCDM_*.*.tl >' + zspace
os.system(cmd)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + zspace
os.system(cmd)
dlen = len(data)
if dlen < 1:
exit(1)
#
#--- files exist. read the data from the last 10 files
#
tlist = data[dlen - 40:]
for ent in tlist:
cmd = 'cat ' + ent + ' >> ' + zspace
os.system(cmd)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + zspace
os.system(cmd)
prev = ''
fo = open('./temp_save', 'w')
#
#--- go though each data line
#
for ent in data:
try:
#
#--- expect the first letter of the data line is numeric (e.g. 2014).
#
val = float(ent[0])
except:
continue
#
#--- only data with "FMT" format will be used
#
mc = re.search('FMT', ent)
if mc is None:
continue
atemp = re.split('\t+', ent)
#
#--- if there are less than 20 entries, something wrong; skip it
#
if len(atemp) < 20:
continue
#
#--- convert time format
#
time = atemp[0]
time = time.strip()
time = time.replace(' ', ':')
time = time.replace(':::', ':00')
time = time.replace('::', ':0')
#
#--- if the time is exactly same as one before, skip it
#
if time == time_2:
continue
#
#--- if the time is already in the database, keip it
#
stime = tcnv.axTimeMTA(time)
if stime <= ltime:
continue
#
#--- use only data which tscpos and fapos have numeric values
#
tscpos = atemp[4].strip()
fapos = atemp[5].strip()
if tscpos == "" or fapos == "":
continue
else:
tscpos = int(float(tscpos))
fapos = int(float(fapos))
# aopc = atemp[11].strip()
# if aopc == '':
# aopc = '0'
mpwm = atemp[12].strip()
if mcf.chkNumeric(mpwm):
mpwm = int(float(mpwm))
mpwm = str(mpwm)
else:
mpwm = '0'
#
#--- we want to print only beginning and ending of the same data entries.
#--- skip the line if all three entiries are same as one before, except the last one
#
if col1_2 == tscpos and col2_2 == fapos and col3_2 == mpwm:
time_2 = time
continue
line = time + '\t' + str(tscpos) + '\t' + str(
fapos) + '\t' + mpwm + '\n'
if line == prev:
continue
else:
pline = time_2 + '\t' + str(col1_2) + '\t' + str(
col2_2) + '\t' + str(col3_2) + '\n'
fo.write(pline)
fo.write(line)
prev = line
time_2 = time
col1_2 = tscpos
col2_2 = fapos
col3_2 = mpwm
fo.close()
sfile2 = sfile + '~'
cmd = 'cp ' + sfile + ' ' + sfile2
os.system(cmd)
cmd = 'cat ./temp_save >> ' + sfile
os.system(cmd)
def sci_run_add_to_rad_zone_list(file='NA'):
'adding radiation zone list to rad_zone_list. input: file name containing: e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto'
#
#--- if file is not given (if it is NA), ask the file input
#
if file == 'NA':
file = raw_input('Please put the intrrupt timing list: ')
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
data.reverse()
f.close()
for ent in data:
#
#--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30)
#--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list
#--- to each date
#
etemp = re.split('\s+', ent)
atemp = re.split(':', etemp[1])
year = atemp[0]
month = atemp[1]
date = atemp[2]
hour = atemp[3]
mins = atemp[4]
#
#--- convert to dom/sec1998
#
ydate = tcnv.findYearDate(int(year), int(month), int(date))
dom = tcnv.findDOM(int(year), int(ydate))
line = year + ':' + str(int(ydate)) + ':' + hour + ':' + mins + ':00'
csec = tcnv.axTimeMTA(line)
#
#--- end date
#
atemp = re.split(':', etemp[2])
eyear = atemp[0]
emonth = atemp[1]
edate = atemp[2]
ehour = atemp[3]
emins = atemp[4]
ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate))
dom2 = tcnv.findDOM(int(eyear), int(ydate))
line = eyear + ':' + str(int(ydate)) + ':' + ehour + ':' + emins + ':00'
csec2 = tcnv.axTimeMTA(line)
#
#--- date stamp for the list
#
list_date = str(year) + str(month) + str(date)
#
#--- check radiation zones for 3 days before to 5 days after from the interruptiondate
#
begin = dom - 3
end = dom2 + 5
#
#--- read radiation zone infornation
#
infile = house_keeping + '/rad_zone_info'
f = open(infile, 'r')
rdata = [line.strip() for line in f.readlines()]
f.close()
status = []
rdate = []
chk = 0
last_st= ''
cnt = 0
for line in rdata:
atemp = re.split('\s+', line)
dtime = float(atemp[1]) #--- dom of the entry or exit
if chk == 0 and atemp[0] == 'ENTRY' and dtime >= begin:
status.append(atemp[0])
rdate.append(dtime)
chk += 1
last_st = atemp[0]
cnt += 1
elif chk > 0 and dtime >= begin and dtime <= end:
status.append(atemp[0])
rdate.append(dtime)
last_st = atemp[0]
cnt += 1
elif atemp[1] > end and last_st == 'EXIT':
break
elif atemp[1] > end and last_st == 'ENTRY':
status.append(atemp[0])
rdate.append(dtime)
cnt += 1
break
f = open('./temp_zone', 'w')
#
#--- a format of the output is, e.g.: '20120313 (4614.2141112963,4614.67081268519):...'
#
line = list_date + '\t'
f.write(line)
i = 0
while i < cnt:
line = '(' + str(rdate[i]) + ','
f.write(line)
i += 1
if i < cnt-1:
line = str(rdate[i]) + '):'
f.write(line)
else:
line = str(rdate[i]) + ')\n'
f.write(line)
i += 1
f.close()
#
#--- append the past rad zone list
#
oldFile = house_keeping + '/rad_zone_list~'
crtFile = house_keeping + '/rad_zone_list'
#> cmd = 'cat ' './temp_zone ' + crtFile + ' > /tmp/mta/temp_comb'
cmd = 'cat '+ './temp_zone ' + crtFile + ' > ./temp_comb'
os.system(cmd)
#> os.systm('rm /tmp/mta/temp_zone')
os.system('rm ./temp_zone')
#
#--- save the old file and move the update file to rad_zone_list
#
cmd = 'chmod 775 ' + crtFile + ' ' + oldFile
os.system(cmd)
cmd = 'mv ' + crtFile + ' ' + oldFile
os.system(cmd)
cmd = 'chmod 644 ' + oldFile
os.system(cmd)
#> cmd = 'mv ' + '/tmp/mta/temp_comb ' + crtFile
cmd = 'mv ' + './temp_comb ' + crtFile
os.system(cmd)
def report_error():
"""
read errors from <cup_usr_list>_error_list, sort it out, clean, and send out email
Input: none but read from <cup_usr_list>_error_list
Output: email sent out
"""
#
#--- find the current time
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime("Local")
#
#--- create surfix for files which will be saved in Past_errors directory
#
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(day)
if day < 10:
sday = '0' + sday
tail = str(year) + smon + sday
for tag in cpu_usr_list:
efile = house_keeping + 'Records/' + tag + '_error_list'
pfile = house_keeping + 'Records/Past_errors/' + tag + '_error_list_' + tail
prev_line = ''
chk = mcf.chkFile(efile)
if chk > 0:
#
#--- read error messages from the file
#
f = open(efile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- sort the data so that we can correct messages to each cron job together
#
data.sort()
task_list = []
time_list = []
mssg_list = []
for ent in data:
atemp = re.split(' : ' , ent)
task_list.append(atemp[0])
stime = int(atemp[1])
dtime = tcnv.axTimeMTA(stime)
time_list.append(dtime)
mssg_list.append(atemp[2])
#
#--- write out cron job name
#
fo = open(zspace, 'w')
cname = task_list[0]
line = '\n\n' + cname + '\n____________________\n\n'
fo.write(line)
for i in range(0, len(mssg_list)):
if task_list[i] != cname:
cname = task_list[i]
line = '\n\n' + cname + '\n____________________\n\n'
fo.write(line)
#
#--- create each line. if it is exactly same as one line before, skip it
#
line = time_list[i] + ' : ' + mssg_list[i] + '\n'
if line != prev_line:
fo.write(line)
prev_line = line
fo.close()
#
#--- send email out
#
# cmd = 'cp ' + zspace + ' ' + '/data/mta/Script/Cron_check/Scripts/' + tag
# os.system(cmd)
send_mail(tag, email_list)
#
#--- move the error list to Past_errors directory
#
cmd = 'mv ' + efile + ' ' + pfile
os.system(cmd)
cmd = 'chmod 755 ' + pfile
os.system(cmd)
def sci_run_compute_gap(file = 'NA'):
'for a given file name which contains a list like: "20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto", recompute the lost science time (excluding radiation zone) '
#
#--- if file is not given (if it is NA), ask the file input
#
if file == 'NA':
file = raw_input('Please put the intrrupt timing list: ')
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30)
#--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list
#--- to each date
#
update = []
for ent in data:
if not ent: #--- if it is a blank line end the operation
break
etemp = re.split('\s+', ent)
atemp = re.split(':', etemp[1])
year = atemp[0]
month = atemp[1]
date = atemp[2]
hour = atemp[3]
mins = atemp[4]
#
#--- convert to dom/sec1998
#
ydate = tcnv.findYearDate(int(year), int(month), int(date)) #--- a function from convertTimeFormat
dom = tcnv.findDOM(int(year), int(ydate), int(hour), int(mins), 0) #--- a function from convertTimeFormat
line = year + ':' + str(ydate) + ':' + hour + ':' + mins + ':00'
csec = tcnv.axTimeMTA(line) #--- a function from convertTimeFormat
#
#--- end date
#
atemp = re.split(':', etemp[2])
eyear = atemp[0]
emonth = atemp[1]
edate = atemp[2]
ehour = atemp[3]
emins = atemp[4]
ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate))
dom2 = tcnv.findDOM(int(eyear), int(ydate), int(ehour), int(emins), 0)
line = eyear + ':' + str(ydate) + ':' + ehour + ':' + emins + ':00'
csec2 = tcnv.axTimeMTA(line)
#
#--- date stamp for the list
#
list_date = str(year) + str(month) + str(date)
#
#--- read radiation zone information from "rad_zone_list" and add up the time overlap with
#--- radiatio zones with the interruption time period
#
line = house_keeping + '/rad_zone_list'
f = open(line, 'r')
rlist = [line.strip() for line in f.readlines()]
f.close()
sum = 0
for record in rlist:
atemp = re.split('\s+', record)
if list_date == atemp[0]:
btemp = re.split(':', atemp[1])
for period in btemp:
t1 = re.split('\(', period)
t2 = re.split('\)', t1[1])
t3 = re.split('\,', t2[0])
pstart = float(t3[0])
pend = float(t3[1])
if pstart >= dom and pstart < dom2:
if pend <= dom2:
diff = pend - pstart
sum += diff
else:
diff = dom2 - pstart
sum += diff
elif pstart < dom2 and pend > dom:
if pend <= dom2:
diff = pend - dom
sum += diff
else:
diff = dom2 - dom
sum += diff
break
sum *= 86400 #--- change unit from day to sec
sciLost = (csec2 - csec - sum) / 1000 #--- total science time lost excluding radiation zone passes
line = etemp[0] + '\t' + etemp[1] + '\t' + etemp[2] + '\t' + "%.1f" % sciLost + '\t' + etemp[4]
update.append(line)
return update
def extract_new_data_sub(dir, col_list, start, stop, year, month):
#
#--- extract data
#
line = 'operation=retrieve\n'
line = line + 'dataset = mta\n'
line = line + 'detector = grad\n'
line = line + 'level = 0.5\n'
line = line + 'filetype =' + dir + '\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
fo = open(zspace, 'w')
fo.write(line)
fo.close()
cmd1 = "/usr/bin/env PERL5LIB="
#cmd2 = ' echo ' + hakama + ' |arc4gl -U' + dare + ' -Sarcocc -i' + zspace
cmd2 = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
mcf.rm_file(zspace)
cmd = 'mv *gz ./Working_dir/.'
os.system(cmd)
cmd = 'gzip -d ./Working_dir/*.gz'
os.system(cmd)
cmd = 'ls ./Working_dir/*.fits > ' + zspace
os.system(cmd)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
mcf.rm_file(zspace)
if len(data) == 0:
return 'na'
#
#--- extract each column data and stored in arrays
#
elen = len(col_list)
dat_save = [[] for x in range(0, elen)]
time_dat = []
for file in data:
dout = fits.getdata(file, 1)
time = dout.field('time')
for ent in time:
time_dat.append(ent)
for k in range(0, elen):
col = col_list[k]
sta = 'ST_' + col
val = dout.field(col)
chk = dout.field(sta)
for j in range(0, len(chk)):
if chk[j] >= 0:
dat_save[k].append(val[j])
#
#--- set the beginning of the and the end of the first day of the data
#
ydate = tcnv.findYearDate(year, month, 1)
tent = str(year) + ':' + str(ydate) + ':00:00:00'
begin = tcnv.axTimeMTA(tent)
smonth = month + 1
syear = year
if smonth > 12:
smonth = 1
syear += 1
ydate = tcnv.findYearDate(syear, smonth, 1)
tent = str(syear) + ':' + str(ydate) + ':00:00:00'
stop = tcnv.axTimeMTA(tent)
end = begin + 86400
if end > stop:
end = stop
dom = int(tcnv.stimeToDom(begin))
sline = str(dom)
dsumming = [[] for x in range(0, elen)]
#
#--- find mean and std of each columns
#
for k in range(0, len(time_dat)):
if time_dat[k] >= begin and time_dat[k] < end:
for i in range(0, elen):
dsumming[i].append(dat_save[i][k])
elif time_dat[k] < begin:
continue
elif time_dat[k] >= end:
for i in range(0, elen):
narray = numpy.array(dsumming[i])
avg = numpy.mean(narray)
avg = '%.4f' % round(avg, 4)
err = numpy.std(narray)
err = '%.4f' % round(err, 4)
sline = sline + '\t' + str(avg) + '\t' + str(err)
sline = sline + '\n'
#
#--- reintialize for the next day
#
begin += 86400
if begin >= stop:
break
end = begin + 86400
if end > stop:
end = stop
dom = int(tcnv.stimeToDom(begin))
sline = sline + str(dom)
dsumming = [[] for x in range(0, elen)]
cmd = 'rm ./Working_dir/*.fits*'
os.system(cmd)
return sline
def get_new_value(year, month):
"""
extract aorwspd values from dataseeker
input: year/month
output: [av1, av2, av3, av4, av5, av6] --- six values of aorwspd(1-6)
"""
#
#--- set month long time interval in sec from 1998.1.1
#
year2 = year
month2 = month -1
if month2 < 1:
mont2 = 12
year2 -= 1
ydate = tcnv.findYearDate(year, month, 15)
t_in = str(year) + ':' + str(ydate) + ':00:00:00'
time1 = tcnv.axTimeMTA(t_in)
ydate = tcnv.findYearDate(year2, month2, 15)
t_in = str(year2) + ':' + str(ydate) + ':00:00:00'
time2 = tcnv.axTimeMTA(t_in)
#
#--- set command to call dataseeker
#
f = open('./test', 'w') #-- we need an empty "test" file to run dataseeker
f.close()
mcf.mk_empty_dir('param') #-- make empty param directory
line = 'columns=_aorwspd1_avg,'
line = line + '_aorwspd2_avg,'
line = line + '_aorwspd3_avg,'
line = line + '_aorwspd4_avg,'
line = line + '_aorwspd5_avg,'
line = line + '_aorwspd6_avg'
line = line + ' timestart=' + str(time2)
line = line + ' timestop=' + str(time1)
cmd = 'punlearn dataseeker; dataseeker.pl infile=test outfile=ztemp.fits search_crit=\''
cmd = 'dataseeker.pl infile=test outfile=ztemp.fits search_crit="'
cmd = cmd + line + '" loginFile="' + loginfile + '"'
#
#--- run dataseeker
#
bash("/usr/bin/env PERL5LIB='' " + cmd, env=ascdsenv)
mcf.rm_file(zspace)
mcf.rm_file('./test')
os.system('rm -rf ./param')
#
#--- read fits file
#
try:
dout = pyfits.getdata('./ztemp.fits')
aw1 = dout.field('AORWSPD1_AVG')
aw2 = dout.field('AORWSPD2_AVG')
aw3 = dout.field('AORWSPD3_AVG')
aw4 = dout.field('AORWSPD4_AVG')
aw5 = dout.field('AORWSPD5_AVG')
aw6 = dout.field('AORWSPD6_AVG')
except:
aw1 = aw2 = aw3 = aw4 = aw5 = aw6 = []
mcf.rm_file("./ztemp.fits")
#
#--- create monthly "sum" of the reaction wheel rotations
#--- dataseeker gives 5 min avg of the value; one day is 24 hr x 60 min / 5min = 288.
#
sum1 = sum2 = sum3 = sum4 = sum5 = sum6 = 0.0
for i in range(0, len(aw1)):
sum1 += abs(aw1[i])
sum2 += abs(aw2[i])
sum3 += abs(aw3[i])
sum4 += abs(aw4[i])
sum5 += abs(aw5[i])
sum6 += abs(aw6[i])
av1 = sum1 / 288
av2 = sum2 / 288
av3 = sum3 / 288
av4 = sum4 / 288
av5 = sum5 / 288
av6 = sum6 / 288
return [av1, av2, av3, av4, av5, av6]
def find_sim_position(start, stop):
"""
extract sim position realated data for a given time span
input start --- starting time
stop --- stopping time
data are in: /data/mta_www/mta_temp/mta_states/MJ/<yyyy>//comprehensive_data_summary<yyyy>
output: acis_i_start/ acis_i_stop --- acis i starting and stopping time
acis_s_start/ acis_s_stop --- acis s starting and stopping time
hrc_i_start/ hrc_i_stop --- hrc i starting and stopping time
hrc_s_start/ hrc_s_stop --- hrc s starting and stopping time
hetg_start/ hetg_stop --- hetg starting and stopping time
letg_start/ letg_stop --- letg starting and stopping time
radmon_start/ radmon_stop --- radmon starting and stopping time
fmt --- fmt format list
time --- time list
"""
#
#--- find year of starting and stopping time. this will be used to fine which data set we need to use
#
ntime = tcnv.axTimeMTA(start)
atemp = re.split(':', ntime)
syear = int(atemp[0])
ntime = tcnv.axTimeMTA(stop)
atemp = re.split(':', ntime)
lyear = int(atemp[0])
#
#-- read all data sets contain the appropriate data
#
data = []
for year in range(syear, lyear+1):
file = '/data/mta_www/mta_temp/mta_states/MJ/' + str(year) + '/comprehensive_data_summary' + str(year)
f = open(file, 'r')
dtmp = [line.strip() for line in f.readlines()]
f.close()
data = data + dtmp
acis_i_start = []
acis_i_stop = []
acis_s_start = []
acis_s_stop = []
hrc_i_start = []
hrc_i_stop = []
hrc_s_start = []
hrc_s_stop = []
hetg_start = []
hetg_stop = []
letg_start = []
letg_stop = []
radmon_start = []
radmon_stop = []
fmt = []
time = []
acis_i_in = 0
acis_s_in = 0
hrc_i_in = 0
hrc_s_in = 0
hetg_in = 0
letg_in = 0
radmon_on = 0
for ent in data:
#
#--- use only data line starting with year (which should be a digits)
#
try:
val = float(ent[0])
except:
continue
atemp = re.split('\s+', ent)
stime = tcnv.axTimeMTA(atemp[0]) #--- converting time to sec from 1998.1.1
if stime < start:
continue
if stime > stop:
break
scpos = float(atemp[1]) #--- 3TSCPOS
hpos = float(atemp[9]) #--- 4HPOSARO
lpos = float(atemp[11]) #--- 4LPOSARO
time.append(stime)
#
#--- ACIS I
#
if scpos > 89000 and acis_i_in == 0:
acis_i_start.append(stime)
acis_i_in = 1
elif scpos < 89000 and acis_i_in == 1:
acis_i_stop.append(stime)
acis_i_in = 0
#
#--- ACIS S
#
elif scpos < 80000 and scpos > 71000 and acis_s_in == 0:
acis_s_start.append(stime)
acis_s_in = 1
elif (scpos > 80000 or scpos < 71000) and acis_s_in == 1:
acis_s_stop.append(stime)
acis_s_in = 0
#
#--- HRC I
#
elif scpos < -45000 and scpos > -55000 and hrc_i_in == 0:
hrc_i_start.append(stime)
hrc_i_in = 1
elif (scpos > -45000 or scpos < -55000) and hrc_i_in == 1:
hrc_i_stop.append(stime)
hrc_i_in = 0
#
#--- HRC S
#
elif scpos < -90000 and hrc_s_in == 0:
hrc_s_start.append(stime)
hrc_s_in = 1
elif scpos > -90000 and hrc_s_in == 1:
hrc_s_stop.append(stime)
hrc_s_in = 0
#
#--- HETIG
#
if hpos < 20 and hetg_in == 0:
hetg_start.append(stime)
hetg_in = 1
elif hpos > 60 and hetg_in == 1:
hetg_stop.append(stime)
hetg_in = 0
#
#--- LETG
#
if lpos < 20 and letg_in == 0:
letg_start.append(stime)
letg_in = 1
elif lpos > 60 and letg_in == 1:
letg_stop.append(stime)
letg_in = 0
#
#--- RADMON
#
if atemp[5] == 'DISA' and radmon_on == 0:
radmon_start.append(stime)
radmon_on = 1
elif atemp[5] == 'ENAB' and radmon_on == 1:
radmon_stop.append(stime)
radmon_on = 0
#
#--- FMT Format
#
fmt.append(atemp[7])
#
#--- for the case the period is not closed during the time interval given
#--- add "stop" time to close the period
#
if len(acis_i_stop) < len(acis_i_start):
acis_i_stop.append(stop)
if len(acis_s_stop) < len(acis_s_start):
acis_s_stop.append(stop)
if len(hrc_i_stop) < len(hrc_i_start):
hrc_i_stop.append(stop)
if len(hrc_s_stop) < len(hrc_s_start):
hrc_s_stop.append(stop)
if len(hetg_stop) < len(hetg_start):
hetg_stop.append(stop)
if len(letg_stop) < len(letg_start):
letg_stop.append(stop)
if radmon_on == 1:
radmon_stop.append(stop)
return[acis_i_start, acis_i_stop, acis_s_start, acis_s_stop, hrc_i_start, hrc_i_stop, \
hrc_s_start, hrc_s_stop, hetg_start, hetg_stop, letg_start, letg_stop, \
radmon_start, radmon_stop, fmt, time]
def create_weekly_report(date, year, debug=0):
"""
main script to set up the weekly report template for the week
input: date --- date in the format of mmdd (e.g. 0910)
year --- year in the format of yyyy (e.g. 2015)
debug --- if it is other than 0, print out some output
output: a direcotry containing templete (e.g. Sep10)
"""
#
#--- if the test is requested, create Test directory
#
if debug != 0:
os.system('mkdir /data/mta/Script/Weekly/Test/')
odir = '/data/mta/Script/Weekly/Test/'
else:
odir = '/data/mta4/www/REPORTS/'
oned = 86400
syear = str(year) #--- 4 digit year
yrd2 = year[2] + year[3] #--- 2 digit year
year = int(float(year)) #--- integer year
date = str(date)
smon = date[0] + date[1] #--- two digit month
mon = int(float(smon)) #--- integer month
lmon = tcnv.changeMonthFormat(mon) #--- month in letter (e.g.Mar)
sday = date[2] + date[3] #--- two digit mday
day = int(float(sday)) #--- integer mday
stop = tcnv.convertDateToCTime(year, mon, day, 0, 0, 0)
day_n = stop - 7 * oned
# tout = tcnv.axTimeMTA(day_n)
# ttemp = re.split(':', tout)
# iru_start = str(ttemp[0]) + '_' + str(ttemp[1])
#day01 = stop - 6 * oned
#day0 = stop - 7 * oned
day01 = stop - 5 * oned
day0 = stop - 6 * oned
lday0 = stime_to_ddate(day0)
sday0 = sdate_to_ldate(lday0)
start = day0
lday1 = stime_to_ddate(day01)
tout = tcnv.axTimeMTA(day0)
ttemp = re.split(':', tout)
iru_start = str(ttemp[0]) + '_' + str(ttemp[1])
#
#--- year of the beginning of the period; could be different from that of the end
#
byear = ttemp[0]
lday0 = stime_to_ddate(day0)
day1 = stop - 5 * oned
lday1 = stime_to_ddate(day1)
day2 = stop - 4 * oned
lday2 = stime_to_ddate(day2)
sday2 = sdate_to_ldate(lday2)
day3 = stop - 3 * oned
lday3 = stime_to_ddate(day3)
day4 = stop - 2 * oned
lday4 = stime_to_ddate(day4)
sday4 = sdate_to_ldate(lday4)
day5 = stop - 1 * oned
lday5 = stime_to_ddate(day5)
day6 = stop
lday6 = stime_to_ddate(day6)
sday6 = sdate_to_ldate(lday6)
#tout = tcnv.axTimeMTA(day5)
tout = tcnv.axTimeMTA(day6)
ttemp = re.split(':', tout)
iru_stop = '_' + str(ttemp[1])
day7 = stop + 1 * oned
lday7 = stime_to_ddate(day7)
#
#---- setting file name
#
atemp = re.split('\/', lday6)
file_date = atemp[0] + atemp[1]
file_date2 = atemp[0] + '/' + atemp[1]
file_name = file_date + '.html'
#
#--- title
#
titledate = lday0 + ' - ' + lday6
ldate = sdate_to_ldate(lday6)
ldate_sp = sdate_to_ldate_with_space(lday6)
#
#--- focal temp file name
#
fptemp = file_date + '_fptemp.gif'
fpext_range = str(start) + ' ' + str(stop)
fpstart = str(start)
fplsub = '"' + sday0 + '", "' + sday2 + '", "' + sday4 + '", "' + sday6 + '"'
fpdsub = str(day0) + ', ' + str(day2) + ', ' + str(day4) + ', ' + str(day6)
#
#--- IRU span
#
irudate = iru_start + iru_stop
#
#--- telemetry idl command
#
telmstart = stime_to_ddate2(start)
telmstop = stime_to_ddate2(stop)
telem_command = 'weekly_telem,' + telmstart + ',' + telmstop
#
#--- telemetry header line
#
daylist = '|' + lday0 + '|' + lday1 + '|' + lday2 + '|' + lday3 + '|' + lday4 + '|' + lday5 + '|' + lday6
#
#--- find trending dates/title
#
tfile = tdir + 'trending_order'
f = open(tfile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
data.reverse()
chk = 0
for ent in data:
if ent == '\s+' or ent == '':
continue
atemp = re.split(' : ', ent)
adate = atemp[0].strip()
aname = atemp[1].strip()
file_date2.strip()
if chk == 0:
if str(adate) == str(file_date2):
title = aname
chk = 1
else:
if aname == title:
last_trend_date = adate
break
#
#--- index.html input
#
s1 = sday0[0:3] + ' ' + sday0[3:5]
s2 = sday6[0:3] + ' ' + sday6[3:5]
index = '<td> <a href="./' + str(
year) + '/' + file_date + '.html">' + s1 + ' - ' + s2 + '</a>'
#
#--- debugging output
#
if debug != 0:
print "file_name; " + file_name
print "title date: " + titledate
print "ldate: " + ldate
print "fptemp: " + fptemp
print "fpext_range: " + fpext_range
print "fpstart: " + fpstart
print " fplsub: " + fplsub
print " fpdsub: " + fpdsub
print "irudate: " + irudate
print "telmstart: " + telmstart
print "telmstop: " + telmstop
print "telmcommand: " + telem_command
print "daylist: " + daylist
print "title: " + title
print "last_trend_date: " + last_trend_date
print "index: " + index
#
#--- create a work directory
#
cmd = 'mkdir ' + wdir + ldate
os.system(cmd)
outdir = wdir + ldate + '/'
cmd = 'cp ' + tdir + 'get_ftemp_data.perl' + ' ' + outdir
os.system(cmd)
cmd = 'cp ' + tdir + 'subst3.perl' + ' ' + outdir
os.system(cmd)
cmd = 'cp ' + tdir + 'test' + ' ' + outdir
os.system(cmd)
cmd = 'mkdir ' + outdir + '/param'
os.system(cmd)
#
#--- create instruction page
#
tfile = tdir + 'instruction'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#LDATE_S#', ldate_sp)
input = input.replace('#LDATE#', ldate)
input = input.replace('#DDATE#', file_date)
input = input.replace('#YEAR#', syear)
input = input.replace('#DAYLIST#', daylist)
input = input.replace('#TELM_CMD#', telem_command)
input = input.replace('#INDEX#', index)
ofile = outdir + ldate.lower()
fo = open(ofile, 'w')
fo.write(input)
fo.close()
#
#--- focal temp related files
#
tfile = tdir + 'get_ftemp_wrap_script'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#MANDD#', ldate)
ofile = outdir + 'get_ftemp_wrap_script'
fo = open(ofile, 'w')
fo.write(input)
fo.close()
cmd = 'chmod 755 ' + outdir + 'get_ftemp_wrap_script'
os.system(cmd)
tfile = tdir + 'get_ftemp_main_script'
f = open(tfile, 'r')
input = f.read()
f.close()
#input = input.replace('#START#', str(start))
#input = input.replace('#STOP#', str(stop))
input = input.replace('#START#', str(start + 86400))
input = input.replace('#STOP#', str(stop + 86400))
ofile = outdir + 'get_ftemp_main_script'
fo = open(ofile, 'w')
fo.write(input)
fo.close()
cmd = 'chmod 755 ' + outdir + 'get_ftemp_main_script'
os.system(cmd)
tfile = tdir + 'plot_erad_time.pro'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#START#', str(start))
input = input.replace('#SDATELIST#', fpdsub)
input = input.replace('#LDATELIST#', fplsub)
ofile = outdir + 'plot_erad_time.pro'
fo = open(ofile, 'w')
fo.write(input)
fo.close()
cmd = 'cp -f ' + outdir + 'plot_erad_time.pro ./Focal/'
os.system(cmd)
#
#--- this_week file
#
tfile = tdir + 'this_week'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#DDATE#', file_date)
input = input.replace('#IRUSPAN1#', irudate)
input = input.replace('#IRUSPAN2#', irudate)
input = input.replace('#TITLE#', title)
input = input.replace('#TITLEDATE#', titledate)
atemp = last_trend_date
atemp = atemp.replace('/', '')
input = input.replace('#PREVREPORT#', atemp)
atemp = re.split('/', last_trend_date)
pmon = int(float(atemp[0]))
lmon = tcnv.changeMonthFormat(pmon)
line = lmon + ' ' + atemp[1]
#
#--- the previous report could be from the last year
#
ryear = syear
if mon < pmon:
ryear = year - 1
ryear = str(ryear)
input = input.replace('#RYEAR#', ryear)
input = input.replace('#PREVDATE#', line)
atitle = str(title)
atitle = atitle.replace(' ', '_')
#file = tdir + 'Headers/' + atitle
#fs = open(file, 'r')
#trend = fs.read()
#fs.close()
#input = input.replace("#TREND#", trend)
[temp1, temp2, temp3, temp4] = read_cti_values()
input = input.replace('#ATEMP#', temp1)
input = input.replace('#ATEMP2#', temp2)
input = input.replace('#DTEMP#', temp3)
input = input.replace('#DTEMP2#', temp4)
[val, step] = read_sim()
input = input.replace('#WMOVE#', val)
input = input.replace('#WSTEP#', step)
#
#--- make photon and bad pixel output
#
run_bad_pix_and_photon(outdir)
#
#--- run to get focal temp fits files
#
tstop = stop + 86400
[fcnt, fdata] = run_focal_temp_data(outdir, start, stop, fptemp)
[fcnt, fdata] = run_focal_temp_data_new()
input = input.replace('#TEMPPEAK#', str(fcnt))
input = input.replace('#TEMPLIST#', fdata)
#
#--- bad pixel
#
file = outdir + 'bad_pix_list'
fs = open(file, 'r')
bdata = fs.read()
fs.close()
input = input.replace('BAD_PIXEL_TABLE', bdata)
#
#--- photon
#
file = outdir + 'photons'
fs = open(file, 'r')
pdata = fs.read()
fs.close()
input = input.replace('PHOTON_TABLE', pdata)
#
#--- telem data
#
update_weekly_telem(year, byear, mon)
tdata = run_telem_data(telem_command, daylist, outdir)
input = input.replace('TELEM_TABLE', tdata)
#
#--- trend data
#
trend = set_trend_data_input(str(date))
input = input.replace('#TREND#', trend)
#
#--- write out the weekly report
#
ofile = outdir + file_name
fo = open(ofile, 'w')
fo.write(input)
fo.close()
#
#--- clean up
#
cmd = 'rm ./out ./out2'
os.system(cmd)
#
#--- move files
#
move_files(date, year, outdir, file_name, fptemp, odir)
#
#--- send out email to admin; notify the job complete
#
send_email_to_admin(date, year)
def useArc5gl(operation, dataset, detector, level, filetype, startYear = 0, startYdate = 0, stopYear = 0 , stopYdate = 0, deposit = './Working_dir/'):
"""
extract data using arc5gl.
input: start --- stop (year and ydate)
operation --- (e.g., retrive)
dataset ---(e.g. flight)
detector --- (e.g. hrc)
level --- (eg 0, 1, 2)
filetype ---(e.g, evt1)
startYear
startYdate
stopYear
stopYdate
output: data --- a list of fits file extracted
"""
#
#--- use arc5gl to extract ephin data
#
(year1, month1, day1, hours1, minute1, second1, ydate1) = tcnv.dateFormatCon(startYear, startYdate)
(year2, month2, day2, hours2, minute2, second2, ydate2) = tcnv.dateFormatCon(stopYear, stopYdate)
stringYear1 = str(year1)
stringYear2 = str(year2)
arc_start = str(month1) + '/' + str(day1) + '/' + stringYear1[2] + stringYear1[3]
arc_start = arc_start + ',' + str(hours1) + ':'+ str(minute1) + ':00'
arc_stop = str(month2) + '/' + str(day2) + '/' + stringYear2[2] + stringYear2[3]
arc_stop = arc_stop + ',' + str(hours2) + ':'+ str(minute2) + ':00'
intime = str(startYear) + ':' + adjust_ydate_format(startYdate) + ':00:00:00'
arc_start = tcnv.axTimeMTA(intime)
intime = str(stopYear) + ':' + adjust_ydate_format(stopYdate) + ':00:00:00'
arc_stop = tcnv.axTimeMTA(intime)
#print "I AM HERE: " + str(arc_start) + "<--->" + str(arc_stop)
line = 'operation=' + operation + '\n'
line = line + 'dataset=' + dataset + '\n'
line = line + 'detector=' + detector + '\n'
line = line + 'level=' + str(level) + '\n'
line = line + 'filetype=' + filetype + '\n'
line = line + 'tstart=' + str(arc_start) + '\n'
line = line + 'tstop=' + str(arc_stop) + '\n'
line = line + 'go\n'
f = open(zspace, 'w')
f.write(line)
f.close()
cmd = 'cd ' + deposit + '; arc5gl -user isobe -script ' + zspace + ' >./zlist'
os.system(cmd)
cmd = 'rm ' + zspace
os.system(cmd)
infile = deposit + '/zlist'
f = open(infile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + infile
os.system(cmd)
return data
def sib_corr_comb(start, stop, lev):
"""
combined fits files into one per ccd
input: start --- start time of the interval <yyyy>:<ddd>:<hh>:<mm>:<ss>
stop --- stop time of the interval <yyyy>:<ddd>:<hh>:<mm>:<ss>
lev --- data level "Lev1" or "Lve2"
output: combined data: lres_ccd<ccd>_merged.fits in Data directory
"""
#
#--- convert the time to seconds from 1998.1.1
#
tstart = tcnv.axTimeMTA(start)
tstop = tcnv.axTimeMTA(stop)
#
#--- make a list of data fits files
#
lres = s_dir + lev + '/Outdir/lres/'
cmd = 'ls ' + lres + '*fits > ' + zspace
os.system(cmd)
data = scf.read_file(zspace, remove=1)
#
#--- initialize ccd_list<ccd>
#
for ccd in range(0, 10):
exec 'ccd_list%s = []' % (str(ccd))
for ent in data:
#
#--- check whether the data are inside of the specified time period
#
[tmin, tmax] = find_time_interval(ent)
if tmin >= tstart and tmax <= tstop:
btemp = re.split('_acis', ent)
head = btemp[0]
#
#--- add the fits file to ccd_list
#
for ccd in range(0, 10):
chk = 'acis' + str(ccd)
mc = re.search(chk, ent)
if mc is not None:
line = str(ent)
exec "ccd_list%s.append('%s')" % (str(ccd), line)
break
#
#--- combined all fits files of a specific ccd into one fits file
#
for ccd in range(0, 10):
exec "alist = ccd_list%s" % (str(ccd))
if len(alist) > 0:
#
#--- the first of the list is simply copied to temp.fits
#
cmd = 'cp ' + alist[0] + ' temp.fits'
os.system(cmd)
for k in range(1, len(alist)):
cmd = 'dmmerge "' + alist[
k] + ',temp.fits" outfile=zmerged.fits outBlock=""'
cmd = cmd + 'columnList="" clobber="yes"'
try:
scf.run_ascds(cmd)
except:
continue
cmd = 'mv ./zmerged.fits ./temp.fits'
os.system(cmd)
cmd = 'mv ./temp.fits ' + s_dir + lev + '/Data/lres_ccd' + str(
ccd) + '_merged.fits'
os.system(cmd)
#
#--- pylab plotting routine related modules
#
from pylab import *
import matplotlib.pyplot as plt
import matplotlib.font_manager as font_manager
import matplotlib.lines as lines
#from matplotlib.transforms import Bbox
mag_plot = 1
if len(sys.argv) == 3:
start = sys.argv[1]
stop = sys.argv[2]
else:
print "Input: start stop in the format of 2014:204:00:00:00 or in seconds from 1998.1.1"
exit(1)
if __name__ == "__main__":
try:
start = int(float(start))
stop = int(float(stop))
except:
start = tcnv.axTimeMTA(start)
stop = tcnv.axTimeMTA(stop)
plot_data(start, stop, mag_plot)
def sib_corr_comb(start, stop, lev):
"""
combined fits files into one per ccd
input: start --- start time of the interval <yyyy>:<ddd>:<hh>:<mm>:<ss>
stop --- stop time of the interval <yyyy>:<ddd>:<hh>:<mm>:<ss>
lev --- data level "Lev1" or "Lve2"
output: combined data: lres_ccd<ccd>_merged.fits in Data directory
"""
#
#--- convert the time to seconds from 1998.1.1
#
tstart = tcnv.axTimeMTA(start)
tstop = tcnv.axTimeMTA(stop)
#
#--- make a list of data fits files
#
lres = s_dir + lev + '/Outdir/lres/'
cmd = 'ls ' + lres + '*fits > ' + zspace
os.system(cmd)
data = scf.read_file(zspace, remove=1)
#
#--- initialize ccd_list<ccd>
#
for ccd in range(0, 10):
exec 'ccd_list%s = []' % (str(ccd))
for ent in data:
#
#--- check whether the data are inside of the specified time period
#
[tmin, tmax] = find_time_interval(ent)
if tmin >= tstart and tmax <= tstop:
btemp = re.split('_acis', ent)
head = btemp[0]
#
#--- add the fits file to ccd_list
#
for ccd in range(0, 10):
chk = 'acis' + str(ccd)
mc = re.search(chk, ent)
if mc is not None:
line = str(ent)
exec "ccd_list%s.append('%s')" % (str(ccd), line)
break
#
#--- combined all fits files of a specific ccd into one fits file
#
for ccd in range(0, 10):
exec "alist = ccd_list%s" % (str(ccd))
if len(alist) > 0:
#
#--- the first of the list is simply copied to temp.fits
#
cmd = 'cp ' + alist[0] + ' temp.fits'
os.system(cmd)
for k in range(1, len(alist)):
cmd = 'dmmerge "' + alist[k] + ',temp.fits" outfile=zmerged.fits outBlock=""'
cmd = cmd + 'columnList="" clobber="yes"'
scf.run_ascds(cmd)
cmd = 'mv ./zmerged.fits ./temp.fits'
os.system(cmd)
cmd = 'mv ./temp.fits ' + s_dir + lev + '/Data/lres_ccd' + str(ccd) + '_merged.fits'
os.system(cmd)
def sci_run_add_to_rad_zone_list(file='NA'):
'adding radiation zone list to rad_zone_list. input: file name containing: e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto'
#
#--- check whether the list alread exists; if it does, read which ones are already in the list
#
cmd = 'ls ' + house_keeping + '* > ./ztemp'
os.system(cmd)
# f = open('./ztemp', 'r')
# test = f.readlines()
# f.close()
test = open('./ztemp').read()
m1 = re.search('rad_zone_list', test)
m2 = re.search('rad_zone_list~', test)
eventList = []
echk = 0
if m1 is not None:
line = house_keeping + 'rad_zone_list'
f = open(line, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
for ent in data:
atemp = re.split('\s+|\t+', ent)
eventList.append(atemp[0])
echk = 1
#
#--- if file is not given (if it is NA), ask the file input
#
if file == 'NA':
file = raw_input('Please put the intrrupt timing list: ')
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- put the list in the reverse order
#
data.reverse()
for ent in data:
if not ent:
break
#
#--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30)
#--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list
#--- to each date
#
etemp = re.split('\s+', ent)
echk = 0
for comp in eventList:
if comp == etemp[0]:
echk = 1
break
if echk == 0:
atemp = re.split(':', etemp[1])
year = atemp[0]
month = atemp[1]
date = atemp[2]
hour = atemp[3]
mins = atemp[4]
#
#--- convert to dom/sec1998
#
ydate = tcnv.findYearDate(int(year), int(month), int(date))
dom = tcnv.findDOM(int(year), int(ydate))
line = year + ':' + str(int(ydate)) + ':' + hour + ':' + mins + ':00'
csec = tcnv.axTimeMTA(line)
#
#--- end date
#
atemp = re.split(':', etemp[2])
eyear = atemp[0]
emonth = atemp[1]
edate = atemp[2]
ehour = atemp[3]
emins = atemp[4]
ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate))
line = eyear + ':' + str(int(ydate)) + ':' + ehour + ':' + emins + ':00'
csec2 = tcnv.axTimeMTA(line)
#
#--- date stamp for the list
#
list_date = str(year) + str(month) + str(date)
#
#--- check radiation zones for 3 days before to 5 days after from the interruptiondate
#--- if the interruption lasted longer than 5 days, extend the range 7 more days
#
begin = dom - 3
end = dom + 5
diff = csec2 - csec
if diff > 432000:
end += 7
#
#--- read radiation zone infornation
#
infile = house_keeping + '/rad_zone_info'
f = open(infile, 'r')
rdata = [line.strip() for line in f.readlines()]
f.close()
status = []
rdate = []
chk = 0
last_st= ''
cnt = 0
for line in rdata:
atemp = re.split('\s+', line)
dtime = float(atemp[1]) #--- dom of the entry or exit
if chk == 0 and atemp[0] == 'ENTRY' and dtime >= begin:
status.append(atemp[0])
rdate.append(dtime)
chk += 1
last_st = atemp[0]
cnt += 1
elif chk > 0 and dtime >= begin and dtime <= end:
status.append(atemp[0])
rdate.append(dtime)
last_st = atemp[0]
cnt += 1
elif atemp[1] > end and last_st == 'EXIT':
break
elif atemp[1] > end and last_st == 'ENTRY':
status.append(atemp[0])
rdate.append(dtime)
cnt += 1
break
f = open('./temp_zone', 'w')
#
#--- a format of the output is, e.g.: '20120313 (4614.2141112963,4614.67081268519):...'
#
line = list_date + '\t'
f.write(line)
upper = cnt -1
i = 0;
while i < cnt:
line = '(' + str(rdate[i]) + ','
f.write(line)
i += 1
if i < upper:
line = str(rdate[i]) + '):'
f.write(line)
else:
line = str(rdate[i]) + ')\n'
f.write(line)
i += 1
f.close()
#
#--- append the past rad zone list
#
oldFile = house_keeping + '/rad_zone_list~'
crtFile = house_keeping + '/rad_zone_list'
if m1 is not None:
cmd = 'cat '+ './temp_zone ' + crtFile + ' > ./temp_comb'
os.system(cmd)
else:
os.system('mv .temp_zone ./temp_comb')
os.system('rm ./temp_zone')
#
#--- save the old file and move the update file to rad_zone_list
#
if m2 is not None:
cmd = 'chmod 775 ' + crtFile + ' ' + oldFile
os.system(cmd)
if m1 is not None:
cmd = 'mv ' + crtFile + ' ' + oldFile
os.system(cmd)
cmd = 'chmod 644 ' + oldFile
os.system(cmd)
cmd = 'mv ' + './temp_comb ' + crtFile
os.system(cmd)
