def plot_each_column_data(data, ccd, dtype):
"""
create a trending plot for each column
input: data --- a list of data
ccd --- ccd nmae
dtype --- data type
output: <plot_dir>/<ccd>_<dtype>_<col_name>.png
"""
#
#--- separate the data by columns
#
dlist = mcf.separate_data_into_col_data(data)
#
#--- convert time from Chandra time to fractional year
#
t_list = []
for ent in dlist[0]:
ftime = mcf.chandratime_to_fraq_year(ent)
t_list.append(ftime)
#
#--- there are 10 different quantities to trend
#
for k in range(0, 10):
cpos = k + 2 #--- seond entry is obsid; so start from third one
pdata = dlist[cpos]
#
#--- clean up the data by dropping none numerical data points
#
[ctime, cdata] = clean_up_data(t_list, pdata)
#
#--- create trending plot for each data
#
create_trend_data_plot(ctime, cdata, ccd, dtype, col_snames[k],
col_dnames[k])
def update_datatable(per0, per1, per2, per3, per4, per5, per6, per7, per8):
"""
this function appends the newest data to the disk space data table
Input: per0 ... per5: new measures for each disk. currently per3 is empty
Output: <data_out>/disk_space_data (updated)
"""
#
#--- find out today's date in Local time frame
#
out = time.strftime('%Y:%j:%H:%M:%S', time.gmtime())
stime = Chandra.Time.DateTime(out).secs
ftime = mcf.chandratime_to_fraq_year(stime)
ftime = '%3.3f' % ftime
#
#--- today's data
#
line = str(ftime) + '\t' + str(per0) + '\t' + str(per1) + '\t' + str(
per2) + '\t'
line = line + str(per4) + '\t' + str(per5) + '\t' + str(per6) + '\t'
line = line + str(per7) + '\t' + str(per8) + '\n'
#
#--- append to the data table
#
ifile = data_out + 'disk_space_data'
with open(ifile, 'a') as f:
f.write(line)
def plot_rej_evt_data():
"""
create rejected event data plots
input: none, but read from <data_dir>/CCD<ccd#>_rej.dat
output: <web_dir>/Plots/ccd<ccd#>_<part>.png
"""
for ccd in range(0, 10):
#print("CCD: " + str(ccd))
ifile = data_dir + 'CCD' + str(ccd) + '_rej.dat'
data = mcf.read_data_file(ifile)
set1 = [[] for x in range(0, 5)]
set2 = [[] for x in range(0, 5)]
for dline in data[1:]:
ent = re.split('\s+', dline)
ytime = mcf.chandratime_to_fraq_year(float(ent[0]))
if float(ent[-2]) > 50000:
set1[0].append(ytime)
set1[1].append(float(ent[1]))
set1[2].append(float(ent[3]))
set1[3].append(float(ent[7]))
set1[4].append(float(ent[9]))
else:
set2[0].append(ytime)
set2[1].append(float(ent[1]))
set2[2].append(float(ent[3]))
set2[3].append(float(ent[7]))
set2[4].append(float(ent[9]))
plot_data(set1, ccd, 'cti')
plot_data(set2, ccd, 'sci')
def read_fid_light_data(infile):
"""
read fid light data
input: infile --- input file name
output: atime --- time in fractional year
acen_i --- acent i
acen_j --- acent j
"""
out = mcf.read_data_file(infile)
atime = []
acen_i = []
acen_j = []
for ent in out:
try:
atemp = re.split('\s+', ent)
var1 = mcf.chandratime_to_fraq_year(float(atemp[0]))
var2 = float(atemp[4])
var3 = float(atemp[5])
except:
continue
atime.append(var1)
acen_i.append(var2)
acen_j.append(var3)
return [atime, acen_i, acen_j]
def read_data_info(year):
"""
read data info data for a given year
input: year --- the year of the data set
output: [time, sim_x, sim_y, sim_z, pitch, yaw]
"""
infile = data_dir + 'data_info_' + str(year)
data = mcf.read_data_file(infile)
time = []
sim_x = []
sim_y = []
sim_z = []
pitch = []
yaw = []
for ent in data:
atemp = re.split('\s+', ent)
out = mcf.convert_date_format(atemp[3],
ifmt='%Y-%m-%dT%H:%M:%S',
ofmt='chandra')
start = mcf.chandratime_to_fraq_year(out)
out = mcf.convert_date_format(atemp[4],
ifmt='%Y-%m-%dT%H:%M:%S',
ofmt='chandra')
stop = mcf.chandratime_to_fraq_year(out)
mid = 0.5 * (start + stop)
try:
val1 = float(atemp[5])
val2 = float(atemp[6])
val3 = float(atemp[7])
val4 = float(atemp[8]) * 3600.0
val5 = float(atemp[9]) * 3600.0
time.append(mid)
sim_x.append(val1)
sim_y.append(val2)
sim_z.append(val3)
pitch.append(val4)
yaw.append(val5)
except:
continue
return [time, sim_x, sim_y, sim_z, pitch, yaw]
def stime_to_frac_year(stime):
"""
convert seconds from 1998.1.1 to fractional year format
input: stime --- seconds from 1998.1.1
etime --- time in fractinal year;, e.g., 2012.223
"""
etime = mcf.chandratime_to_fraq_year(stime)
return etime
def read_focus_data(infile):
"""
read data file and return lists of times and values
input: infile --- data file name
output: t_list --- a list of time data
c1_list --- a list of data (ax slf 10%)
c2_list --- a list of data (ax slf 50%)
c3_list --- a list of data (streak slf 10%)
c4_list --- a list of data (streak slf 50%)
c5_list --- a list of data (ax slf fwhm)
c6_list --- a list of data (streak slf fwhm)
c7_list --- a list of data (ax slf fwhm error)
c8_list --- a list of data (streak slf fwhm error)
"""
infile = data_dir + infile
print("Data: " + str(infile))
data = mcf.read_data_file(infile)
t_list = []
c1_list = []
c2_list = []
c3_list = []
c4_list = []
c5_list = []
c6_list = []
c7_list = []
c8_list = []
for ent in data:
atemp = re.split('\s+', ent)
try:
t = mcf.chandratime_to_fraq_year(float(atemp[0]))
v1 = float(atemp[1])
v2 = float(atemp[2])
v3 = float(atemp[3])
v4 = float(atemp[4])
v5 = float(atemp[5])
v6 = float(atemp[6])
v7 = float(atemp[6])
v8 = float(atemp[6])
except:
continue
t_list.append(t)
c1_list.append(v1)
c2_list.append(v2)
c3_list.append(v3)
c4_list.append(v4)
c5_list.append(v5)
c6_list.append(v6)
c7_list.append(v7)
c8_list.append(v8)
return [
t_list, c1_list, c2_list, c3_list, c4_list, c5_list, c6_list, c7_list,
c8_list
]
def convert_stime_to_fyear_list(dlist):
save = []
for ent in dlist:
fyear = mcf.chandratime_to_fraq_year(ent)
save.append(fyear)
save = numpy.array(save)
return save
def current_time():
"""
return current time in fractional year
input: none
output: fyear
"""
otime = time.strftime('%Y:%j:%H:%M:%S', time.gmtime())
stime = Chandra.Time.DateTime(otime).secs
fyear = mcf.chandratime_to_fraq_year(stime)
return fyear
def convert_to_fyear_list(t_list):
"""
convert time data in seconds in a list to fractional year in the list
input: t_list --- a list of time data in seconds from 1998.1.1
output: t_list --- an array of time data in fractional year
"""
save = []
for ent in t_list:
save.append(mcf.chandratime_to_fraq_year(ent))
save = numpy.array(save)
return save
def plot_dusk_result(diskName, duskName, nameList, pastData):
"""
read dusk result, update data file, and plot the data
input: diskName --- name of the disk
duskName --- a file name in which dusk result is written
nameList --- subdirectory names
pastData --- data file name
output: udated <data_out>/disk_data_<disk name>
<plot_dir>/<disk_name>_disk.png
"""
#
#--- find the disk capacity of the given disk
#
disk_capacity = diskCapacity(diskName)
#
#--- read the output from dusk
#
line = run_dir + duskName
data = mcf.read_data_file(line)
capacity = {} #---- make a dictionary
for ent in data:
atemp = re.split('\s+|\t+', ent)
try:
val = 100.0 * float(atemp[0]) / disk_capacity
val = round(val, 2)
atemp[1] = atemp[1].replace('./', '')
capacity[atemp[1]] = val
except:
pass
#
#--- today's date
#
out = time.strftime('%Y:%j:%H:%M:%S', time.gmtime())
stime = Chandra.Time.DateTime(out).secs
ftime = mcf.chandratime_to_fraq_year(stime)
#
#--- append the new data to the data table
#
sline = '%3.3f' % ftime
for dName in nameList:
sline = sline + ':' + str(capacity[dName])
sline = sline + '\n'
with open(pastData, 'a') as fo:
fo.write(sline)
#
#---- start plotting history data
#
xxx = 999
if xxx == 999:
#try:
plot_history_trend(diskName, duskName, nameList, pastData)
def create_recent_one_year_plot():
"""
create most recent one year trending plots
input: none, but read from <data_dir>/<slot name>_<#>
output: <web_dir>/Plots/<slot name>_<#>_recent_1yr.png
"""
for m in range(0, 3):
ifile = data_dir + data_list[m]
#
#--- read data, separate into column data, and select the data only between time period specified
#
data_set = mcf.read_data_file(ifile)
data_set = mcf.separate_data_to_arrays(data_set)
#
#--- find the data starting time (one year before the last data point)
#
tstop = data_set[0][-1]
tstart = tstop - oneyear
data_set = select_data_for_time_period(data_set, tstart, tstop)
#
#--- save time in fractional year
#
time_list = []
for tval in data_set[0]:
time_list.append(mcf.chandratime_to_fraq_year(tval))
for k in range(1, len(data_set)):
#
#--- acacent_mtatr hold 2 sets of data
#
if m == 2:
if k < 9:
y_name = slot_name[m] + 'ynea_' + str(k - 2)
else:
y_name = slot_name[m] + 'znea_' + str(k - 9)
else:
y_name = slot_name[m] + '_' + str(k - 2)
out_name = web_dir + 'Plots/' + y_name + '_recent_1yr.png'
[t_list, d_list] = drop_nan(time_list, data_set[k])
if len(t_list) < 1:
cmd = 'cp ' + house_keeping + 'no_data.png' + ' ' + out_name
os.system(cmd)
continue
xlabel = 'Time (year)'
plot_data(t_list, d_list, xlabel, y_name, out_name, xs=1)
def generate_all_plot():
"""
a control function to create bias - overclock plots
input: None but read from:
<data_dir>/Bias_save/CCD<ccd>/quad<quad>
output: <web_dir>/Plots/Sub2/bias_plot_ccd<ccd>_quad<quad>.png
"""
for ccd in range(0, 10):
for quad in range(0, 4):
#
#--- set input and output file names
#
ifile = data_dir + 'Bias_save/CCD' + str(ccd) + '/quad' + str(quad)
oname = web_dir + 'Plots/Sub2/bias_plot_ccd' + str(ccd)
oname = oname + '_quad' + str(quad) + '.png'
#
#--- read data
#
data = mcf.read_data_file(ifile)
x = [] #---- original x
y = [] #---- original y
for ent in data:
atemp = re.split('\s+', ent)
try:
stime = float(atemp[0])
valx = mcf.chandratime_to_fraq_year(stime)
valy = float(atemp[1]) - float(atemp[3])
#
#--- if the difference si too large, drop it
#
if abs(valy) > 10:
continue
x.append(valx)
y.append(valy)
except:
pass
#
#--- compute moving average and upper and lower envelopes
#--- here, we set moving interval to 0.08 year (about 30 days) and asked 5th degree
#--- polynomial fit for smoothing
#
moving_avg = fmv.find_moving_average(x, y, 0.08, 5)
#
#--- plot the data and fittings
#
plot_bias_trend(x, y, moving_avg, oname, ccd, quad)
def convert_time(time, format = 0):
"""
convert time format from seconds from 1998.1.1 to data of year or fractional year
Input: time --- a list of time in seconds
format --- if 0, convert into day of year, otherwise, fractional year
Output: timeconverted --- a list of conveted time
"""
t_list = []
if format == 0:
for ent in time:
t_list.append(mcf.chandratime_to_yday(ent))
else:
for ent in time:
t_list.append(mcf.chandratime_to_fraq_year(ent))
return t_list
def read_data(infile):
"""
read data file and return lists of times and values
input: infile --- data file name
output: t_list --- a list of time data
v_list --- a list of data
"""
data = mcf.read_data_file(infile)
t_list = []
v_list = []
for ent in data:
atemp = re.split('\s+', ent)
t_list.append(mcf.chandratime_to_fraq_year(int(atemp[0])))
v_list.append(float(atemp[1]))
return [t_list, v_list]
def get_data_set(head, ccd, skip=1):
"""
read data a data file and create lists of x and y
input: head --- header of the data file
ccd --- ccd #
skip --- step; default: 1 -- read every data point
output: [x, y] --- a list of lists of x and y
"""
ifile = data_dir + head + '_ccd' + str(ccd) + '.dat'
data = mcf.read_data_file(ifile)
x = []
y = []
for ent in data[::skip]:
atemp = re.split('\s+', ent)
val = int(float(atemp[0]))
val = mcf.chandratime_to_fraq_year(val)
x.append(val)
y.append(int(float(atemp[1])))
return [x, y]
def read_zero_data(infile):
"""
read data file and return lists of times and values
input: infile --- data file name
output: t_list --- a list of time data
c1_list --- a list of data (sky_x)
c2_list --- a list of data (sky_y)
c3_list --- a list of data (chip_x)
c4_list --- a list of data (chip_y)
"""
infile = data_dir + infile
data = mcf.read_data_file(infile)
dout = mcf.separate_data_to_arrays(data)
t_list = [mcf.chandratime_to_fraq_year(x) for x in dout[0]]
c1_list = dout[1]
c2_list = dout[2]
c3_list = dout[3]
c4_list = dout[4]
return [t_list, c1_list, c2_list, c3_list, c4_list]
def read_ede_data(ifile):
"""
read data from a given file
Input: ifile --- input file name
Output: date_list --- a list of date
ede_list --- a list of ede value
error_list --- a list of computed ede error
"""
data = mcf.read_data_file(ifile)
date_list = []
date_list2 = []
ede_list = []
error_list = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.is_neumeric(atemp[0]) == False:
continue
fwhm = float(atemp[2])
ferr = float(atemp[3])
ede = float(atemp[4])
date = atemp[5]
sdate = float(atemp[6])
stime = Chandra.Time.DateTime(date).secs
fyear = mcf.chandratime_to_fraq_year(stime)
date_list.append(fyear)
date_list2.append(sdate)
ede_list.append(ede)
#
#--- the error of EdE is computed using FWHM and its error value
#
error = math.sqrt(ede * ede * ((ferr * ferr) / (fwhm * fwhm)))
error_list.append(error)
return [date_list, date_list2, ede_list, error_list]
def read_data_extracted(year):
"""
read extracted data for a given year
input: year --- the year of the data set
ouptput: [time, dy, dz, dtheta, inst]
"""
#
#--- sometime there are out of spot data in the file; so remove them
#
xbot = str(year) + ':001:00:00:00'
xbot = out = Chandra.Time.DateTime(xbot).secs
infile = data_dir + 'data_extracted_' + str(year)
data = mcf.read_data_file(infile)
time = []
dy = []
dz = []
dtheta = []
inst = []
for ent in data:
atemp = re.split('\s+', ent)
try:
if float(atemp[0]) < xbot:
continue
tval = mcf.chandratime_to_fraq_year(atemp[0])
yval = float(atemp[3])
zval = float(atemp[4])
hval = float(atemp[5]) * 3600.0
time.append(tval)
dy.append(yval)
dz.append(zval)
dtheta.append(hval)
inst.append(atemp[2])
except:
continue
return [time, dy, dz, dtheta, inst]
def create_adjusted_cti_tables(ccd_list, sub_dir, sub2_dir_list, out_dir):
"""
create adjusted cti table: Data119, Data2000, Data7000, Data_cat-adjust
Input: ccd_list --- a list of ccd #
sub_dir --- a name of sub directory which cti lists are read
sub2_dir_list --- a list of sub directories which adjusted cti tables are deposited
out_dir --- a directory name which adjucted cti will be deposited
cti data are read from <data_dir>/<sub_dir>/<elm>_ccd<ccd #>
Output: <data_dir>/<dir in sub2_dir_list>/<elm>_ccd<ccd#>
"""
for elm in elm_list:
for ccd in ccd_list:
if ccd in [5, 7]:
factor = 0.045 #--- these factors are given by C. Grant
else:
factor = 0.036
#
#--- read the main data set
#
ifile = data_dir + sub_dir + '/' + elm + '_ccd' + str(ccd)
data = mcf.read_data_file(ifile)
save1 = []
save2 = []
save3 = []
save4 = []
time3 = []
time4 = []
del_temp = []
for ent in data:
#
#--- convert time format from 2013-10-05T17:35:58 to year, stime and fractional year
#
atemp = re.split('\s+', ent)
out = time.strftime(
'%Y:%j:%H:%M:%S',
time.strptime(atemp[0], '%Y-%m-%dT%H:%M:%S'))
sectime = Chandra.Time.DateTime(out).secs
fyear = mcf.chandratime_to_fraq_year(sectime)
tspan = int(atemp[12]) - int(atemp[11])
temperature = float(atemp[7])
#
#--- use only data with the integration time longer than 1000 sec before 2003
#--- and only data with the integration time longer than 2000 sec after 2003
#
if tspan < 1000:
pass
elif (tspan < 2000) and (fyear >= 2003):
pass
line = ent + '\n'
save3.append(line)
time3.append(fyear)
#
#--- we need to adjust focal plane temperature between 9/16/2005 - 10/16/2005
#--- a reported temperature is about 1.3 warmer than a real focal temperature
#--- (from 12/1/05 email from C. Grant)
#
if (sectime >= 243215999) and (sectime <= 245894399):
temperature -= 1.3
if temperature <= -119.5:
line = ent + '\n'
save1.append(line)
if tspan >= 7000:
save4.append(line)
time4.append(fyear)
#
#--- correct temperature dependency with C. Grat factors
#
val = factor * (temperature + 119.87)
quad0 = select_grant_cti(atemp[1], val)
quad1 = select_grant_cti(atemp[2], val)
quad2 = select_grant_cti(atemp[3], val)
quad3 = select_grant_cti(atemp[4], val)
if (quad0 != 'na') and (quad1 != 'na') and (
quad2 != 'na') and (quad3 != 'na'):
line = atemp[0] + '\t'
line = line + quad0 + '\t'
line = line + quad1 + '\t'
line = line + quad2 + '\t'
line = line + quad3 + '\t'
line = line + atemp[5] + '\t'
line = line + atemp[6] + '\t'
line = line + atemp[7] + '\t'
line = line + atemp[8] + '\t'
line = line + atemp[9] + '\t'
line = line + atemp[10] + '\t'
line = line + atemp[11] + '\t'
line = line + atemp[12] + '\n'
save2.append(line)
#
#--- print out adjsted cti data table
#
j = 0
for sdir in sub2_dir_list:
j += 1
sdata = eval('save%s' % (j))
print_cti_results(sdir, elm, ccd, sdata)
#
#---- compute adjusted cti values and update tables
#
compute_adjusted_cti(elm, ccd, time3, save3, time4, save4, out_dir)
def acis_gain_plot_trend():
"""
plotting trends of gain and offset
Input: none, but read from <data_dir>
Output: <web_dir>/Plots/gain_plot_ccd<ccd>.png
<web_dir>/Plots/offset_plot_ccd<ccd>.png
"""
for ccd in range(0, 10):
#
#--- plotting 4 nodes on one panel, but gain and offset separately
#
Xset_gain = []
Yset_gain = []
Eset_gain = []
yMin_gain = []
yMax_gain = []
Label_gain = []
Xset_offset = []
Yset_offset = []
Eset_offset = []
yMin_offset = []
yMax_offset = []
Label_offset = []
for node in range(0, 4):
#
#--- read data for given CCD and Node #
#
ifile = data_dir + 'ccd' + str(ccd) + '_' + str(node)
data = mcf.read_data_file(ifile)
time = []
gain = []
gerr = [] #--- error for gain
offset = []
oerr = [] #--- error for offset
#
#--- setting lower and upper limits to remove outlyers
#
sum1 = 0.0
sum2 = 0.0
for ent in data:
atemp = re.split('\s+', ent)
gval = float(atemp[4])
sum1 += gval
sum2 += gval * gval
avg = sum1 / len(data)
sig = math.sqrt(sum2 / len(data) - avg * avg)
blim = avg - 3.0 * sig
tlim = avg + 3.0 * sig
for ent in data:
atemp = re.split('\s+', ent)
#
#--- convert time into year date (e.g.2012.14)
#
gval = float(atemp[4])
if (gval <= blim) or (gval >= tlim):
continue
ytime = mcf.chandratime_to_fraq_year(float(atemp[0]))
time.append(ytime)
gain.append(float(atemp[4]))
gerr.append(float(atemp[5]))
offset.append(float(atemp[6]))
oerr.append(float(atemp[7]))
xmax = max(time)
Xset_gain.append(time)
Yset_gain.append(gain)
Eset_gain.append(gerr)
#
#--- set plotting range
#
avg = mean(gain)
ymin = avg - 0.002
ymin = round(ymin, 3) - 0.001
ymax = avg + 0.002
ymax = round(ymax, 3) + 0.001
yMin_gain.append(ymin)
yMax_gain.append(ymax)
name = 'Gain (ADU/eV) Node' + str(node)
Label_gain.append(name)
Xset_offset.append(time)
Yset_offset.append(offset)
Eset_offset.append(oerr)
avg = mean(offset)
ymin = avg - 8.0
ymin = int(ymin)
ymax = avg + 8.0
ymax = int(ymax)
yMin_offset.append(ymin)
yMax_offset.append(ymax)
name = 'Offset (ADU) Node' + str(node)
Label_offset.append(name)
xmin = int(2000)
xtmp = xmax
xmax = int(xmax) + 1
#
#--- if the year is already passed a mid point, add another year
#
if (xtmp - xmax) > 0.5:
xmax += 1
xname = 'Time (year)'
#
#--- actual plotting starts here
#
yname = 'Gain'
outname = web_dir + "/Plots/gain_plot_ccd" + str(ccd) + '.png'
plotPanel(xmin, xmax, yMin_gain, yMax_gain, Xset_gain, Yset_gain, \
Eset_gain, xname, yname, Label_gain, outname)
yname = 'Offset'
outname = web_dir + "/Plots/offset_plot_ccd" + str(ccd) + '.png'
plotPanel(xmin, xmax, yMin_offset, yMax_offset, Xset_offset, Yset_offset,\
Eset_offset, xname, yname, Label_offset, outname)
def clean_cti_data_table(idir):
"""
remmove data points which are extrme outlyers and then clean up output data tables.
Input: idir --- the directory where the data files are kept
Output: updated data files in the directory <dir>
"""
dropped_obsids = []
sline = ''
for elm in elm_list:
sline = sline + 'ELM: ' + elm + '\n'
for ccd in range(0, 10):
#
#--- drop_factor sets the boundray of the outlyer: how may signam away?
#
if ccd in [5, 7]:
drop_factor = 5.0
else:
drop_factor = 4.0
#
#--- check the input file exists
#
dname = data_dir + idir + '/' + elm + '_ccd' + str(ccd)
if os.stat(dname).st_size > 0:
sline = sline + 'CCD: ' + str(ccd) + '\n'
data = mcf.read_data_file(dname)
#
#--- separate data into separate array data sets
#
dcolumns = separate_data(data)
cti = ['' for x in range(4)]
cti[0] = dcolumns[0]
cti[1] = dcolumns[1]
cti[2] = dcolumns[2]
cti[3] = dcolumns[3]
obsid = dcolumns[10]
fy_list = []
for ent in dcolumns[-2]:
#
#--- dcolumns[-2] is the end time in seconds from 1998.1.1; convert to fractional year
#
fyr = mcf.chandratime_to_fraq_year(ent)
fy_list.append(fyr)
#
#--- go around quads
#
drop_list = []
for i in range(0, 4):
sline = sline + "QUAD" + str(i) + '\n'
#
#--- fit a lienar line
#
(intc, slope) = linear_fit(fy_list, cti[i])
isum = 0
#
#--- compute a deviation from the fitted line
#
diff_save = []
for j in range(0, len(fy_list)):
diff = float(
cti[i][j]) - (intc + slope * float(fy_list[j]))
diff_save.append(diff)
isum += diff * diff
sigma = math.sqrt(isum / len(fy_list))
#
#--- find outlyers
#
out_val = drop_factor * sigma
for j in range(0, len(fy_list)):
if diff_save[j] > out_val:
drop_list.append(j)
sline = sline + data[j] + '\n'
#
#--- clean up the list; removing duplicated lines
#
drop_list = sorted(list(set(drop_list)))
cleaned_data = []
for i in range(0, len(fy_list)):
chk = 0
for comp in drop_list:
if i == comp:
chk = 1
break
if chk == 0:
cleaned_data.append(data[i])
cleaned_data = sorted(set(cleaned_data))
for ent in drop_list:
dropped_obsids.append(obsid[ent])
with open(dname, 'w') as f:
for ent in cleaned_data:
f.write(ent + '\n')
dropped = data_dir + idir + '/dropped_data'
with open(dropped, 'w') as fo:
fo.write(sline)
dropped_obsids = sorted(set(dropped_obsids))
out = data_dir + idir + '/bad_data_obsid'
with open(out, 'w') as f:
for ent in dropped_obsids:
f.write(ent + '\n')
def create_sim_temp_plots(start, stop):
"""
control function to run sim temperature trending plots etc
input: start --- starting year
stop --- stopping year
if they are the same, it will cerate the plot for the year
else, it will create the entire range (from 1999 to current)
output: plots in <web_dir>/Plots/
sim_temp_<year>.png
sim)translation_<year>.png
"""
#
#--- check whether to create full range or indivisual year plots
#
if stop - start > 1:
fname = web_dir + 'Plots/sim_temp_fullrange.png'
oname = web_dir + 'Plots/sim_translation_fullrange.png'
title = str(start) + ' - ' + str(stop - 1)
yind = 0
else:
fname = web_dir + 'Plots/sim_temp_' + str(start) + '.png'
oname = web_dir + 'Plots/sim_translation_' + str(start) + '.png'
title = 'Year: ' + str(start)
yind = 1
if mcf.is_leapyear(start):
base = 366
else:
base = 365
#
#--- read sim temperature data file
#
ifile = data_dir + 'tsc_temps.txt'
data = mcf.read_data_file(ifile)
x = []
ts = []
te = []
delta = []
steps = []
for ent in data[1:]:
atemp = re.split('\s+', ent)
try:
time = convert_to_fyear(atemp[0])
if time < start:
continue
if time > stop:
break
if yind == 1:
time = (time - start) * base
t1 = float(atemp[2])
t2 = float(atemp[3])
t3 = t2 - t1
t4 = float(atemp[5])
if t1 > 60 or t1 < -40:
continue
if t2 > 60 or t2 < -40:
continue
x.append(time) #--- time in year or yday
ts.append(t1) #--- sim temperature at the beginning
te.append(t2) #--- sim temperature at the ending
delta.append(t3) #--- the sim temperature difference
steps.append(t4) #--- the number of steps
except:
continue
#
#--- run moving average
#
mvstep = 30
[mxs, mys] = smooth_data(x, ts, mvstep)
if len(mxs) == 0:
skip = 1
else:
skip = 0
[mxe, mye] = smooth_data(x, te, mvstep)
if len(mxe) == 0:
skip = 1
else:
skip = 0
#
#--- read pitch angle data
#
ifile = '/data/mta/DataSeeker/data/repository/orb_angle.rdb'
data = mcf.read_data_file(ifile)
stime = []
angle = []
m = 0
for ent in data[2:]:
atemp = re.split('\s+', ent)
try:
time = mcf.chandratime_to_fraq_year(float(atemp[0]))
if time < start:
continue
if time > stop:
break
if yind == 1:
time = (time - start) * base
#
#--- just take the data every 20 mins or so (data is taken every 300 sec)
#
if m % 4 == 0:
stime.append(time)
angle.append(float(atemp[1]))
m += 1
except:
continue
#
#--- if yind == 0, full range, otherwise, year plot
#
if yind == 0:
xmin = start
xmax = stop
tdff = 1.0e-5
else:
xmin = 1
xmax = base
tdff = 3.5e-3
#
#--- plot time tred of sim temperature
#
plot_sim_temp_data(xmin, xmax, x, ts, te, mxs, mys, mxe, mye,\
delta, stime, angle, fname, yind, title, skip)
#
#--- plot translation step - delta sim temperature
#
plot_step_delta(steps, delta, oname, title)
def plot_data(data_set, ccd, part):
"""
create multi-panel plots
input: data_set --- a list of lists of data
ccd --- ccd #
part --- either "cti" or "sci"
output: <web_dir>/Plots/ccd<ccd#>_<part>.png
"""
ylab_list = ['EVTSENT', 'DROP_AMP', 'DROP_GRD', 'THR_PIX']
ymax_list = [300, 300, 700, 3e4]
now = time.strftime("%Y:%j:%H:%M:%S", time.gmtime())
now = Chandra.Time.DateTime(now).secs
now = mcf.chandratime_to_fraq_year(now)
xmin = 2000.0
now += 1.4
xmax = int(round(now, 1))
plt.close('all')
#
#---- set a few parameters
#
mpl.rcParams['font.size'] = 10
mpl.rcParams['font.weight'] = 'medium'
props = font_manager.FontProperties(size=10)
props = font_manager.FontProperties(weight='medium')
plt.subplots_adjust(hspace=0.08)
for ax in range(0, 4):
axnam = 'ax' + str(ax)
j = ax + 1
line = '61' + str(j)
exec("%s = plt.subplot(%s)" % (axnam, line))
exec("%s.set_autoscale_on(False)" % (axnam))
exec("%s.set_xbound(xmin,xmax)" % (axnam))
exec("%s.set_xlim(xmin=xmin, xmax=xmax, auto=False)" % (axnam))
exec("%s.set_ylim(ymin=0, ymax=%s, auto=False)" %
(axnam, str(ymax_list[ax])))
plt.plot(data_set[0], data_set[j],marker='.', mec='blue', \
markerfacecolor='blue', markersize='2.0', lw=0)
ylabel(ylab_list[ax], fontweight='medium')
#
#--- add x ticks label only on the last panel
#
for i in range(0, 4):
axnam = 'ax' + str(i)
if i != 3:
line = eval("%s.get_xticklabels()" % (axnam))
for label in line:
label.set_visible(False)
else:
pass
xlabel('Time (Year)', fontweight='medium')
#
#--- set the size of the plotting area in inch (width: 10.0in, height 2.08in x number of panels)
#
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(10.0, 8.0)
#
#--- save the plot in png format
#
outname = html_dir + 'Plots/ccd' + str(ccd) + '_' + part + '.png'
plt.savefig(outname, format='png', dpi=200, bbox_inches='tight')
plt.close('all')
def prep_and_plot(pos, xmin_range, x_list, y_list, inst, drange, prefix, today,
yname):
"""
prepare data for specific plot and plot it
input: pos --- indicator of instrument
xmin_range --- a list of starting time
x_list --- an array of time data
y_list --- an array of selected msid data
inst --- an array of instrument (in values between 0 and 4)
drange --- a list of data range
prefix --- indicator of which data set
today --- today's time in seconds from 1998.1.1
yname --- y axis label
output: <web_dir>/Position/<msid>_<inst>_<range>.png
"""
#
#--- select data for the instrument
#
if pos == 4: #--- combine data set
xr = x_list
yr = y_list
else:
indx = inst == pos
xr = x_list[indx]
yr = y_list[indx]
title = detectors[pos].replace('-', '_')
y_range = drange[pos]
#
#--- week, month, year, and full range plots
#
#for k in range(0, 4):
for k in range(3, 4):
tail = tail_list[k]
outname = web_dir + 'Position/' + prefix + '_' + title.lower(
) + '_' + tail + '.png'
x_range = [xmin_range[k], today + add_list[k]]
#
#--- converting to yday
#
if k in [0, 1, 2]:
indx = xr > xmin_range[k]
x = xr[indx]
y = yr[indx]
if len(x) < 1:
cmd = 'cp ' + house_keeping + 'no_plot.png ' + outname
os.system(cmd)
continue
byear, x = sms.convert_time_format(x, 0)
[year1, start] = sms.chandratime_to_yday(x_range[0])
[year2, stop] = sms.chandratime_to_yday(x_range[1])
if year1 == year2:
x_range = [start, stop]
else:
if mcf.is_leapyear(byear):
base = 366
else:
base = 365
if byear == year1:
x_range = [start, stop + base]
else:
x_range = [start - base, stop]
xname = 'Time (YDay in Year: ' + str(byear) + ')'
#
#--- converting to fractional year
#
else:
byear, x = sms.convert_time_format(xr, 1)
y = yr
start = mcf.chandratime_to_fraq_year(x_range[0])
stop = mcf.chandratime_to_fraq_year(x_range[1])
x_range = [start, stop]
xname = 'Time (in Year)'
sms.plot_panel(x, y, x_range, y_range, xname, yname, title, outname)
def plot_dead_time_trend():
"""
update hrc_i_115_dtf_plot.png plot file
input: none, but read from hrc_i_115_stat_results
output: <plot dir>/hrc_i_115_dtf_plot.png
"""
#
#--- read data
#
dfile = data_dir + 'Stats/hrc_i_115_stat_results'
data = mcf.read_data_file(dfile)
time = []
cnt_rate = []
v_rate = []
dtf = []
#
#--- computer dead time correciton fuctor
#
for ent in data:
atemp = re.split('\s+', ent)
date = mcf.chandratime_to_fraq_year(float(atemp[1]))
a_rate = float(atemp[5]) / float(atemp[13])
time.append(date)
cnt_rate.append(float(atemp[5]))
v_rate.append(float(atemp[13]))
dtf.append(a_rate)
#
#--- setting a plotting surface
#
prep_plot()
mpl.rcParams['font.size'] = 11
props = font_manager.FontProperties(size=11)
psize = 8
tfsize = 12
#
#--- start plotting
#
xmin = 1999
xmax = int(max(time)) + 1
#
#--- Count Rate per Sec
#
a1 = plt.subplot(311)
ymin = 3.45
ymax = 3.55
title = 'Count Rate per Sec'
plt.axis([xmin, xmax, ymin, ymax])
plt.plot(time, cnt_rate, color='blue', lw=0, marker='+', markersize=psize)
plt.text(2000, 3.538, title, size=tfsize, weight='bold')
#
#--- Valid Count Rate per Sec
#
a2 = plt.subplot(312)
ymin = 0
ymax = 200
title = 'Valid Count Rate per Sec'
plt.axis([xmin, xmax, ymin, ymax])
plt.plot(time, v_rate, color='blue', lw=0, marker='+', markersize=psize)
plt.text(2000, 160, title, size=tfsize, weight='bold')
#
#--- Dead Time Correction Fator
#
a3 = plt.subplot(313)
ymin = 0
ymax = 0.1
title = 'Dead Time Correctin Factor'
plt.axis([xmin, xmax, ymin, ymax])
plt.plot(time, dtf, color='blue', lw=0, marker='+', markersize=psize)
plt.text(2000, 0.02, title, size=tfsize, weight='bold')
#
#--- add x ticks label only on the panels of the last row
#
plt.setp(a1.get_xticklabels(), visible=False)
plt.setp(a2.get_xticklabels(), visible=False)
a3.set_xlabel("Time (Year)", size=11)
#
#--- save the plot in file
#
outname = 'hrc_i_115_dtf_plot.png'
outname = plot_dir + outname
create_plot_file(outname)
def read_stat_data(infile):
"""
read a stat data file and return a list of lists of data
input: infile --- input file name (with a full path)
output: 0: time --- time in year date
1: tstart --- start time in seconds from 1998.1.1
2: tstop --- stop time in seconds from 1998.1.1
3: duration --- duration in seconds
4: total_count --- total counts
5: count_per_sec --- counts per seonds
6: pha_mean --- pha mean
7: pha_median --- pha median
8: pha_sigma --- pha sigma
9: t_mean --- total count rate mean
10: t_median --- total count rate median
11: t_sigma --- total count rate sigma
12: v_mean --- valid count rate mean
13: v_median --- valid count rate median
14: v_sigma --- valid count rate sigma
15: s_mean --- shield count rate mean
16: s_median --- shield count rate median
17: s_sigma --- shield count rate sigma
18: anti_co_mean --- anti conicidnece rate mean
19: anti_co_median --- anti conicidnece rate median
20: anti_co_sigma --- anti conicidnece rate sigma
21: s2hvst --- s2hvst value
22: s2hvlv --- s2jvlv valie
23: scint --- scint
24: scint_std --- scint sigma
"""
data = mcf.read_data_file(infile)
date = []
tstart = []
tstop = []
duration = []
total_count = []
count_per_sec = []
pha_mean = []
pha_median = []
pha_sigma = []
t_mean = []
t_median = []
t_sigma = []
v_mean = []
v_median = []
v_sigma = []
s_mean = []
s_median = []
s_sigma = []
anti_co_mean = []
anti_co_median = []
anti_co_sigma = []
s2hvst = []
s2hvlv = []
scint = []
scint_std = []
time = []
for ent in data:
aval = re.split('\s+', ent)
aval3 = float(aval[3])
#
#--- if the observation interval is shorter than 900 sec, drop the data set.
#--- these data set are not accurate enough.
#
if aval3 < 900:
continue
else:
try:
date.append(aval[0])
start = float(aval[1])
tstart.append(start)
tstop.append(float(aval[2]))
duration.append(aval3)
total_count.append(float(aval[4]))
count_per_sec.append(float(aval[5]))
pha_mean.append(float(aval[6]))
pha_median.append(float(aval[7]))
pha_sigma.append(float(aval[8]))
t_mean.append(float(aval[9]))
t_median.append(float(aval[10]))
t_sigma.append(float(aval[11]))
v_mean.append(float(aval[12]))
v_median.append(float(aval[13]))
v_sigma.append(float(aval[14]))
#
#--- changing a plotting range for an easy view
#
s_mean.append(0.001 * float(aval[15]))
s_median.append(0.001 * float(aval[16]))
s_sigma.append(0.001 * float(aval[17]))
anti_co_mean.append(float(aval[18]))
anti_co_median.append(float(aval[19]))
anti_co_sigma.append(float(aval[20]))
s2hvst.append(int(float(aval[21])))
s2hvlv.append(int(float(aval[22]) + 0.5))
scint.append(float(aval[23]))
scint_std.append(float(aval[24]))
tval = mcf.chandratime_to_fraq_year(start)
time.append(tval)
except:
continue
return [time, tstart, tstop, duration, total_count, count_per_sec, \
pha_mean, pha_median, pha_sigma, \
t_mean, t_median, t_sigma, \
v_mean, v_median, v_sigma, \
s_mean, s_median, s_sigma, \
anti_co_mean, anti_co_median, anti_co_sigma, \
s2hvst, s2hvlv, scint, scint_std]
def readData(dataname):
"""
read data and set plotting range
input: dataname --- data file name (need a full path to the file)
output: xval --- an array of independent values (time in seconds from 1998.1.1)
cval --- cumulative counts
dval --- daily counts
bval --- actual bad point counts
pval --- potential bad point counts
"""
#
#--- read data
#
data = mcf.read_data_file(dataname)
xval = []
cval = []
dval = []
bval = []
pval = []
prev = 0
for ent in data:
atemp = re.split('<>', ent)
try:
val = float(atemp[0])
if val < 0:
continue
if val == prev:
continue
stime = int(float(val))
ytime = mcf.chandratime_to_fraq_year(stime)
xval.append(ytime)
prev = ytime
val1 = float(atemp[2])
val2 = float(atemp[3])
val3 = float(atemp[4])
val4 = float(atemp[5])
cval.append(val1)
dval.append(val2)
bval.append(val3)
pval.append(val3 + val4)
except:
pass
#
#-- find plotting ranges and make a list of data lists
#
xmin_list = []
xmax_list = []
ymin_list = []
ymax_list = []
x_list = []
y_list = []
for dlist in (cval, dval, bval, pval):
(xmin, xmax, ymin, ymax) = findPlottingRange(xval, dlist)
xmin_list.append(xmin)
xmax_list.append(xmax)
ymin_list.append(ymin)
ymax_list.append(ymax)
x_list.append(xval)
y_list.append(dlist)
return [xmin_list, xmax_list, ymin_list, ymax_list, x_list, y_list]
def convert_time_format(t_list, ind=0):
"""
cnvert time format to either ydate or fractional year
input: t_list --- a list of time in seconds from 1998.1.1
ind --- if 0: ydate, else franctional year
output: <year> <a list of time in the new format>
"""
save = []
#
#--- convert to the ydate
#
if ind == 0:
t_list1 = []
t_list2 = []
for ent in t_list:
[year, ydate] = chandratime_to_yday(ent)
t_list1.append(year)
t_list2.append(ydate)
#
#--- everything happnes in the same year
#
if t_list1[0] == t_list1[-1]:
save = t_list2
byear = t_list1[0]
#
#--- when the data are over two years
#
else:
if mcf.is_leapyear(t_list1[0]):
base = 366
else:
base = 365
#
#--- checking in which year the majority of the data is in
#
t_array = numpy.array(t_list1)
indx = t_array == t_list1[0]
y_len = len(t_array[indx])
#
#--- the majority of the data is in the last year
#
if y_len > 0.5 * len(t_list1):
byear = t_list1[0]
for j in range(0, len(t_list1)):
if t_list1[j] == byear:
save.append(t_list2[j])
else:
save.append(t_list2[j] + base)
#
#--- the majority of the data is in this year
#
else:
byear = t_list[-1]
for j in range(0, len(t_list1)):
if t_list1[j] == byear:
save.append(t_list2[j])
else:
save.append(t_list2[j] - base)
return byear, save
#
#--- convert to the fractional year
#
else:
for ent in t_list:
fyear = mcf.chandratime_to_fraq_year(ent)
save.append(fyear)
return 1999, save
def run_for_msid_list(msid_list, dtype):
"""
extract data from ska database for a given msid_list
input: misd_list --- the file name of the msid_list
dtype --- data type , week, short, or long
output: updated data fits files
"""
[lim_dict, cnd_dict] = rlt.get_limit_table()
ifile = house_keeping + msid_list
data = mcf.read_data_file(ifile)
for ent in data:
if ent[0] == '#':
continue
elif ent.strip() == '':
continue
atemp = re.split('\s+', ent)
msid = atemp[0].strip()
catg = atemp[1].strip()
print("MSID: " + catg + '/' + msid)
#
#--- just in a case the data category directory does not exist
#
cmd = 'mkdir -p ' + data_dir + atemp[1]
os.system(cmd)
#
#--- set data period
#
[dfile, start, stop] = find_data_collection_period(msid, catg, dtype)
#
#--- extract new data part; saved as a local fits file
#
try:
alimit = lim_dict[msid]
cnd_msid = cnd_dict[msid]
#
#--- if the collection time is larger than a month, extract data for 30 day chunk
#
diff = stop - start
if diff > a_month:
mcnt = int(diff / a_month)
for m in range(0, mcnt):
mstart = start + a_month * m
mstop = mstart + a_month
lstart = "%4.2f" % mcf.chandratime_to_fraq_year(mstart)
lstop = "%4.2f" % mcf.chandratime_to_fraq_year(mstop)
print("Computing: " + str(lstart) + '<-->' + str(lstop))
#
#--- extract data and make a local fits file
#
out = extract_data_from_ska(msid, mstart, mstop, dtype,
alimit, cnd_msid)
#
#--- update the main fits file, either move the local file or append the new part
#
if out == True:
update_data_file(dfile, msid, dtype)
out = extract_data_from_ska(msid, mstop, stop, dtype, alimit,
cnd_msid)
if out == True:
update_data_file(dfile, msid, dtype)
#
#--- the data collection period is < 30 days
#
else:
out = extract_data_from_ska(msid, start, stop, dtype, alimit,
cnd_msid)
if out == True:
update_data_file(dfile, msid, dtype)
except:
#print(msid + ' is not in glimmon database')
print(msid + ' is not in ska fetch database')
continue
