def find_nth(fits_file = 'NA', cut= 10):
"""
find nth brightest value input: fits file/ cut = upper limit
"""
if fits_file == 'NA':
fits_file = raw_input('Fits file name: ')
cut = raw_input('Where to Cut?: ')
#
#-- make histgram
#
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmimghist infile=' + fits_file + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 =' dmlist infile=outfile.fits outfile=./zout opt=data'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
f = open('./zout', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
os.system('rm outfile.fits ./zout')
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
for ent in data:
try:
atemp = re.split('\s+|\t+', ent)
if (len(atemp) > 3) and mtac.chkNumeric(atemp[1]) and mtac.chkNumeric(atemp[2]) and (int(atemp[4]) > 0):
hbin.append(float(atemp[1]))
hcnt.append(int(atemp[4]))
except:
pass
#
#--- checking 10 th bright position
#
limit = cut -1
try:
j = 0
for i in range(len(hbin)-1, 0, -1):
if j == limit:
val = i
break
else:
if hcnt[i] > 0: #---- only when the value is larger than 0, record as count
j += 1
return hbin[val]
except:
return 'I/INDEF'
def find_nth(fits_file='NA', cut=10):
"""
find nth brightest value input: fits file/ cut = upper limit
"""
if fits_file == 'NA':
fits_file = raw_input('Fits file name: ')
cut = raw_input('Where to Cut?: ')
#
#-- make histgram
#
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmimghist infile=' + fits_file + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmlist infile=outfile.fits outfile=./zout opt=data'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
f = open('./zout', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
os.system('rm outfile.fits ./zout')
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
for ent in data:
try:
atemp = re.split('\s+|\t+', ent)
if (len(atemp) > 3) and mtac.chkNumeric(
atemp[1]) and mtac.chkNumeric(
atemp[2]) and (int(atemp[4]) > 0):
hbin.append(float(atemp[1]))
hcnt.append(int(atemp[4]))
except:
pass
#
#--- checking 10 th bright position
#
limit = cut - 1
try:
j = 0
for i in range(len(hbin) - 1, 0, -1):
if j == limit:
val = i
break
else:
if hcnt[i] > 0: #---- only when the value is larger than 0, record as count
j += 1
return hbin[val]
except:
return 'I/INDEF'
def find_10th(fits_file):
"""
find 10th brightest value input: fits file
"""
#
#-- make histgram
#
cmd = ' dmimghist infile=' + fits_file + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
os.system(cmd)
os.system('dmlist infile=outfile.fits outfile=./zout opt=data')
f = open('./zout', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
os.system('rm outfile.fits ./zout')
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
for ent in data:
try:
atemp = re.split('\s+|\t+', ent)
if (len(atemp) > 3) and mtac.chkNumeric(
atemp[1]) and mtac.chkNumeric(
atemp[2]) and (int(atemp[4]) > 0):
hbin.append(float(atemp[1]))
hcnt.append(int(atemp[4]))
except:
pass
#
#--- checking 10 th bright position
#
try:
j = 0
for i in range(len(hbin) - 1, 0, -1):
if j == 9:
val = i
break
else:
if hcnt[i] > 0: #---- only when the value is larger than 0, record as count
j += 1
return hbin[val]
except:
return 'I/INDEF'
def convert_to_ydate(time, startYear):
"""
convert the time in second to ydate
input: time --- time in seconds from 1998.1.1
startYear --- the year of the first data point
output: xdate --- a list of day of year
"""
if isLeapYear(startYear) == 1:
base = 366
else:
base = 365
xdate = []
for ent in time:
if mcf.chkNumeric(ent) and ent != 'NA':
ytime = tcnv.axTimeMTA(int(ent))
atemp = re.split(':', ytime)
year = int(float(atemp[0]))
dofy = float(atemp[1]) + float(atemp[2]) / 24.0
dofy = dofy + float(atemp[3]) / 1440.0 + float(atemp[4]) / 86400.0
if year > startYear:
dofy += base
xdate.append(dofy)
return xdate
def convert_to_fyear(cdate):
"""
convert time format to fractional year
input: cdate --- time in either <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss> or seconds from 1998.1.1
output: fyear --- time in fractional year
"""
try:
mc = re.search('T', cdate)
except:
mc = None
#
#--- for the case the time format is: <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
#
if mc is not None:
atemp = re.split('T', cdate)
btemp = re.split('-', atemp[0])
ctemp = re.split(':', atemp[1])
year = float(btemp[0])
mon = float(btemp[1])
day = float(btemp[2])
hh = float(ctemp[0])
mm = float(ctemp[1])
ss = float(ctemp[2])
ydate = mon_list[int(mon) - 1] + day
if tcnv.isLeapYear(year) == 1:
if mon > 2:
ydate += 1
#
#---- for the case the time format is seconds from 1998.1.1
#
elif mcf.chkNumeric(cdate):
out = Chandra.Time.DateTime(float(cdate)).date
atemp = re.split(':', out)
year = float(atemp[0])
ydate = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
else:
atemp = re.split(':', cdate)
year = float(atemp[0])
ydate = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
ydate = ydate + hh / 24.0 + mm / 1440.0 + ss / 86400.0
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
fyear = year + ydate / base
return fyear
def hrc_gain_run(c_input):
"""
extract hrc evt2 file, find the brightest object and create pha distribution
this is a control script
Input: c_inut --- if it is obsid, use it as a input
otherwise, a list of new candidates will be create based database
Output: <header>_pha.dat --- pha distribution data
<header>_gfit.png --- a plot of pha data
fitting_results --- a table of fitted results
"""
#
#--- if an obsid is provided, analyize that, else get new obsids from databases
#
if mcf.chkNumeric(c_input):
candidate_list = [c_input]
elif isinstance(c_input, list):
candidate_list = c_input
else:
candidate_list = get_arlac_list()
#
#--- run the main script only when candidates exist
#
if len(candidate_list) > 0:
hgfv.hrc_gain_fit_voigt(candidate_list)
hgtp.hrc_gain_trend_plot()
def hrc_gain_run(c_input):
"""
extract hrc evt2 file, find the brightest object and create pha distribution
this is a control script
Input: c_inut --- if it is obsid, use it as a input
otherwise, a list of new candidates will be create based database
Output: <header>_pha.dat --- pha distribution data
<header>_gfit.png --- a plot of pha data
fitting_results --- a table of fitted results
"""
#
#--- if an obsid is provided, analyize that, else get new obsids from databases
#
if mcf.chkNumeric(c_input):
candidate_list = [c_input]
elif isinstance(c_input, list):
candidate_list = c_input
else:
candidate_list = get_arlac_list()
#
#--- run the main script only when candidates exist
#
if len(candidate_list) > 0:
hgfv.hrc_gain_fit_voigt(candidate_list)
hgtp.hrc_gain_trend_plot()
def find_two_sigma_value(fits):
#
#-- make histgram
#
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmimghist infile=' + fits + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmlist infile=outfile.fits outfile=' + zspace + ' opt=data'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
f= open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
mcf.rm_file(zspace)
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
vsum = 0
for ent in data:
atemp = re.split('\s+|\t+', ent)
if mcf.chkNumeric(atemp[0]):
hbin.append(float(atemp[1]))
val = int(atemp[4])
hcnt.append(val)
vsum += val
#
#--- checking one sigma and two sigma counts
#
if len(hbin) > 0:
v68= int(0.68 * vsum)
v95= int(0.95 * vsum)
v99= int(0.997 * vsum)
sigma1 = -999
sigma2 = -999
sigma3 = -999
acc= 0
for i in range(0, len(hbin)):
acc += hcnt[i]
if acc > v68 and sigma1 < 0:
sigma1 = hbin[i]
elif acc > v95 and sigma2 < 0:
sigma2 = hbin[i]
elif acc > v99 and sigma3 < 0:
sigma3 = hbin[i]
break
return (sigma1, sigma2, sigma3)
else:
return(0, 0, 0)
def find_data_collection_interval():
tlist = tcnv.currentTime(format='UTC')
tyear = tlist[0]
tyday = tlist[7]
tdom = tcnv.YdateToDOM(tyear, tyday)
file = data_dir + 'Disp_dir/hist_ccd3'
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
chk = 0
k = 1
while(chk == 0):
atemp = re.split('<>', data[len(data)-k])
ldom = atemp[0]
if mcf.chkNumeric(ldom) == True:
ldom = int(ldom)
chk = 1
break
else:
k += 1
ldom += 1
return(ldom, tdom)
def check_time_format(intime):
"""
return time in Chandra time
input: intime --- time in <yyyy>:<ddd>:<hh>:<mm>:<ss> or <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss> or chandra time
output: time in chandra time (seconds from 1998.1.1)
"""
mc1 = re.search('-', intime)
mc2 = re.search(':', intime)
#
#--- it is already chandra format
#
if mcf.chkNumeric(intime):
return int(float(intime))
#
#--- time in <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
#
elif mc1 is not None:
mc2 = re.search('T', intime)
if mc2 is not None:
stime = mcf.convert_date_format(intime,
ifmt='%Y-%m-%d:%H:%M:%S',
ofmt='chandra')
else:
stime = mcf.convert_date_format(intime,
ifmt='%Y-%m-%d',
ofmt='chandra')
return stime
#
#--- time in <yyyy>:<ddd>:<hh>:<mm>:<ss>
#
elif mc2 is not None:
return Chandra.Time.DateTime(intime).secs
def find_data_collection_interval():
tlist = tcnv.currentTime(format='UTC')
tyear = tlist[0]
tyday = tlist[7]
tdom = tcnv.YdateToDOM(tyear, tyday)
file = data_dir + 'Disp_dir/hist_ccd3'
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
chk = 0
k = 1
while (chk == 0):
atemp = re.split('<>', data[len(data) - k])
ldom = atemp[0]
if mcf.chkNumeric(ldom) == True:
ldom = int(ldom)
chk = 1
break
else:
k += 1
ldom += 1
return (ldom, tdom)
def set_format_for_col(name, cdata):
"""
find a format of the input data and set column object
input: name --- column name
cdata --- column data in numpy array form
output: column object
"""
test = str(list(cdata)[0])
mc1 = re.search('True', test)
mc2 = re.search('False', test)
#
#--- check whether the value is numeric
#
if mcf.chkNumeric(test):
ft = 'E'
#
#--- check whether the value is logical
#
elif (mc1 is not None) or (mc2 is not None):
tcnt = len(list(cdata)[0])
ft = str(tcnt) + 'L'
#
#--- all others are set as character set
#
else:
tcnt = len(test)
ft = str(tcnt) + 'A'
return fits.Column(name=name, format=ft, array=cdata)
def find_two_sigma_value(fits):
#
#-- make histgram
#
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmimghist infile=' + fits + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmlist infile=outfile.fits outfile=' + zspace + ' opt=data'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
mcf.rm_file(zspace)
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
vsum = 0
for ent in data:
atemp = re.split('\s+|\t+', ent)
if mcf.chkNumeric(atemp[0]):
hbin.append(float(atemp[1]))
val = int(atemp[4])
hcnt.append(val)
vsum += val
#
#--- checking one sigma and two sigma counts
#
if len(hbin) > 0:
v68 = int(0.68 * vsum)
v95 = int(0.95 * vsum)
v99 = int(0.997 * vsum)
sigma1 = -999
sigma2 = -999
sigma3 = -999
acc = 0
for i in range(0, len(hbin)):
acc += hcnt[i]
if acc > v68 and sigma1 < 0:
sigma1 = hbin[i]
elif acc > v95 and sigma2 < 0:
sigma2 = hbin[i]
elif acc > v99 and sigma3 < 0:
sigma3 = hbin[i]
break
return (sigma1, sigma2, sigma3)
else:
return (0, 0, 0)
def check_time_format(intime):
"""
return time in Chandra time
input: intime --- time in <yyyy>:<ddd>:<hh>:<mm>:<ss> or <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss> or chandra time
output: time in chandra time (seconds from 1998.1.1)
"""
mc1 = re.search('-', intime)
mc2 = re.search(':', intime)
#
#--- it is already chandra format
#
if mcf.chkNumeric(intime):
return int(float(intime))
#
#--- time in <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
#
elif mc1 is not None:
mc2 = re.search('T', intime)
if mc2 is not None:
atemp = re.split('T', intime)
btemp = re.split('-', atemp[0])
year = int(float(btemp[0]))
mon = int(float(btemp[1]))
day = int(float(btemp[2]))
ctemp = re.split(':', atemp[1])
hrs = ctemp[0]
mins = ctemp[1]
secs = ctemp[2]
else:
btemp = re.split('-', intime)
year = int(float(btemp[0]))
mon = int(float(btemp[1]))
day = int(float(btemp[2]))
hrs = '00'
mins = '00'
secs = '00'
yday = datetime.date(year, mon, day).timetuple().tm_yday
cyday = str(yday)
if yday < 10:
cyday = '00' + cyday
elif yday < 100:
cyday = '0' + cyday
ytime = btemp[0] + ':' + cyday + ':' + hrs + ':' + mins + ':' + secs
return Chandra.Time.DateTime(ytime).secs
#
#--- time in <yyyy>:<ddd>:<hh>:<mm>:<ss>
#
elif mc2 is not None:
return Chandra.Time.DateTime(intime).secs
def generate_ephin_rate_plot(directory):
"""
create ephin rate plots
Input: directory --- a directory where the data is kept and the plot will be created
<directory>/ephin_rate --- ephin data file
Ouput: <directory>/ephin_rate.png
"""
xname = 'Time (DOM)'
yname = 'Count/Sec'
file = directory + '/ephin_rate'
chk = mcf.chkFile(file)
if chk == 0:
return ""
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
dom = []
p4 = []
e150 = []
e300 = []
e1300 = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0]) and mcf.chkNumeric(atemp[1]):
dom.append(float(atemp[0]))
p4.append(float(atemp[1]) / 300.0)
e150.append(float(atemp[2]) / 300.0)
e300.append(float(atemp[3]) / 300.0)
e1300.append(float(atemp[4]) / 300.0)
x_set_list = [dom, dom, dom, dom]
y_set_list = [p4, e150, e300, e1300]
yname_list = [yname, yname, yname, yname]
title_list = ['P4', 'E150', 'E300', 'E1300']
outname = directory + '/ephin_rate.png'
plot_multi_panel(x_set_list, y_set_list, xname, yname_list, title_list, outname)
def generate_ephin_rate_plot(directory):
"""
create ephin rate plots
Input: directory --- a directory where the data is kept and the plot will be created
<directory>/ephin_rate --- ephin data file
Ouput: <directory>/ephin_rate.png
"""
xname = 'Time (DOM)'
yname = 'Count/Sec'
file = directory + '/ephin_rate'
chk = mcf.chkFile(file)
if chk == 0:
return ""
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
dom = []
p4 = []
e150 = []
e300 = []
e1300 = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0]) and mcf.chkNumeric(atemp[1]):
dom.append(float(atemp[0]))
p4.append(float(atemp[1]) / 300.0)
e150.append(float(atemp[2]) / 300.0)
e300.append(float(atemp[3]) / 300.0)
e1300.append(float(atemp[4]) / 300.0)
x_set_list = [dom, dom, dom, dom]
y_set_list = [p4, e150, e300, e1300]
yname_list = [yname, yname, yname, yname]
title_list = ['P4', 'E150', 'E300', 'E1300']
outname = directory + '/ephin_rate.png'
plot_multi_panel(x_set_list, y_set_list, xname, yname_list, title_list,
outname)
def clean_the_input(line):
"""
check the input is numeric and if so, round to two decimal
input: line --- input quantity
output: line --- if it is a numeric value, a value of two decimal,
otherwise, just return the value as it was
"""
if mcf.chkNumeric(line):
line = str(ecf.round_up(float(line)))
return line
def clean_the_input(line):
"""
check the input is numeric and if so, round to two decimal
input: line --- input quantity
output: line --- if it is a numeric value, a value of two decimal,
otherwise, just return the value as it was
"""
if mcf.chkNumeric(line):
line = str(ecf.round_up(float(line)))
return line
def quickChandra_time(ent):
"""
axTime3 replacement
input: ent --- either seconds from 1998.1.1 or date in <yyyy>:<ddd>:<hh>:<mm>:<ss>
output: out --- either seconds from 1998.1.1 or date in <yyyy>:<ddd>:<hh>:<mm>:<ss>
"""
if mcf.chkNumeric(ent):
out = Chandra.Time.DateTime(float(ent)).date
else:
out = Chandra.Time.DateTime(str(ent)).secs
return out
def checkValue(self, param, value):
"""
for a given parameter and numeric value, return whether the value is in the range
this also include 'NULL' value
"""
vrange = self.showRange(
param
) #--- a value or the list of possible values for the parameter
v_list = re.split('\,', vrange)
mchk = 0 #--- whether the value is in the vrange. if so 1. if 2, skip the secondary test.
if vrange == 'MUST':
if ocf.null_value(value) != True:
mchk = 1
ipos = 0
elif vrange == 'OPEN' or vrange == 'NA':
mchk = 1
ipos = 0
if ocf.null_value(value):
mchk = 2
elif ocf.null_value(value): #--- value is "NULL" case
if ocf.find_pattern('NULL', vrange):
mchk = 1
ipos = 0
elif mcf.chkNumeric(value): #--- value is a numeric case
for ipos in range(0, len(v_list)):
if v_list[ipos] == 'OPEN':
mchk = 1
break
elif ocf.find_pattern('<>', v_list[ipos]):
cond = re.split('<>', v_list[ipos])
vtest = float(value)
lower = float(cond[0])
upper = float(cond[1])
if vtest >= lower and vtest <= upper:
mchk = 1
break
else: #--- value is a word case
for ipos in range(0, len(v_list)):
if value == v_list[ipos]:
mchk = 1
break
return [mchk, ipos]
def extract_hist_data(file):
'''
extracting acis hist data from fits file
input: fits file name
output: one cloumn histgram data
'''
tdata = Table.read(file, hdu=1)
cols = tdata.columns
sdata = tdata['COUNTS']
hist_data = []
for ent in sdata:
if mcf.chkNumeric(ent):
hist_data.append(float(ent))
return hist_data
def check_time_format(intime):
mc1 = re.search('-', intime)
mc2 = re.search(':', intime)
if mcf.chkNumeric(intime):
return int(float(intime))
elif mc1 is not None:
mc2 = re.search('T', intime)
if mc2 is not None:
atemp = re.split('T', intime)
btemp = re.split('-', atemp[0])
year = int(float(btemp[0]))
mon = int(float(btemp[1]))
day = int(float(btemp[2]))
ctemp = re.split(':', atemp[1])
hrs = ctemp[0]
mins = ctemp[1]
secs = ctemp[2]
else:
btemp = re.split('-', intime)
year = int(float(btemp[0]))
mon = int(float(btemp[1]))
day = int(float(btemp[2]))
hrs = '00'
mins = '00'
secs = '00'
yday = datetime.date(year, mon, day).timetuple().tm_yday
cyday = str(yday)
if yday < 10:
cyday = '00' + cyday
elif yday < 100:
cyday = '0' + cyday
ytime = btemp[0] + ':' + cyday + ':' + hrs + ':' + mins + ':' + secs
return Chandra.Time.DateTime(ytime).secs
elif mc2 is not None:
return Chandra.Time.DateTime(intime).secs
def read_data(file):
"""
read data from a given file
Input: file --- input file name
Output: date_list --- a list of date
ede_list --- a list of ede value
error_list --- a list of computed ede error
"""
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
date_list = []
ede_list = []
error_list = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0])== False:
continue
fwhm = float(atemp[2])
ferr = float(atemp[3])
ede = float(atemp[4])
dom = float(atemp[10])
(year, ydate) = tcnv.DOMtoYdate(dom)
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
fyear = year + ydate/base
date_list.append(fyear)
ede_list.append(ede)
#
#--- the error of EdE is computed using FWHM and its error value
#
error = math.sqrt(ede*ede* ((ferr*ferr) / (fwhm*fwhm)))
error_list.append(error)
return [date_list, ede_list, error_list]
def obsid_from_mp_report(nlist):
"""
extract obsid from a given path to the data
entry line looks line: "/data/mta/www/mp_reports/photons/acis/cti/62311_0"
Input: nlist --- a list of data pathes to the data
Ouput: obsid_list --- a list of obsids
"""
obsid_list = []
for ent in nlist:
atemp = re.split('/', ent)
btemp = re.split('_', atemp[len(atemp) - 1])
#
#--- check whether the obsid is really number
#
if mcf.chkNumeric(btemp[0]):
obsid_list.append(btemp[0])
return obsid_list
def read_data(file):
"""
read data from a given file
Input: file --- input file name
Output: date_list --- a list of date
ede_list --- a list of ede value
error_list --- a list of computed ede error
"""
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
date_list = []
date_list2 = []
ede_list = []
error_list = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0])== False:
continue
fwhm = float(atemp[2])
ferr = float(atemp[3])
ede = float(atemp[4])
date = atemp[5]
sdate = float(atemp[6])
fyear = change_time_format_fyear(date)
date_list.append(fyear)
date_list2.append(sdate)
ede_list.append(ede)
#
#--- the error of EdE is computed using FWHM and its error value
#
error = math.sqrt(ede*ede* ((ferr*ferr) / (fwhm*fwhm)))
error_list.append(error)
return [date_list, date_list2, ede_list, error_list]
def get_quad_cti(ent, pos = 0):
"""
get cti parts of error parts from input lines
Input: ent --- line contains cti of quad0, 1, 2, 3 values in the positin 1, 2, 3, 4
pos --- if 0, cti value, if 1, error value
Output: list --- a list of 4 values of either quad cti or their error
"""
list = []
atemp = re.split(ent)
for i in range(1,5):
btemp = re.split('+-', atemp[i])
if mcf.chkNumeric(btemp[pos]):
val = float(btemp[pos])
else:
val = -99999;
list.append(val)
return list
def read_data(file):
"""
read data from a given file
Input: file --- input file name
Output: date_list --- a list of date
ede_list --- a list of ede value
error_list --- a list of computed ede error
"""
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
date_list = []
ede_list = []
error_list = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0]) == False:
continue
fwhm = float(atemp[2])
ferr = float(atemp[3])
ede = float(atemp[4])
dom = float(atemp[10])
(year, ydate) = tcnv.DOMtoYdate(dom)
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
fyear = year + ydate / base
date_list.append(fyear)
ede_list.append(ede)
#
#--- the error of EdE is computed using FWHM and its error value
#
error = math.sqrt(ede * ede * ((ferr * ferr) / (fwhm * fwhm)))
error_list.append(error)
return [date_list, ede_list, error_list]
def read_xmm_orbit():
"""
read xmm orbital elements and return list of distance to XMM
input: none but read from /data/mta4/proj/rac/ops/ephem/TLE/xmm.spctrk
output: [time, alt] time in seconds from 1998.1.1
alt in km from the center of the Earth
"""
#
#--- read xmm orbital elements
#
cxofile = "/data/mta4/proj/rac/ops/ephem/TLE/xmm.spctrk"
f = open(cxofile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
atime = []
alt = []
chk = 0
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0]) == False:
continue
#
#--- compute the distance to xmm from the center of the earth
#
try:
x = float(atemp[6])
except:
continue
y = float(atemp[7])
z = float(atemp[8])
dist = math.sqrt(x * x + y * y + z * z)
alt.append(dist)
#
#--- convert time to seconds from 1998.1.1
#
atime.append(float(atemp[0])- tcorrect)
return[atime, alt]
def read_data(infile, remove=0):
try:
f = open(infile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
if remove == 1:
mcf.rm_file(infile)
out = []
for ent in data:
if mcf.chkNumeric(ent):
val = int(float(ent))
else:
val = ent
out.append(val)
return out
except:
return []
def extract_hist_data(file):
'''
extracting acis hist data from fits file
input: fits file name
output: one cloumn histgram data
'''
#
#--- check whether the temp file exists. if so, remove it
#
cfile = exc_dir + 'zout'
chk = mtac.chkFile(cfile)
if chk > 0:
cmd = 'rm ' + exc_dir + 'zout'
os.system(cmd)
#
#--- extract data
#
cmd = 'dmlist "' + file + '[cols counts]" outfile = ' + exc_dir + 'zout opt=data'
os.system(cmd)
file = exc_dir + 'zout'
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + exc_dir + 'zout'
os.system(cmd)
hist_data = []
for ent in data:
atemp = re.split('\s+|\t+', ent)
if mtac.chkNumeric(atemp[0]):
hist_data.append(float(atemp[1]))
return hist_data
def extract_ephin_data(file, out_dir, comp_test=''):
"""
extract ephine data from a given data file name and save it in out_dir
Input: file --- ephin data file name
out_dir --- directory which the data is saved
Output: <out_dir>/ephin_data --- ephin data (300 sec accumulation)
"""
#
#--- extract time and ccd id information from the given file
#
data = pyfits.getdata(file, 1)
time_r = data.field("TIME")
scp4_r = data.field("SCP4")
sce150_r = data.field("SCE150")
sce300_r = data.field("SCE300")
sce1500_r = data.field("SCE1300")
#
#--- initialize
#
diff = 0
chk = 0
ephin_data = []
#
#--- sdata[0]: scp4, sdata[1]: sce150, sdata[2]: sce300, and sdata[3]: sce1300
#
sdata = [0 for x in range(0,4)]
#
#--- check each line and count the numbers of ccd in the each 300 sec intervals
#
for k in range(0, len(time_r)):
if mcf.chkNumeric(time_r[k]):
ftime = float(time_r[k])
if ftime > 0:
if chk == 0:
# for j in range(2, 6):
# sdata[j-2] += atemp[j]
if mcf.chkNumeric(scp4_r[k]) and mcf.chkNumeric(sce150_r[k]) \
and mcf.chkNumeric(sce300_r[k]) and mcf.chkNumeric(sce1500_r[k]):
sdata[0] += float(scp4_r[k])
sdata[1] += float(sce150_r[k])
sdata[2] += float(sce300_r[k])
sdata[3] += float(sce1500_r[k])
s_time = ftime
diff = 0
chk = 1
elif diff >= 300.0:
#
#--- convert time in dom
#
dom = tcnv.stimeToDom(s_time)
#
#--- print out counts per 300 sec
#
line = str(dom) + '\t'
for j in range(0, 4):
line = line + str(sdata[j]) + '\t'
sdata[j] = 0
line = line + '\n'
ephin_data.append(line)
chk = 0
#
#--- re initialize for the next round
#
if mcf.chkNumeric(scp4_r[k]) and mcf.chkNumeric(sce150_r[k]) \
and mcf.chkNumeric(sce300_r[k]) and mcf.chkNumeric(sce1500_r[k]):
sdata[0] += float(scp4_r[k])
sdata[1] += float(sce150_r[k])
sdata[2] += float(sce300_r[k])
sdata[3] += float(sce1500_r[k])
s_time = ftime
diff = 0
#
#--- accumurate the count until the 300 sec interval is reached
#
else:
diff = ftime - s_time
if mcf.chkNumeric(scp4_r[k]) and mcf.chkNumeric(sce150_r[k]) \
and mcf.chkNumeric(sce300_r[k]) and mcf.chkNumeric(sce1500_r[k]):
sdata[0] += float(scp4_r[k])
sdata[1] += float(sce150_r[k])
sdata[2] += float(sce300_r[k])
sdata[3] += float(sce1500_r[k])
#
#--- for the case the last interval is less than 300 sec,
#--- estimate the the numbers of hit and adjust
#
if diff > 0 and diff < 300:
line = str(dom) + '\t'
ratio = 300.0 / diff
for j in range(0, 4):
var = sdata[j] * ratio
line = line + str(var) + '\t'
line = line + '\n'
ephin_data.append(line)
#
#--- if this is a test, reutrn the result
#
if comp_test == 'test':
return ephin_data
#
#--- otherwise, print the result
#
else:
file = out_dir + '/ephin_rate'
f = open(file, 'a')
for ent in ephin_data:
f.write(ent)
f.close()
def extract_sim_data():
"""
extract sim data from PRIMARYCCDM_*.*.tl
input: none but read from <dumpdir>/PRIMARYCCDM_*.*.tl
output: <outdir>sim_data.out
"""
#
#--- find the time of the last entry from the sim_data.out
#
sfile = outdir + 'sim_data.out'
f = open(sfile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- cleaning up the data; drop the data which the date starts from ":" e.g. :2014
#
pdata = []
for ent in data:
if re.search('^:', ent):
continue
else:
pdata.append(ent)
#
#--- the last entiry values
#
if len(pdata) > 0:
atemp = re.split('\s+', pdata[len(pdata)-1])
ltime = tcnv.axTimeMTA(atemp[0]) #--- converting time to sec from 1998.1.1
time_2 = atemp[0]
col1_2 = atemp[1]
col2_2 = atemp[2]
col3_2 = atemp[3]
else:
ltime = 0
time_2 = 0
col1_2 = ''
col2_2 = ''
col3_2 = ''
#
#--- check whether input files exists
#
cmd = 'ls -rt ' + dumpdir + 'PRIMARYCCDM_*.*.tl >' + zspace
os.system(cmd)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + zspace
os.system(cmd)
dlen = len(data)
if dlen < 1:
exit(1)
#
#--- files exist. read the data from the last 10 files
#
tlist = data[dlen-40:]
for ent in tlist:
cmd = 'cat ' +ent + ' >> ' + zspace
os.system(cmd)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
cmd = 'rm ' + zspace
os.system(cmd)
prev = ''
fo = open('./temp_save', 'w')
#
#--- go though each data line
#
for ent in data:
try:
#
#--- expect the first letter of the data line is numeric (e.g. 2014).
#
val = float(ent[0])
except:
continue
#
#--- only data with "FMT" format will be used
#
mc = re.search('FMT', ent)
if mc is None:
continue
atemp = re.split('\t+', ent)
#
#--- if there are less than 20 entries, something wrong; skip it
#
if len(atemp) < 20:
continue
#
#--- convert time format
#
time = atemp[0]
time = time.strip();
time = time.replace(' ', ':')
time = time.replace(':::', ':00')
time = time.replace('::', ':0')
#
#--- if the time is exactly same as one before, skip it
#
if time == time_2:
continue
#
#--- if the time is already in the database, keip it
#
stime = tcnv.axTimeMTA(time)
if stime <= ltime:
continue
#
#--- use only data which tscpos and fapos have numeric values
#
tscpos = atemp[4].strip()
fapos = atemp[5].strip()
if tscpos == "" or fapos == "":
continue
else:
tscpos = int(float(tscpos))
fapos = int(float(fapos))
# aopc = atemp[11].strip()
# if aopc == '':
# aopc = '0'
mpwm = atemp[12].strip()
if mcf.chkNumeric(mpwm):
mpwm = int(float(mpwm))
mpwm = str(mpwm)
else:
mpwm = '0'
#
#--- we want to print only beginning and ending of the same data entries.
#--- skip the line if all three entiries are same as one before, except the last one
#
if col1_2 == tscpos and col2_2 == fapos and col3_2 == mpwm:
time_2 = time
continue
line = time + '\t' + str(tscpos) + '\t' + str(fapos) + '\t' + mpwm + '\n'
if line == prev:
continue
else:
pline = time_2 + '\t' + str(col1_2) + '\t' + str(col2_2) + '\t' + str(col3_2) + '\n'
fo.write(pline)
fo.write(line)
prev = line
time_2 = time
col1_2 = tscpos
col2_2 = fapos
col3_2 = mpwm
fo.close()
sfile2 = sfile + '~'
cmd = 'cp ' + sfile + ' ' + sfile2
os.system(cmd)
cmd = 'cat ./temp_save >> ' + sfile
os.system(cmd)
def find_excess_file(lev = 'Lev2'):
"""
find data with extremely high radiation and remove it.
this is done mainly in Lev2 and copied the procesure in Lev2
input: lev --- level. default Lev2 (other option is Lev1)
output: excess radiation data fits files in ./lres/Save/.
"""
if lev == 'Lev2':
lres = s_dir + lev + '/Outdir/lres/'
cmd = 'ls ' + lres + 'mtaf*fits > ' + zspace
os.system(cmd)
data = scf.read_file(zspace, remove=1)
cmd = 'mkdir ' + lres + 'Save'
os.system(cmd)
for ent in data:
cmd = 'dmlist ' + ent + ' opt=data > ' + zspace
scf.run_ascds(cmd)
out = scf.read_file(zspace, remove=1)
ssoft = 0.0
soft = 0.0
med = 0.0
hard = 0.0
harder = 0.0
hardest = 0.0
tot = 0
for val in out:
atemp = re.split('\s+', val)
if mcf.chkNumeric(atemp[0]):
ssoft += float(atemp[6])
soft += float(atemp[7])
med += float(atemp[8])
hard += float(atemp[9])
harder += float(atemp[10])
hardest += float(atemp[11])
tot += 1
else:
continue
if tot > 1:
ssoft /= tot
soft /= tot
med /= tot
hard /= tot
harder /= tot
hardest /= tot
mc = re.search('acis6', ent)
chk = 0
if mc is not None:
if (med > 200):
chk = 1
else:
if (soft > 500) or (med > 150):
chk = 1
if chk > 0:
cmd = 'mv ' + ent + ' ' + lres + 'Save/.'
os.system(cmd)
else:
#
#--- for Lev1, we move the files which removed in Lev2. we assume that we already
#--- run Lev2 on this function
#
epath = s_dir + '/Lev2/Outdir/lres/Save/'
if os.listdir(epath) != []:
cmd = 'ls ' + s_dir + '/Lev2/Outdir/lres/Save/*fits > ' + zspace
os.system(cmd)
data = scf.read_file(zspace, remove=1)
l1_lres = s_dir + '/Lev1/Outdir/lres/'
l1_dir = l1_lres + '/Save/'
cmd = 'mkdir ' + l1_dir
os.system(cmd)
for ent in data:
atemp = re.split('mtaf', ent)
btemp = re.split('N', atemp[1])
mc = re.search('_', btemp[0])
if mc is not None:
ctemp = re.split('_', btemp[0])
obsid = ctemp[0]
else:
obsid = btemp[0]
atemp = re.split('acis', ent)
btemp = re.split('lres', atemp[1])
ccd = btemp[0]
cid = 'acis' + str(ccd) + 'lres_sibkg.fits'
cmd = 'mv ' + l1_lres + 'mtaf' + obsid + '*' + cid + ' ' + l1_dir + '/.'
os.system(cmd)
def get_lines(grating):
"""
extract line statistics for a given grating
input: grating --- hetg, metg, or letg
output: acis_<grating>_<line>_data
"""
#
#--- read data file ehader
#
infile = house_keeping + 'data_header'
f = open(infile, 'r')
header = f.read()
f.close()
#
#--- set which grating data to extract
#
if grating == 'hetg':
cmd = 'ls ' + gdata_dir + '/*/*/obsid_*_L1.5_S1HEGp1_linelist.txt >' + zspace
ofile = 'acis_hetg_'
l_list = h_lines
elif grating == 'metg':
cmd = 'ls ' + gdata_dir + '/*/*/obsid_*_L1.5_S1MEGp1_linelist.txt >' + zspace
ofile = 'acis_metg_'
l_list = m_lines
else:
cmd = 'ls ' + gdata_dir + '/*/*/obsid_*_L1.5_S1LEGp1_linelist.txt >' + zspace
ofile = 'hrc_letg_'
l_list = l_lines
os.system(cmd)
d_list = read_data_file(zspace, remove=1)
sdate_list = [[], [], [], [], [], [], []]
line_list = [{}, {}, {}, {}, {}, {}, {}]
lcnt = len(l_list)
#
#---- go though each files
#
for dfile in d_list:
out = find_info(dfile)
if out == 'na':
continue
else:
[obsid, ltime, stime] = out
#
#--- extract line information. if energy or fwhm are either "*" or "NaN", skip
#
data = read_data_file(dfile)
for ent in data:
atemp = re.split('\s+', ent.strip())
if mcf.chkNumeric(atemp[0]):
energy = atemp[2]
fwhm = atemp[3]
if energy == 'NaN':
continue
if (fwhm == '*') or (fwhm == 'NaN'):
continue
energy = adjust_digit(energy, 6)
peak = float(energy)
err = atemp[4]
ede = atemp[5]
line = str(
obsid
) + '\t' + energy + '\t' + fwhm + '\t' + err + '\t' + ede + '\t'
line = line + str(ltime) + '\t' + str(int(stime)) + '\n'
#
#--- find the line value within +/-5 of the expected line center position
#
for k in range(0, lcnt):
center = l_list[k]
low = (center - 5) / 1000.0
top = (center + 5) / 1000.0
if (peak >= low) and (peak <= top):
sdate_list[k].append(stime)
line_list[k][stime] = line
#
#--- output file name
#
for k in range(0, lcnt):
val = str(l_list[k])
if len(val) < 4:
val = '0' + val
odata = data_dir + ofile + val + '_data'
fo = open(odata, 'w')
fo.write(header)
fo.write('\n')
#
#--- print out the data
#
slist = sdate_list[k]
slist.sort()
for sdate in slist:
ent = line_list[k][sdate]
fo.write(ent)
fo.close()
def update_data(msid, l_list, dset = 1, time=[], vals=[]):
"""
update data for the given msid
input: msid --- msid
l_list --- a list of list of [<start time>, <stop time>, <yellow min>, <yellow max>, <red min>, <red max>]
dset --- indicate which data set we are handling. 1: main, 2: secondary, i, s, off: hrc sub category
time --- a list of time entry for secondary case
vals --- a list of msid values for secondary case
output: data for msid updated (<data_dir>/<msid>_data
"""
#
#--- set which data periods we need to process the data; start from the one after the last one
#
periods = set_data_periods(msid)
tc = 0
for tlist in periods:
start = tlist[0]
stop = tlist[1]
#
#--- occasionally the data period accross the date of limit changes.
#--- if that happens, devide the period into before and after the date
#--- lperiods has atmost two entries, but usually only one
#
lperiods = set_limits_for_period(l_list, start, stop)
if dset != 1: #--- setting for mta comp case only
kcnt = 0
dcnt = len(time)
for limits in lperiods:
tstart = limits[0]
tstop = limits[1]
y_min = limits[2]
y_max = limits[3]
r_min = limits[4]
r_max = limits[5]
#
#--- main trend case: need to extract data
#
if dset == 1:
sdata = get_data(msid, tstart, tstop)
#
#--- secondary trend case: data are passed from the calling function
#
else:
sdata = []
for m in range(0, dcnt):
if (time[m] >= tstart) and (time[m] < tstop):
sdata.append(vals[m])
if time[m] >=tstop:
kcnt = m -1
break
#
#--- if the data is too small, just skip the period
#
if len(sdata) < 5:
continue
#
#--- compute statistics
#
stat_results = compute_stats(sdata, y_min, y_max, r_min, r_max)
#
#--- special treatment for mta comp hrc values: it has i, s, or off suffix
#
if mcf.chkNumeric(dset):
oname = msid
else:
oname = msid + '_' + dset
#
#--- append data to the data file
#
print_results(oname, start, stop, stat_results, y_min, y_max, r_min, r_max)
btemp = re.split('\)', atemp[1])
ent = re.split('\,', btemp[0])
temp = tcnv.DOMtoYdate(float(ent[0]))
start_list.append(temp[1])
temp = tcnv.DOMtoYdate(float(ent[1]))
stop_list.append(temp[1])
rad_zone = [start_list, stop_list]
return rad_zone
#---------------------------------------------------------------------------------------------------
if __name__ == '__main__':
#
#--- a period name (e.g. 2061206) is given, the script computes only for that period
#--- otherwise, it will recompute the entire period
#
if len(sys.argv) == 2:
if mcf.chkNumeric(sys.argv[1]):
period = sys.argv[1]
else:
period = 'all'
else:
period = 'all'
read_xmm_and_process(period)
def print_html_page(comp_test, in_year=1, in_mon=1):
"""
driving function to print all html pages for ACIS Dose Plots
Input: comp_test --- test indicator. if it is "test", it will run the test version
in_year/in_mon --- if in_year and in_mon are given, the file is created for
that year/month, otherwise, the files are created in the current year/month
Output: html pages in <web_dir> and <web_dir>/<mon_dir_name> (e.g. JAN2013)
"""
#
#--- find today's date and convert them appropriately
#
if comp_test == 'test':
bchk = 0
tday = 13;
umon = 2;
uyear = 2013;
cmon = tcnv.changeMonthFormat(umon)
cmon = cmon.upper()
ldate = str(uyear) + '-' + str(umon) + '-' + str(tday) #-- update date
else:
#
#--- find today's date
#
[uyear, umon, tday, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
#
#--- change month in digit into letters
#
cmon = tcnv.changeMonthFormat(umon)
cmon = cmon.upper()
ldate = str(uyear) + '-' + str(umon) + '-' + str(tday) #-- update date
#
#--- if year and month is given, create for that month. otherwise, create for this month
#
bchk = 0
if mcf.chkNumeric(in_year) and mcf.chkNumeric(in_mon):
if in_year > 1900 and (in_mon >0 and in_mon < 13):
bchk = 1
if bchk > 0:
uyear = in_year
umon = in_mon
cmon = tcnv.changeMonthFormat(umon)
cmon = cmon.upper()
mon_dir_name = cmon + str(uyear);
#
#--- check whether this monnth web page already opens or not
#
dname = web_dir + mon_dir_name
chk = mcf.chkFile(dname)
if chk > 0:
if bchk == 0:
#
#-- create only when it is working for the current month
#
print_main_html(ldate, uyear, umon);
print_month_html(mon_dir_name, ldate, uyear, umon);
print_png_html(mon_dir_name, ldate, uyear, umon);
#
#--- change permission level and the owner of the files
#
cmd = 'chgrp mtagroup ' + web_dir + '/* ' + web_dir + '/*/*'
os.system(cmd)
cmd = 'chmod 755 '+ web_dir + '/* ' + web_dir + '/*/*'
os.system(cmd)
def extract_data(file, out_dir, comp_test =''):
"""
extract time and ccd_id from the fits file and create count rate data
Input: file --- fits file data
out_dir --- the directory in which data will be saved
Output: ccd<ccd>--- 5 min accumulated count rate data file
"""
#
#--- extract time and ccd id information from the given file
#
data = pyfits.getdata(file, 0)
time_col = data.field('TIME')
ccdid_col = data.field('CCD_ID')
#
#--- initialize
#
diff = 0
chk = 0
ccd_c = [0 for x in range(0, 10)]
ccd_h = [[] for x in range(0, 10)]
#
#--- check each line and count the numbers of ccd in the each 300 sec intervals
#
for k in range(0, len(time_col)):
if mcf.chkNumeric(time_col[k]) and mcf.chkNumeric(ccdid_col[k]) :
ftime = float(time_col[k])
if ftime > 0:
ccd_id = int(ccdid_col[k])
if chk == 0:
ccd_c[ccd_id] += 1
s_time = ftime
diff = 0
chk = 1
elif diff >= 300.0:
#
#--- convert time in dom
#
dom = tcnv.stimeToDom(s_time)
#
#--- print out counts per 300 sec
#
for i in range(0, 10):
line = str(dom) + '\t' + str(ccd_c[i]) + '\n'
ccd_h[i].append(line)
#
#--- re initialize for the next round
#
ccd_c[i] = 0
ccd_c[ccd_id] += 1
s_time = ftime
diff = 0
chk = 0
#
#--- accumurate the count until the 300 sec interval is reached
#
else:
diff = ftime - s_time
ccd_c[ccd_id] += 1
#
#--- for the case the last interval is less than 300 sec,
#--- estimate the the numbers of hit and adjust
#
if diff > 0 and diff < 300:
ratio = 300.0 / diff
for i in range(0, 10):
ccd_c[i] *= ratio
line = str(dom) + '\t' + str(ccd_c[i]) + '\n'
ccd_h[i].append(line)
#
#--- if this is a test, return output
#
if comp_test == 'test':
return ccd_h
#
#--- otherwise, print out the results
#
else:
for i in range(0, 10):
file = out_dir + '/ccd' + str(i)
f = open(file, 'a')
for ent in ccd_h[i]:
f.write(ent)
f.close()
def generate_count_rate_plot(directory):
"""
create count rate plots
Input: directory --- the directory where data is located and the plot will be created
<directory>/ccd<ccd> --- count rate data file
Output: <directory>/acis_dose_ccd<ccd>.png
<directory>/acis_dose_ccd_5_7.png
"""
xname = 'Time (DOM)'
yname = 'Count/Sec'
data1_x = []
data1_y = []
data2_x = []
data2_y = []
data3_x = []
data3_y = []
#
#--- plot count rates for each ccd
#
for ccd in range(0, 10):
file = directory + '/ccd' + str(ccd)
chk = mcf.chkFile(file)
if chk == 0:
continue
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
xdata = []
ydata = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0]) and mcf.chkNumeric(atemp[1]):
xdata.append(float(atemp[0]))
#
#--- normalized to cnts/sec
#
ydata.append(float(atemp[1]) / 300.0)
title = 'ACIS Count Rate: CCD' + str(ccd)
outname = directory + '/acis_dose_ccd' + str(ccd) + '.png'
plot_panel(xdata, ydata, xname, yname, title, outname)
#
#--- save data for three panel plot
#
if ccd == 5:
data1_x = xdata
data1_y = ydata
elif ccd == 6:
data2_x = xdata
data2_y = ydata
elif ccd == 7:
data3_x = xdata
data3_y = ydata
#
#--- create three panel plot for ccd5, ccd6, and ccd7
#
title1 = 'ACIS Count Rate: CCD5'
title2 = 'ACIS Count Rate: CCD6'
title3 = 'ACIS Count Rate: CCD7'
outname = directory + '/acis_dose_ccd_5_7.png'
x_set_list = [data1_x, data2_x, data3_x]
y_set_list = [data1_y, data2_y, data3_y]
yname_list = [yname, yname, yname]
title_list = [title1, title2, title3]
plot_multi_panel(x_set_list, y_set_list, xname, yname_list, title_list, outname)
def convert_avg(input_list):
"""
compute avg and std of each column entry in gradkodak for the given data set
input: input_list --- a list of lists. each sub-list contains dataseeker output for "_avg"
each entry is a dictionary which contains avg, std, min, max
output: a list of avg and std of each columns of gradkodak. first len(gradkodak)
are avg of the columns and the next len(gradkodak) are std.
"""
#
#--- open the list of lists
#
(_4rt575t_avg, _4rt700t_avg, _4rt701t_avg, _4rt702t_avg, _4rt703t_avg, _4rt704t_avg,\
_4rt705t_avg, _4rt706t_avg, _4rt707t_avg, _4rt708t_avg, _4rt709t_avg, _4rt710t_avg,\
_4rt711t_avg, \
ohrthr02_avg, ohrthr03_avg, ohrthr04_avg, ohrthr05_avg, ohrthr06_avg, ohrthr07_avg, \
ohrthr08_avg, ohrthr09_avg, ohrthr10_avg, ohrthr11_avg, ohrthr12_avg, ohrthr13_avg, \
ohrthr14_avg, ohrthr15_avg, ohrthr17_avg, ohrthr21_avg, ohrthr22_avg, ohrthr23_avg, \
ohrthr24_avg, ohrthr25_avg, ohrthr26_avg, ohrthr27_avg, ohrthr28_avg, ohrthr29_avg, \
ohrthr30_avg, ohrthr31_avg, ohrthr33_avg, ohrthr34_avg, ohrthr35_avg, ohrthr36_avg, \
ohrthr37_avg, ohrthr39_avg, ohrthr40_avg, ohrthr42_avg, ohrthr44_avg, ohrthr45_avg, \
ohrthr46_avg, ohrthr47_avg, ohrthr49_avg, ohrthr50_avg, ohrthr51_avg, ohrthr52_avg, \
ohrthr53_avg, ohrthr54_avg, ohrthr55_avg, ohrthr56_avg, ohrthr57_avg, ohrthr58_avg, \
ohrthr60_avg, ohrthr61_avg, \
oobthr02_avg, oobthr03_avg, oobthr04_avg, oobthr05_avg, oobthr06_avg, oobthr07_avg, \
oobthr08_avg, oobthr09_avg, oobthr10_avg, oobthr11_avg, oobthr12_avg, oobthr13_avg, \
oobthr14_avg, oobthr15_avg, oobthr17_avg, oobthr18_avg, oobthr19_avg, oobthr20_avg, \
oobthr21_avg, oobthr22_avg, oobthr23_avg, oobthr24_avg, oobthr25_avg, oobthr26_avg, \
oobthr27_avg, oobthr28_avg, oobthr29_avg, oobthr30_avg, oobthr31_avg, oobthr33_avg, \
oobthr34_avg, oobthr35_avg, oobthr36_avg, oobthr37_avg, oobthr38_avg, oobthr39_avg, \
oobthr40_avg, oobthr41_avg, oobthr42_avg, oobthr43_avg, oobthr44_avg, oobthr45_avg, \
oobthr48_avg, oobthr49_avg, oobthr50_avg, oobthr51_avg, oobthr52_avg, oobthr53_avg, \
oobthr54_avg, oobthr55_avg, oobthr56_avg, oobthr57_avg, oobthr58_avg, oobthr59_avg, \
oobthr60_avg, oobthr61_avg, oobthr62_avg, oobthr63_avg \
) = input_list
#
#--- get avg and std of the enter entries of the given list
#
hrmarange_list = range(2, 14) + range(21, 27) + [28, 29, 30, 33, 36, 37
] + range(44, 49) + range(
49, 54) + [55, 56]
[hrmaavg, hrmadev] = get_avg_std('ohrthr', hrmarange_list, input_list)
#------------
hrmacs_list = range(6, 16) + [17, 25, 26, 29, 30, 31] + range(
33, 38) + [39, 40] + range(50, 59) + [60, 61]
[hrmacavg, hrmacdev] = get_avg_std('ohrthr', hrmacs_list, input_list)
#-----------
hrmaxgrad_list1 = [10, 11, 34, 35, 55, 56]
[hrmaxgrd1, xxx] = get_avg_std('ohrthr', hrmaxgrad_list1, input_list)
hrmaxgrad_list2 = [12, 13, 35, 37, 57, 58]
[hrmaxgrd2, xxx] = get_avg_std('ohrthr', hrmaxgrad_list2, input_list)
hrmaxgrd = hrmaxgrd1 - hrmaxgrd2
#
#--- compute std separately for the case two outputs are add/subtracted after computed
#
dev_list = hrmaxgrad_list1 + hrmaxgrad_list2
[z, hrmaxgrd_dev] = get_avg_std('ohrthr', dev_list, input_list)
#-----------
hrmarad1grd_list = [8, 31, 33, 52]
[hrmarad1grd, xxx] = get_avg_std('ohrthr', hrmarad1grd_list, input_list)
hrmarad2grd_list = [9, 53, 54]
[hrmarad2grd, xxx] = get_avg_std('ohrthr', hrmarad2grd_list, input_list)
hrmaradgrd = hrmarad1grd - hrmarad2grd
dev_list = hrmarad1grd_list + hrmarad2grd_list
[z, hrmaradgrd_dev] = get_avg_std('ohrthr', dev_list, input_list)
#-----------
obaavg_list = range(8, 33) + range(33, 42) + [44, 45]
[obaavg, obadev] = get_avg_std('oobthr', obaavg_list, input_list)
#-----------
obacone_list = range(8, 16) + range(17, 31) + range(57, 62)
[obaconeavg, obacone_dev] = get_avg_std('oobthr', obacone_list, input_list)
#-----------
#
#--- for the case two different header entries are needed;
#
fwblkhd_list1 = [62, 63]
fwblkhd_list2 = [700, 712]
[fwblkhdt, fwblkhdt_dev] = get_avg_std('oobthr',
fwblkhd_list1,
input_list,
'_4rt',
fwblkhd_list2,
tail='t')
aftblkhdt_list = [31, 33, 34]
[aftblkhdt, aftblkhdt_dev] = get_avg_std('oobthr', aftblkhdt_list,
input_list)
obaaxgrd = fwblkhdt - aftblkhdt
s_list = fwblkhd_list1 + aftblkhdt_list
[z, obaaxgrd_dev] = get_avg_std('oobthr',
s_list,
input_list,
'_4rt',
fwblkhd_list2,
tail='t')
#-----------
mzoba_list1 = [8, 19, 25, 31, 57, 60]
mzoba_list2 = [575]
[mzobacone, mzobacone_dev] = get_avg_std('oobthr',
mzoba_list1,
input_list,
'_4rt',
mzoba_list2,
tail='t')
pzoba_list = [13, 22, 23, 28, 29, 61]
[pzobacone, pzobacone_dev] = get_avg_std('oobthr', pzoba_list, input_list)
obadiagrad = mzobacone - pzobacone
d_list = mzoba_list1 + pzoba_list
[z, obadiagrad_dev] = get_avg_std('oobthr',
d_list,
input_list,
'_4rt',
mzoba_list2,
tail='t')
#-----------
#
#--- compute the range of the data
#
clist = range(2, 14) + range(21, 28) + [29, 30, 33, 36, 37, 42] + range(
45, 54) + [55, 56]
[hrmarange, hrmarange_dev] = get_range('ohrthr', clist, input_list)
clist = range(2, 8)
[hrmastrutrnge, hrmastrutrnge_dev] = get_range('oobthr', clist, input_list)
clist = range(42, 45)
[tfterange, tfterange_dev] = get_range('oobthr', clist, input_list)
clist = range(49, 55)
[scstrutrnge, scstrutrnge_dev] = get_range('oobthr', clist, input_list)
out = []
for val in [hrmaavg, hrmacavg, hrmaxgrd, hrmaradgrd, obaavg, obaconeavg, obaaxgrd, obadiagrad, fwblkhdt, \
aftblkhdt, mzobacone, pzobacone, hrmarange, tfterange, hrmastrutrnge, scstrutrnge, \
hrmadev, hrmacdev, hrmaxgrd_dev, hrmaradgrd_dev, obadev, obacone_dev, obaaxgrd_dev, obadiagrad_dev, fwblkhdt_dev, \
aftblkhdt_dev, mzobacone_dev, pzobacone_dev, hrmarange_dev, tfterange_dev, hrmastrutrnge_dev, scstrutrnge_dev]:
if mcf.chkNumeric(val) == False or str(val) == 'nan':
val = -99.0
out.append(val)
return out
atemp = re.split('\(', line)
btemp = re.split('\)', atemp[1])
ent = re.split('\,', btemp[0])
temp = tcnv.DOMtoYdate(float(ent[0]))
start_list.append(temp[1])
temp = tcnv.DOMtoYdate(float(ent[1]))
stop_list.append(temp[1])
rad_zone = [start_list, stop_list]
return rad_zone
#---------------------------------------------------------------------------------------------------
if __name__ == '__main__':
#
#--- a period name (e.g. 2061206) is given, the script computes only for that period
#--- otherwise, it will recompute the entire period
#
if len(sys.argv) == 2:
if mcf.chkNumeric(sys.argv[1]):
period = sys.argv[1]
else:
period = 'all'
else:
period = 'all'
read_xmm_and_process(period)
def update_mta_comp_database():
"""
updata database of mta computed msids
input: none but read from /data/mta4/Deriv/*fits files
outpu: updated data file: <data_dir>/<msid>_data
"""
#
#--- get a list of data fits file names
#
infile = house_keeping + 'mta_comp_fits_files'
data = ecf.read_file_data(infile)
for fits in data:
#
#--- hrc has 4 different cases (all data, hrc i, hrc s, and off). tail contain which one this one is
#--- if this is not hrc (or hrc all), tail = 2
#
mc = re.search('hrc', fits)
if mc is not None:
atemp = re.split('_', fits)
btemp = re.split('.fits', atemp[1])
tail = btemp[0]
else:
tail = 2
[cols, tbdata] = ecf.read_fits_file(fits)
time = []
for ent in tbdata.field('time'):
stime = float(ent)
#
#--- check whether the time is in dom
#
if stime < 31536000:
stime = ecf.dom_to_stime(float(ent))
time.append(stime)
for col in cols:
col = col.lower()
#
#--- we need only *_avg columns
#
mc = re.search('_avg', col)
if mc is not None:
vals = tbdata.field(col)
ctime = []
cvals = []
for m in range(0, len(time)):
#
#--- skip the data value "nan" and dummy values (-999, -998, -99, 99, 998, 999)
#
if str(vals[m]) in ['nan', 'NaN', 'NAN']:
continue
nval = float(vals[m])
if nval in [-999, -998, -99, 99, 998, 999]:
continue
else:
ctime.append(time[m])
cvals.append(nval)
atemp = re.split('_', col)
msid = atemp[-2]
if mcf.chkNumeric(tail):
oname = msid
else:
oname = msid + '_' + tail
print "MSID: " + str(oname)
cmd = 'rm ' + data_dir + oname + '_data'
os.system(cmd)
#
#--- read limit table for the msid
#
l_list = ecf.set_limit_list(msid)
if len(l_list) == 0:
try:
l_list = mta_db[msid]
except:
l_list = []
update_data(msid, l_list, dset = tail, time=ctime, vals=cvals)
def dateFormatCon(elm, *arg):
"convert various date format into a tuple of (year, month, day, hours, minutes, second, ydate) "
if mtac.chkNumeric(elm): #--- for the case, inputs are digits, e.g, year, month, ...
year = elm
cnt = len(arg)
if cnt == 1:
ydate = arg[0]
#
#--- check whether ydate is a fractional ydate or not, if so, compute hours, minutes, and second
#
diff = arg[0] - int(ydate)
if diff == 0:
hours = 0
minutes = 0
seconds = 0
else:
temp = 24 * diff
hours = int(temp)
temp2 = 60 * (temp - hours)
minutes = int(temp2)
temp3 = 60 * (temp2 - minutes)
seconds = int(temp3)
(month, day) = changeYdateToMonDate(year, int(ydate))
elif cnt == 2:
month = arg[0]
day = int(arg[1])
ydate = findYearDate(year, month, day)
elif cnt == 4:
ydate = arg[0]
hours = arg[1]
minutes = arg[2]
seconds = arg[3]
(month, day) = changeYdateToMonDate(year, int(ydate))
elif cnt == 5:
month = arg[0]
day = arg[1]
hours = arg[2]
minutes = arg[3]
seconds = arg[4]
ydate = findYearDate(year, month, day)
else:
ydate = 1
month = 1
day = 1
hours = 0
minutes = 0
seconds = 0
else:
atemp = re.split('\s+', elm)
m = re.search('\/', elm)
m2= re.search('\,', elm)
n = re.search('T', elm)
if len(atemp) == 6: #--- for the case, e.g: Wed Apr 4 10:34:32 EDT 2012
year = int(atemp[5])
month = changeMonthFormat(atemp[1])
day = int(atemp[2])
btemp = re.split(':', atemp[3])
hours = int(btemp[0])
minutes = int(btemp[1])
seconds = int(btemp[2])
yday = float(day) + float(hours/24.0) + float(minutes/1440.0) + float(seconds/86400)
ydate = findYearDate(year, month, yday)
elif (m is not None) and (m2 is not None): #--- for the case, e.g. 03/28/12,00:00:00
atemp = re.split('\,', elm)
btemp = re.split('\/', atemp[0])
year = int(btemp[2])
if year > 90 and year < 1900:
year += 1900
elif year < 90:
year += 2000
month = int(btemp[0])
day = int(btemp[1])
btemp = re.split(':', atemp[1])
hours = int(btemp[0])
minutes = int(btemp[1])
seconds = int(btemp[2])
yday = float(day) + float(hours/24.0) + float(minutes/1440.0) + float(seconds/86400)
ydate = findYearDate(year, month, yday)
elif n is not None: #--- for the case, e.g. 03/28/12T00:00:00
atemp = re.split('T', elm)
m1 = re.search('\/', elm)
m2 = re.search('-', elm)
if m1 is not None:
btemp = re.split('\/', atemp[0])
year = int(btemp[2])
if year > 90 and year < 1900:
year += 1900
elif year < 90:
year += 2000
month = int(btemp[0])
day = int(btemp[1])
elif m2 is not None:
btemp = re.split('-', atemp[0])
year = int(btemp[0])
month = int(btemp[1])
day = int(btemp[2])
btemp = re.split(':', atemp[1])
hours = int(btemp[0])
minutes = int(btemp[1])
seconds = int(btemp[2])
yday = float(day) + float(hours/24.0) + float(minutes/1440.0) + float(seconds/86400)
ydate = findYearDate(year, month, yday)
else:
atemp = re.split(':', elm)
if len(atemp) == 6: #--- for the case yyy:mm:dd:hh:mm:ss
year = int(atemp[0])
month = int(atemp[1])
day = int(atemp[2])
hours = int(atemp[3])
minutes = int(atemp[4])
seconds = int(atemp[5])
yday = float(day) + float(hours/24.0) + float(minutes/1440.0) + float(seconds/86400)
ydate = findYearDate(year, month, yday)
else: #--- for the cae yyyy:yday:hh:mm:ss
year = int(atemp[0])
ydate = int(atemp[1])
hours = int(atemp[2])
minutes = int(atemp[3])
seconds = int(atemp[4])
[month, day] = changeYdateToMonDate(year, ydate)
yday = float(day) + float(hours/24.0) + float(minutes/1440.0) + float(seconds/86400)
ydate = findYearDate(year, month, yday)
line = (year, month, day, hours, minutes, seconds, ydate)
return line
def collect_potential_obs():
"""
read mta mail archive and find potential ddt/too triggered email
input: none but read from /stage/mail/mta
output: obsids --- a list of obsids
prpnum --- a list of proposla numbers
seqnum --- a list of sequence numbers
otypes --- a list of otype
"""
#
#--- a list of obsids currently in the lists
#
obs_list = create_obsids_on_the_list()
#
#--- read email archive
#
ifile = m_dir + 'mta_mail'
data = read_data_file(ifile)
chk = 0
kdate = ''
obsids = []
prpnum = []
seqnum = []
otype = []
for ent in data:
mc = re.search('Date:', ent)
if mc is not None:
kdate = ent
if chk == 0:
mc1 = re.search('Recently Approved ', ent)
if mc1 is not None:
mc = re.search('Subject', ent)
if mc is not None:
mc = re.search('Re:', ent)
if mc is None:
chk = 1
else:
continue
else:
mc2 = re.search('From:', ent)
if mc2 is not None:
chk = 0
continue
mc3 = re.search('Obsid ', ent)
mc4 = re.search('Obsids ', ent)
if (mc3 is not None) or (mc4 is not None):
#
#--- only when the observation is triggered in the past two hours, notify the obsid
#
# if check_time_limit(kdate, lhr=2) == 0:
# continue
atemp = re.split('\s+', ent)
for wrd in atemp:
mc5 = re.search('\(', wrd)
if mc5 is not None:
break
else:
wrd = wrd.replace('\,', '')
wrd.strip()
if mcf.chkNumeric(wrd):
val = int(float(wrd))
#
#--- check whether the obsid is already in too_list or ddt_list
#
if val in obs_list:
continue
#
#--- check repeater
#
if not (val in obsids):
obsids.append(val)
#
#--- get proposal # and seq #
#
atemp = re.split('#: ' , ent)
btemp = re.split(',', atemp[1])
prpnum.append(btemp[0].strip())
btemp = re.split('\)', atemp[2])
seqnum.append(btemp[0].strip())
mc = re.search('DDT', ent)
if mc is not None:
otype.append('ddt')
else:
oype.append('too')
chk = 0
#
#--- if there is a new obs, notify
#
if len(obsids) > 0:
line = 'The following obsid is activated via email:\n\n'
for obsid in obsids:
line = line + 'Obsid: ' + str(obsid) + '\n'
fo = open(zspace, 'w')
fo.write(line)
fo.close()
cmd = 'cat ' + zspace + '| mailx -s "Subject:TEST!! TEST !! New TOO/DDT in Email" [email protected]'
os.system(cmd)
mcf.rm_file(zspace)
return [obsids, prpnum, seqnum, otype]
def dateFormatCon(elm, *arg):
"convert various date format into a tuple of (year, month, day, hours, minutes, second, ydate) "
if mtac.chkNumeric(
elm): #--- for the case, inputs are digits, e.g, year, month, ...
year = elm
cnt = len(arg)
if cnt == 1:
ydate = arg[0]
#
#--- check whether ydate is a fractional ydate or not, if so, compute hours, minutes, and second
#
diff = arg[0] - int(ydate)
if diff == 0:
hours = 0
minutes = 0
seconds = 0
else:
temp = 24 * diff
hours = int(temp)
temp2 = 60 * (temp - hours)
minutes = int(temp2)
temp3 = 60 * (temp2 - minutes)
seconds = int(temp3)
(month, day) = changeYdateToMonDate(year, int(ydate))
elif cnt == 2:
month = arg[0]
day = int(arg[1])
ydate = findYearDate(year, month, day)
elif cnt == 4:
ydate = arg[0]
hours = arg[1]
minutes = arg[2]
seconds = arg[3]
(month, day) = changeYdateToMonDate(year, int(ydate))
elif cnt == 5:
month = arg[0]
day = arg[1]
hours = arg[2]
minutes = arg[3]
seconds = arg[4]
ydate = findYearDate(year, month, day)
else:
ydate = 1
month = 1
day = 1
hours = 0
minutes = 0
seconds = 0
else:
atemp = re.split('\s+', elm)
m = re.search('\/', elm)
m2 = re.search('\,', elm)
n = re.search('T', elm)
if len(atemp
) == 6: #--- for the case, e.g: Wed Apr 4 10:34:32 EDT 2012
year = int(atemp[5])
month = changeMonthFormat(atemp[1])
day = int(atemp[2])
btemp = re.split(':', atemp[3])
hours = int(btemp[0])
minutes = int(btemp[1])
seconds = int(btemp[2])
yday = float(day) + float(hours / 24.0) + float(
minutes / 1440.0) + float(seconds / 86400)
ydate = findYearDate(year, month, yday)
elif (m is not None) and (
m2 is not None): #--- for the case, e.g. 03/28/12,00:00:00
atemp = re.split('\,', elm)
btemp = re.split('\/', atemp[0])
year = int(btemp[2])
if year > 90 and year < 1900:
year += 1900
elif year < 90:
year += 2000
month = int(btemp[0])
day = int(btemp[1])
btemp = re.split(':', atemp[1])
hours = int(btemp[0])
minutes = int(btemp[1])
seconds = int(btemp[2])
yday = float(day) + float(hours / 24.0) + float(
minutes / 1440.0) + float(seconds / 86400)
ydate = findYearDate(year, month, yday)
elif n is not None: #--- for the case, e.g. 03/28/12T00:00:00
atemp = re.split('T', elm)
m1 = re.search('\/', elm)
m2 = re.search('-', elm)
if m1 is not None:
btemp = re.split('\/', atemp[0])
year = int(btemp[2])
if year > 90 and year < 1900:
year += 1900
elif year < 90:
year += 2000
month = int(btemp[0])
day = int(btemp[1])
elif m2 is not None:
btemp = re.split('-', atemp[0])
year = int(btemp[0])
month = int(btemp[1])
day = int(btemp[2])
btemp = re.split(':', atemp[1])
hours = int(btemp[0])
minutes = int(btemp[1])
seconds = int(btemp[2])
yday = float(day) + float(hours / 24.0) + float(
minutes / 1440.0) + float(seconds / 86400)
ydate = findYearDate(year, month, yday)
else:
atemp = re.split(':', elm)
if len(atemp) == 6: #--- for the case yyy:mm:dd:hh:mm:ss
year = int(atemp[0])
month = int(atemp[1])
day = int(atemp[2])
hours = int(atemp[3])
minutes = int(atemp[4])
seconds = int(atemp[5])
yday = float(day) + float(hours / 24.0) + float(
minutes / 1440.0) + float(seconds / 86400)
ydate = findYearDate(year, month, yday)
else: #--- for the cae yyyy:yday:hh:mm:ss
year = int(atemp[0])
ydate = int(atemp[1])
hours = int(atemp[2])
minutes = int(atemp[3])
seconds = int(atemp[4])
[month, day] = changeYdateToMonDate(year, ydate)
yday = float(day) + float(hours / 24.0) + float(
minutes / 1440.0) + float(seconds / 86400)
ydate = findYearDate(year, month, yday)
line = (year, month, day, hours, minutes, seconds, ydate)
return line
def find_coordinate(target, cyear=''):
"""
find coordinates of the target
input: target --- target name
cyear --- the year of the observed. it can be in year date.
if it is not given, no corrections for the propoer motion
output: coordinate [ra, dec] in decimal format.
"""
#
#--- set several initial values
#
target = target.strip()
target = target.replace(' ', '\ ')
tra = 'na'
tdec = 'na'
pra = 0
pdec = 0
schk = 0
#
#--- call simbad to get the coordinate data from simbad site
#
cmd = 'lynx -source http://simbad.u-strasbg.fr/simbad/sim-id\?output.format=ASCII\&Ident=' + target + '>' + zspace
os.system(cmd)
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
mcf.rm_file(zspace)
#
#--- read the coordinates and the proper motion
#
tchk = 0
pchk = 0
for ent in data:
mc1 = re.search('Coordinates', ent)
mc2 = re.search('Proper', ent)
if tchk == 0 and mc1 is not None:
try:
atemp = re.split(':', ent)
btemp = re.split('\s+', atemp[1])
ahr = float(btemp[1])
amin = float(btemp[2])
asec = float(btemp[3])
tra = 15.0 * (ahr + amin / 60.0 + asec / 3600.0)
deg = float(btemp[4])
dmin = float(btemp[5])
dsec = float(btemp[6])
sign = 1
if deg < 0:
sign = -1
tdec = abs(deg) + dmin / 60.0 + dsec / 3600.0
tdec *= sign
tchk += 1
except:
schk = 1
break
if pchk == 0 and mc2 is not None:
try:
atemp = re.split(':', ent)
btemp = re.split('\s+', atemp[1])
pra = btemp[1]
pdec = btemp[2]
if mcf.chkNumeric(pra):
pra = float(pra) / 3600.0 / 1000.0
else:
pra = 0.0
if mcf.chkNumeric(pdec):
pdec = float(pdec) / 3600.0 / 1000.0
else:
pdec = 0.0
pchk += 1
except:
pass
if tchk == 1 and pchk == 1:
break
#
#--- if the year is given, correct for the proper motion
#
if schk == 0 and mcf.chkNumeric(cyear):
dyear = float(cyear) - 2000.0
try:
tra += dyear * pra
tdec += dyear * pdec
except:
pass
return [tra, tdec]
def hrc_gain_fit_gaus(c_input):
"""
extract hrc evt2 file, find the brightest object and create pha distribution
Input: c_inut --- if it is obsid, use it as a input
otherwise, a list of new candidates will be create based database
Output: <header>_pha.dat --- pha distribution data
<header>_gfit.png --- a plot of pha data
fitting_results --- a table of fitted results
"""
#
#--- if an obsid is provided, analyize that, else get new obsids from databases
#
if mcf.chkNumeric(c_input):
candidate_list = [c_input]
else:
candidate_list = arlist.hrc_gain_find_ar_lac()
if len(candidate_list) > 0:
for obsid in candidate_list:
file = extract_hrc_evt2(obsid)
if file == 'na':
continue
#
#--- get a file name header for the later use
#
temp = re.split('N', file)
hname = temp[0]
#
#--- extract information from the fits file header
#
[
obsid, detnam, date_obs, date_end, tstart, tstop, ra_pnt,
dec_pnt, ra_nom, dec_nom, roll_pnt, foc_len, defocus, sim_x,
sim_y, sim_z
] = find_header_info(file)
#
#--- find the diffrence between real AR Lac position and nominal postion so that we can determin how much area we should include
#
ra_diff = abs(ra - ra_nom) * 60.0
dec_diff = abs(dec - dec_nom) * 60.0
rad_diff = math.sqrt(ra_diff * ra_diff + dec_diff * dec_diff)
if rad_diff < 10.0:
fit_rad = 60.0
else:
fit_rad = 200.0
#
#--- find a location of the brightest object (assume it is AR Lac) in sky coordinates
#
[x, y] = find_center(file)
#
#--- extract pha values in the given area
#
pha = extract_pha(file, x, y, fit_rad)
#
#--- create pha count distribution
#
pmax = max(pha) + 1
pha_bin = [x for x in range(0, pmax)]
pha_hist = [0 for x in range(0, pmax)]
for ent in pha:
pha_hist[ent] += 1
#
#--- print out the distirbution results
#
outfile = data_dir + hname + '_pha.dat'
fo = open(outfile, 'w')
for i in range(0, pmax):
line = str(pha_bin[i]) + '\t' + str(pha_hist[i]) + '\n'
fo.write(line)
fo.close()
#
#--- find median point
#
med = find_med(pha_hist)
#
#--- fit a normal distribution on the data
#
[amp, center, width] = fit_gauss(pha_bin, pha_hist)
#
#--- print out the fitting result
#
outfile = house_keeping + 'fitting_results'
copied_file = outfile + '~'
cmd = 'cp ' + outfile + ' ' + copied_file
os.system(cmd)
fo = open(outfile, 'a')
line = str(obsid) + '\t' + date_obs + '\t' + str(
tstart) + '\t' + detnam + '\t' + str(ra_pnt) + '\t' + str(
dec_pnt) + '\t\t'
line = line + str(round(ra_diff, 3)) + '\t' + str(
round(dec_diff, 3)) + '\t' + str(round(
rad_diff, 3)) + '\t' + str(med) + '\t\t'
line = line + str(round(center, 3)) + '\t' + str(round(
amp, 3)) + '\t' + str(round(width, 3)) + '\t'
line = line + str(roll_pnt) + '\t' + str(foc_len) + '\t' + str(
defocus) + '\t'
line = line + str(sim_x) + '\t' + str(sim_y) + '\t' + str(
sim_z) + '\n'
fo.write(line)
fo.close()
#
#--- plot the data
#
outfile = plot_dir + hname + '_gfit.png'
plot_gauss(pha_bin, pha_hist, amp, center, width, file, outfile)
#
#--- remove the evt2 file
#
mcf.rm_file(file)
def exclude_sources(fits):
"""
remove the area around the main source and all point sources from data
input: fits --- input fits file name
output: out_name --- source removed fits file (<header>_ccd<ccd>_cleaned.fits)
"""
#
#--- read which ccds are used and several other info from fits header
#
cmd = ' dmlist ' + fits + ' opt=head > ' + zspace
scf.run_ascds(cmd)
data = scf.read_file(zspace, remove=1)
ccd_list = []
for ent in data:
mc = re.search('bias file used', ent)
if mc is not None:
atemp = re.split('CCD', ent)
val = atemp[1].strip()
ccd_list.append(val)
continue
for name in ['SIM_X', 'SIM_Y', 'SIM_Z', 'RA_NOM', 'DEC_NOM', 'ROLL_NOM', 'RA_TARG', 'DEC_TARG']:
mc = re.search(name, ent)
if mc is not None:
lname = name.lower()
atemp = re.split('\s+', ent)
val = atemp[2].strip()
exec "%s = %s" % (lname, val)
break
#
#--- sort ccd list
#
ccd_list.sort()
#
#--- guess a source center position on the sky coordinates from the information extracted from the header
#
cmd = ' dmcoords none none opt=cel '
cmd = cmd + ' ra=' + str(ra_targ) + ' dec=' + str(dec_targ )
cmd = cmd + ' sim="' + str(sim_x) + ' ' + str(sim_y) + ' ' + str(sim_z) + '" '
cmd = cmd + ' detector=acis celfmt=deg '
cmd = cmd + ' ra_nom=' + str(ra_nom) + ' dec_nom=' + str(dec_nom) + ' roll_nom=' + str(roll_nom) + ' '
cmd = cmd + ' ra_asp=")ra_nom" dec_asp=")dec_nom" verbose=1 >' + zspace
scf.run_ascds(cmd)
data = scf.read_file(zspace, remove=1)
for ent in data:
mc = re.search('SKY', ent)
if mc is not None:
atemp = re.split('\s+', ent)
skyx = atemp[1]
skyy = atemp[2]
break
#
#-- keep the record of the source position for the later use (e.g. used for evt1 processing);
#
o_fits = fits.replace('.gz', '')
coord_file = o_fits.replace('.fits', '_source_coord')
ofile = './Reg_files/' + coord_file
line = str(skyx) + ':' + str(skyy) + '\n'
fo = open(ofile, 'w')
fo.write(line)
fo.close()
#
#-- remove the 200 pix radius area around the source
#
cmd = ' dmcopy "' + fits + '[exclude sky=circle(' + skyx + ',' + skyy + ',200)]" '
cmd = cmd + ' outfile=source_removed.fits clobber="yes"'
scf.run_ascds(cmd)
#
#--- get a file size: will be used to measure the size of removed area later.
#--- assumption here is the x-ray hit ccd evenly, but of course it is not,
#--- but this is the best guess we canget
#
size = {}
for ccd in ccd_list:
cmd = ' dmcopy "' + fits + '[ccd_id=' + str(ccd) + ']" outfile=test.fits clobber=yes'
scf.run_ascds(cmd)
cmd = 'ls -l test.fits > ' + zspace
os.system(cmd)
data = scf.read_file(zspace, remove=1)
for line in data:
atemp = re.split('\s+', line)
if mcf.chkNumeric(atemp[4]):
size[ccd] = int(float(atemp[4]))
else:
size[ccd] = int(float(atemp[3]))
mcf.rm_file('test.fits')
#
#--- now separate observations to indivisual ccds
#
file_list = []
for ccd in ccd_list:
tail = '_ccd' + str(ccd) + '.fits'
out = o_fits.replace('.fits', tail)
file_list.append(out)
cmd = ' dmcopy "source_removed.fits[ccd_id=' + ccd + ']" outfile= ' + out + ' clobber=yes'
scf.run_ascds(cmd)
mcf.rm_file('source_removed.fits')
#
#--- process each ccd
#
for pfits in file_list:
reg_file = pfits.replace('.fits', '_block_src.reg')
#
#--- find point sources
#
cmd = ' celldetect infile=' + pfits
cmd = cmd + ' outfile=acisi_block_src.fits regfile=acisi_block_src.reg clobber=yes'
scf.run_ascds(cmd)
data = scf.read_file('acisi_block_src.reg')
exclude = []
for ent in data:
atemp = re.split('\,', ent)
#
#--- increase the area covered around the sources 3time to make sure leaks from a bright source is minimized
#
val2 = float(atemp[2]) * 3
val3 = float(atemp[3]) * 3
line = atemp[0] + ',' + atemp[1] + ',' + str(val2) + ',' + str(val3) +',' + atemp[4]
exclude.append(line)
out_name = pfits.replace('.gz','')
out_name = out_name.replace('.fits', '_cleaned.fits')
#
#--- if we actually found point sources, remove them from the ccds
#
e_cnt = len(exclude)
if e_cnt > 0:
cnt = 0
chk = 0
round = 0
line = ''
while cnt < e_cnt:
#
#--- remove 6 sources at a time so that it won't tax memory too much
#
for i in range(cnt, cnt + 6):
if i >= e_cnt:
chk += 1
break
if line == '':
line = exclude[i]
else:
line = line + '+' + exclude[i]
cnt += 6
if round == 0:
cmd = ' dmcopy "' + pfits + '[exclude sky=' + line +']" outfile=out.fits clobber="yes"'
scf.run_ascds(cmd)
round += 1
else:
cmd = 'mv out.fits temp.fits'
os.system(cmd)
cmd = ' dmcopy "temp.fits[exclude sky=' + line +']" outfile=out.fits clobber="yes"'
scf.run_ascds(cmd)
round += 1
if chk > 0:
break
else:
line = ''
mcf.rm_file('temp.fits')
cmd = 'mv out.fits ' + out_name
os.system(cmd)
else:
cmd = 'cp ' + pfits + ' ' + out_name
os.system(cmd)
#
#--- find the size of cleaned up file size
#
cmd = 'ls -l ' + out_name + '>' + zspace
os.system(cmd)
data = scf.read_file(zspace, remove=1)
for line in data:
atemp = re.split('\s+', line)
if mcf.chkNumeric(atemp[4]):
asize = float(atemp[4])
else:
asize = float(atempp[3])
for pccd in range(0, 10):
check = 'ccd' + str(pccd)
mc = re.search(check, out_name)
if mc is not None:
break
#
#--- compute the ratio of the cleaned to the original file; 1 - ratio is the potion that we removed
#--- from the original data
#
ratio = asize / float(size[str(pccd)])
#
#--- record the ratio for later use
#
fo = open('./Reg_files/ratio_table', 'a')
line = reg_file + ': ' + str(ratio) + '\n'
fo.write(line)
fo.close()
cmd = 'mv acisi_block_src.reg ./Reg_files/' + reg_file
os.system(cmd)
mcf.rm_file('acisi_block_src.fits')
def full_range_plot():
"""
create long term trending plots
Input: none but all data are read from web_dir/<MON><YEAR> directories
Output: <web_dir>/long_term_plot.png
<web_dir>/month_avg_img.png
<web_dir>/month_avg_spc.png
<web_dir>/month_avg_bi.png
"""
ccd5_x = []
ccd5_y = []
ccd6_x = []
ccd6_y = []
ccd7_x = []
ccd7_y = []
for ccd in range(0, 10):
xname = 'mccd'+ str(ccd) + '_x'
yname = 'mccd'+ str(ccd) + '_y'
sname = 'mccd'+ str(ccd) + '_s'
exec "%s = []" % (xname)
exec "%s = []" % (yname)
exec "%s = []" % (sname)
cmd = 'ls ' + web_dir + '/*/ccd' + str(ccd) + ' >' + zspace
os.system(cmd)
try:
f = open(zspace, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
mcf.rm_file(zspace)
except:
continue
for ent in data:
f = open(ent, 'r')
fdata = [line.strip() for line in f.readlines()]
f.close()
time = []
sum = 0
sum2 = 0
cnt = 0
for line in fdata:
atemp = re.split('\s+', line)
if mcf.chkNumeric(atemp[0]) and mcf.chkNumeric(atemp[1]):
xt = float(atemp[0])
yt = float(atemp[1])
if xt >= 0 and xt < 20000 and yt >= 0:
sum += yt
sum2+= yt * yt
cnt += 1
time.append(xt)
#
#--- for ccd 5, 6, 7, we need 5 min average data (then normalized to cnt /sec)
#
if ccd == 5:
ccd5_x.append(xt)
ccd5_y.append(yt/300.0)
if ccd == 6:
ccd6_x.append(xt)
ccd6_y.append(yt/300.0)
if ccd == 7:
ccd7_x.append(xt)
ccd7_y.append(yt /300.0)
#
#--- get monthly average
#
date = int(0.5 * (min(time) + max(time)))
avg = sum / float(cnt) / 300.0
sig = sqrt(sum2 / float(cnt) / 90000.0 - avg * avg)
xname = 'mccd'+ str(ccd) + '_x'
yname = 'mccd'+ str(ccd) + '_y'
sname = 'mccd'+ str(ccd) + '_s'
exec "%s.append(%s)" % (xname , date)
exec "%s.append(%s)" % (yname , avg)
exec "%s.append(%s)" % (sname , sig)
#
#--- ploting starts here
#
xname = 'Time (DOM)'
yname = 'Counts/Sec'
#
#---- ccd 7 full history
#
title = 'ACIS Count Rate: CCD 7'
outname = web_dir + '/acis_ccd7_dose_plot.png'
plot_panel(ccd7_x, ccd7_y, xname, yname, title, outname, autox='yes')
#
#--- long term plot (for ccd 5, 6, and 7)
#
x_set_list = [ccd5_x, ccd6_x, ccd7_x]
y_set_list = [ccd5_y, ccd6_y, ccd7_y]
yname_list = [yname, yname, yname, yname]
title_list = ['CCD5', 'CCD6', 'CCD7']
outname = web_dir + '/long_term_plot.png'
y_limit = [1000, 1000, 1000]
plot_multi_panel(x_set_list, y_set_list, xname, yname_list, title_list, outname, ylim =2, y_limit=y_limit, autox='yes')
#
#--- imaging ccds full history
#
x_set_list = [mccd0_x, mccd1_x, mccd2_x, mccd3_x]
y_set_list = [mccd0_y, mccd1_y, mccd2_y, mccd3_y]
yname_list = [yname, yname, yname, yname]
title_list = ['CCD0', 'CCD1', 'CCD2', 'CCD3']
outname = web_dir + '/month_avg_img.png'
plot_multi_panel(x_set_list, y_set_list, xname, yname_list, title_list, outname, linew=0, mrk='+', ylim=1, autox='yes')
#
#--- spectral ccds full history
#
x_set_list = [mccd4_x, mccd6_x, mccd8_x, mccd9_x]
y_set_list = [mccd4_y, mccd6_y, mccd8_y, mccd9_y]
yname_list = [yname, yname, yname, yname]
title_list = ['CCD4', 'CCD6', 'CCD8', 'CCD9']
outname = web_dir + '/month_avg_spc.png'
plot_multi_panel(x_set_list, y_set_list, xname, yname_list, title_list, outname, linew=0, mrk='+', ylim=1, autox='yes')
#
#--- backside ccds full history
#
x_set_list = [mccd5_x, mccd7_x]
y_set_list = [mccd7_y, mccd7_y]
yname_list = [yname, yname, yname, yname]
title_list = ['CCD5', 'CCD7']
outname = web_dir + '/month_avg_bi.png'
plot_multi_panel(x_set_list, y_set_list, xname, yname_list, title_list, outname, linew=0, mrk='+', ylim=1, autox='yes')
plot_multi_panel(x_set_list, y_set_list, xname, yname_list, title_list, outname, autox='yes')
#
#--- write out monthly average data
#
for ccd in range(0, 10):
name = web_dir + 'monthly_avg_data_ccd' + str(ccd) + '.dat'
f = open(name, 'w')
exec 'xdat = mccd%s_x' % (ccd)
exec 'ydat = mccd%s_y' % (ccd)
exec 'ysig = mccd%s_s' % (ccd)
xdat =numpy.array(xdat)
ydat =numpy.array(ydat)
ysig =numpy.array(ysig)
sorted_index = numpy.argsort(xdat)
xsorted = xdat[sorted_index]
ysorted = ydat[sorted_index]
ssorted = ysig[sorted_index]
for i in range(0, len(xsorted)):
yrnded = '%.3f' % ysorted[i]
ysrnded = '%.3f' % ssorted[i]
line = str(xsorted[i]) + '\t' + yrnded + '+/-' + ysrnded + '\n'
f.write(line)
f.close()
def read_unit_list():
"""
read unit list and make into a dictionary form
input: none but read from <house_keeping>/unit_list
output: udict --- a dictionary of <msid> <---> <unit>
"""
#
#--- read the main unit file and description of msid
#
ulist = house_keeping + 'unit_list'
data = read_file_data(ulist)
udict = {}
ddict = read_description_from_mta_list()
for ent in data:
atemp = re.split('\s+', ent)
try:
udict[atemp[0]] = atemp[1]
except:
pass
#
#--- read dataseeker unit list and replace if they are not same
#
ulist = house_keeping + 'dataseeker_entry_list'
data = read_file_data(ulist)
for ent in data:
if ent[0] == '#':
continue
atemp = re.split('\t+', ent)
if len(atemp) < 3:
continue
msid =atemp[0].lower()
if mcf.chkNumeric(atemp[2]) == False:
if atemp[2] != '':
udict[msid] = atemp[2]
else:
try:
test = udict[msid]
except:
udict[msid] = ''
ddict[msid] = atemp[-1]
#
#--- farther read supplemental lists
#
ulist = house_keeping + 'unit_supple'
data = read_file_data(ulist)
for ent in data:
atemp = re.split('\s+', ent)
udict[atemp[0]] = atemp[1]
dlist = house_keeping + 'description_supple'
data = read_file_data(dlist)
for ent in data:
atemp = re.split('\:\:', ent)
msid = atemp[0].strip()
descr = atemp[1].strip()
ddict[msid] = descr
return [udict, ddict]
def create_history_file(head):
"""
create count history file and the information file containing current bad entry information
Input: head --- ccd, hccd, or col to indicate which data to handle
Output: <head>_ccd<ccd>_cnt --- count history data:<dom><><year:ydate><><cumlative cnt><><cnt for the day>
<head>_ccd<ccd>_information --- current information of the bad entries. For example, list warm pixels,
flickering pixels, totally new pixels, and all past and current warm pixels.
"""
for ccd in range(0, 10):
#
#--- read data file head is either ccd, hccd, or col
#
file = data_dir + 'Disp_dir/hist_' + head + str(ccd)
data = mcf.readFile(file)
bad_dat_list = [] #--- save all bad data as elements
bad_dat_save = [] #--- save all bad data as a list for each day
dom = []
ydate = []
dcnt = [] #--- keep discreate count history
ccnt = [] #--- keep cumulative count history
new = [] #--- keep totally new bad entries in the last 5 days
pcnt = 0
k = 0
tot = len(data)
for ent in data:
#
#--- read only data entries written in a correct format: <dom><><year>:<ydate><>:<bad_data>...
#
atemp = re.split('<>', ent)
chk1 = mcf.chkNumeric(atemp[0])
btemp = re.split(':', atemp[1])
chk2 = mcf.chkNumeric(btemp[1])
if (chk1 == True) and (int(atemp[0]) > 0) and (chk2 == True) and (int(btemp[1]) > 0):
dom.append(atemp[0])
ydate.append(atemp[1])
#
#--- check the bad data is recorded for the given day
#
if head == 'ccd' or head == 'hccd':
m1 = re.search('\(', atemp[2])
else:
btemp = re.split(':', atemp[2]) #--- case for warm columns
if mcf.chkNumeric(btemp[len(btemp) -1]):
m1 = 'OK'
else:
m1 = None
if m1 is not None:
btemp = re.split(':', atemp[2])
if btemp != '':
dcnt.append(len(btemp))
#
#--- for the last five days, check whether there are any totally new bad entries exists
#
if k > tot - 5:
for test in btemp:
chk = 0
for comp in bad_dat_list:
if test == comp:
chk = 1
continue
if chk == 0:
new.append(test)
bad_dat_list = bad_dat_list + btemp
out = list(set(bad_dat_list))
pcnt = len(out)
ccnt.append(pcnt)
bad_dat_save.append(btemp)
else:
dcnt.append(0)
bad_dat_save.append([])
ccnt.append(pcnt)
k += 1 #--- k is inlimented to check the last 5 days
#
#--- find out which entries are warm/hot and flickering
#
[warm, flick, b_list, p_list]= find_warm_and_flickering(bad_dat_save)
#
#--- open output file to print current information
#
line = data_dir + '/Disp_dir/'+ head + str(ccd) + '_information'
fo = open(line, 'w')
fo.write("warm:\t")
print_data(fo, warm)
fo.write('flick:\t')
print_data(fo, flick)
fo.write('new:\t')
out = list(set(new))
print_data(fo, out)
fo.write('past:\t')
out = list(set(bad_dat_list))
print_data(fo, out)
fo.close()
#
#--- open output file to print out count history
#
ofile = data_dir + 'Disp_dir/' + head + str(ccd) + '_cnt'
fo = open(ofile, 'w')
for i in range(0, len(dom)):
if i < 13:
line = dom[i] + '<>' + ydate[i] + '<>' + str(ccnt[i]) + '<>' + str(dcnt[i]) + '<>0<>0\n'
else:
line = dom[i] + '<>' + ydate[i] + '<>' + str(ccnt[i]) + '<>' + str(dcnt[i]) + '<>'+ str(b_list[i-13]) + '<>' + str(p_list[i-13]) + '\n'
fo.write(line)
fo.close()
