def remove_na_from_lists(alist, blist):
out1 = []
out2 = []
for k in range(0, len(alist)):
if mcf.is_neumeric(alist[k]) and mcf.is_neumeric(blist[k]):
out1.append(alist[k])
out2.append(blist[k])
return [out1, out2]
def find_10th(fits_file):
"""
find 10th brightest value
input: fits_file --- image fits file name
output: 10th brightest value
"""
#
#-- make histgram
#
cmd = ' dmimghist infile=' + fits_file
cmd = cmd + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
expf.run_ascds(cmd)
cmd = ' dmlist infile=outfile.fits outfile=./zout opt=data'
expf.run_ascds(cmd)
data = mcf.read_data_file('./zout', remove=1)
mcf.rm_files('outfile.fits')
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
for ent in data:
try:
atemp = re.split('\s+|\t+', ent)
if (len(atemp) > 3) and mcf.is_neumeric(atemp[1]) \
and mcf.is_neumeric(atemp[2]) and (int(atemp[4]) > 0):
hbin.append(float(atemp[1]))
hcnt.append(int(atemp[4]))
except:
pass
#
#--- checking 10 th bright position
#
try:
j = 0
for i in range(len(hbin) - 1, 0, -1):
if j == 9:
val = i
break
else:
#
#--- only when the value is larger than 0, record as count
#
if hcnt[i] > 0:
j += 1
return hbin[val]
except:
return 'I/INDEF'
def set_format_for_col(name, cdata):
"""
find a format of the input data and set column object
input: name --- column name
cdata --- column data in numpy array form
output: column object
"""
test = str(list(cdata)[0])
mc1 = re.search('True', test)
mc2 = re.search('False', test)
#
#--- check whether the value is numeric
#
if mcf.is_neumeric(test):
ft = 'E'
#
#--- check whether the value is logical
#
elif (mc1 is not None) or (mc2 is not None):
tcnt = len(list(cdata)[0])
ft = str(tcnt) + 'L'
#
#--- all others are set as character set
#
else:
tcnt = len(test)
ft = str(tcnt) + 'A'
return pyfits.Column(name=name, format=ft, array=cdata)
def find_processed_data(inst):
"""
find the hrc obsids which are already re-processed
input: inst --- instrument designation: "i" or "s"
output: out --- a list of obsids
"""
if inst == 'i':
data_dir = '/data/hrc/i/'
else:
data_dir = '/data/hrc/s/'
cmd = 'ls -d ' + data_dir + '/* > ' + zspace
os.system(cmd)
data = mcf.read_data_file(zspace, remove=1)
out = []
for ent in data:
atemp = re.split('\/', ent)
try:
val = int(float(atemp[-1]))
except:
continue
if mcf.is_neumeric(val):
out.append(val)
#
#--- remove duplicate
#
oset = set(out)
out = list(oset)
return out
def adjust_vega_position(cyear=''):
"""
adjust Vega position with proper motion
input: cyear --- the year of the observed. it can be in year date.
if it is not given, no corrections for the propoer motion
output: coordinate [ra, dec] in decimal format.
"""
if not mcf.is_neumeric(cyear):
cyear = 2000.0
#
#--- set several initial values
#
pfile = house_keeping + 'vega_pos'
with open(pfile, 'r') as f:
out = f.read()
atemp = re.split('::', out)
tra = float(atemp[0])
tdec = float(atemp[1])
pra = float(atemp[2]) / 3600.0 / 1000.0
pdec = float(atemp[3]) / 3600.0 / 1000.0
dyear = float(cyear) - 2000.0
tra += dyear * pra
tdec += dyear * pdec
return [tra, tdec]
def three_sigma_values(fits_file):
"""
find 1, 2, and 3 sigma values of an image fits file
input: fits_file --- input image fits file name
output: sigma1 --- one sigma value
sigma2 --- two sigma value
sigma3 --- three sigma value
"""
#
#-- make histgram
#
cmd = ' dmimghist infile=' + fits_file
cmd = cmd + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
run_ascds(cmd)
cmd = ' dmlist infile=outfile.fits outfile=' + zspace + ' opt=data'
run_ascds(cmd)
data = mcf.read_data_file(zspace, remove=1)
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
vsum = 0
for ent in data:
atemp = re.split('\s+|\t+', ent)
if mcf.is_neumeric(atemp[0]):
hbin.append(float(atemp[1]))
val = int(atemp[4])
hcnt.append(val)
vsum += val
#
#--- checking one, two and three sigma counts
#
if len(hbin) > 0:
v68 = int(0.68 * vsum)
v95 = int(0.95 * vsum)
v99 = int(0.997 * vsum)
sigma1 = -999
sigma2 = -999
sigma3 = -999
acc= 0
for i in range(0, len(hbin)):
acc += hcnt[i]
if acc > v68 and sigma1 < 0:
sigma1 = hbin[i]
elif acc > v95 and sigma2 < 0:
sigma2 = hbin[i]
elif acc > v99 and sigma3 < 0:
sigma3 = hbin[i]
break
return [sigma1, sigma2, sigma3]
else:
return[0, 0, 0]
def find_file_and_mp(n_list, nchk=0):
"""
find all *.or files and their mp for given directory
input: n_list --- a list of direcotries
nchk --- an indicator of cartain operation (check the next month case)
output: f_list --- a list of directories
u_list --- a list of mp responsible for the directories
"""
#
#--- find all files with .or ending
#
f_list = []
u_list = []
for idir in n_list:
atemp = re.split('\s+', idir)
tdir = atemp[-1]
line = ''
chk = tdir + '/input/'
if os.path.isdir(chk):
if check_orfiles(chk):
line = line + chk + '*.or '
chk = tdir + '/scheduled/'
if os.path.isdir(chk):
if check_orfiles(chk):
line = line + chk + '*.or '
chk = tdir + '/pre_scheduled/'
if os.path.isdir(chk):
if check_orfiles(chk):
line = line + chk + '*.or '
cmd = 'ls -lrt ' + line + '> ' + zspace
os.system(cmd)
sd_list = mcf.read_data_file(zspace, remove=1)
sd_list.reverse()
#
#--- use the lastest valid file
#
for dfile in sd_list:
mc = re.search('\.or', dfile)
if mc is None:
continue
try:
atemp = re.split('\/', dfile)
btemp = re.split('\.', atemp[-1])
ctemp = re.split('_', btemp[0])
chk = ctemp[1][0]
except:
continue
if mcf.is_neumeric(chk) or (nchk == 0 and (chk in chr_list)):
dtemp = re.split('\s+', dfile)
f_list.append(dtemp[-1])
u_list.append(dtemp[2])
return [f_list, u_list]
def extract_date_from_old_html(ifile):
"""
read the original html page created by flt_run_pipe
input: ifile --- original html file
output: save --- a list of lists of data
tsave --- a list of stating and stopping time in seconds from 1998.1.1
"""
data = mcf.read_data_file(ifile)
#
#--- save the reading in a matirx; initialize with 0.0
#
save = []
tsave = []
for k in range(0, 14):
alist = []
for m in range(0, 7):
alist.append(0.0)
save.append(alist)
#
#--- find the section of each instrument by looking for the instrument name
#
dchk = 0
for ent in data:
#
#--- checking starting and stopping time
#
if dchk < 2:
dmc = re.search('Seconds since', ent)
if dmc is not None:
atemp = re.split('\s+', ent)
stime = float(atemp[0])
tsave.append(stime)
dchk += 1
for n in range(0, 14):
mc = re.search(inst_list[n], ent)
if mc is not None:
k = n
m = 0
break
#
#--- assume that the value appears without any html coding around
#
if mcf.is_neumeric(ent):
if m < 7:
save[k][m] = float(ent)
m += 1
return save, tsave
def chk_proccess_status(inst, hlist):
"""
check whether new processed data actually exist, and if so send out a notification
input: inst --- instrument: i or s
hlist --- a list of obsid
output: email sent out
"""
if inst == 'i':
data_dir = '/data/hrc/i/'
else:
data_dir = '/data/hrc/s/'
cmd = 'ls ' + data_dir + '* > ' + zspace
data = mcf.read_data_file(zspace, remove=1)
d_list = []
for ent in data:
if mcf.is_neumeric(ent):
d_list.append(int(float(ent)))
done = []
for obsid in hlist:
if obsid in d_list:
done.append(obsid)
if len(done) > 0:
line = 'Following obsids are processed for hrc-' + str(inst) + ':\n'
for obsid in done:
line = line + '\t' + str(obsid) + '\n'
#
#--- change the status of processed data
#
cmd = 'chgrp -R hat /data/hrc/i/' + str(obsid)
os.system(cmd)
cmd = 'find /data/hrc/i/ -type d -user isobe -exec chmod a+rx,ug+w,o-w {}'
os.system(cmd)
cmd = 'chmod -fR a+r,g+w,o-w /data/hrc/i/' + str(obsid)
os.system(cmd)
with opne(zspace, 'w') as fo:
fo.write(line)
cmd = 'cat ' + zspace + ' |mailx -s "Subject: HRC Obs Re-processed" [email protected]'
os.system(cmd)
cmd = 'cat ' + zspace + ' |mailx -s "Subject: HRC Obs Re-processed" [email protected]'
os.system(cmd)
mcf.rm_files(zspace)
def extract_hist_data(ifile):
"""
extracting acis hist data from fits file
input: ifile --- fits file name
output: hist_data --- a list of histgram data
"""
tdata = Table.read(ifile, hdu=1)
cols = tdata.columns
sdata = tdata['COUNTS']
hist_data = []
for ent in sdata:
if mcf.is_neumeric(ent):
hist_data.append(float(ent))
return hist_data
def correct_hrc_file_naming():
"""
correct wrong naming of data files
example: hrcf59_evt2.fits ---> hrcf00059_evt2.fits
input: none, but read from /data/mta/<inst>/
output: corrected file names
"""
for inst in ['s', 'i']:
cmd = 'ls -d /data/hrc/'+ inst + '/* > ' + zspace
os.system(cmd)
data = mcf.read_data_file(zspace, remove=1)
obsid_list = []
for ent in data:
atemp = re.split('\/', ent)
obsid = atemp[-1]
if mcf.is_neumeric(obsid):
correct_naming(obsid, inst)
def hrc_gain_run(c_input):
"""
extract hrc evt2 file, find the brightest object and create pha distribution
this is a control script
Input: c_inut --- if it is obsid, use it as a input
otherwise, a list of new candidates will be create based database
Output: <header>_pha.dat --- pha distribution data
<header>_gfit.png --- a plot of pha data
fitting_results --- a table of fitted results
"""
#
#--- if an obsid is provided, analyize that, else get new obsids from databases
#
if mcf.is_neumeric(c_input):
candidate_list = [c_input]
elif isinstance(c_input, list):
candidate_list = c_input
else:
out = ead.extract_calb_arlac()
candidate_list = out[0] + out[1]
#
#--- run the main script only when candidates exist
#
if len(candidate_list) > 0:
#
#--- analyze data
#
chk = hgfv.hrc_gain_fit_voigt(candidate_list)
#
#--- create plots
#
if chk == True:
obt.order_by_time()
hgtp.hrc_gain_trend_plot()
send_notification()
cmd = 'rm -rf ' + exc_dir + 'hrcf*fits*'
os.system(cme)
def read_ede_data(ifile):
"""
read data from a given file
Input: ifile --- input file name
Output: date_list --- a list of date
ede_list --- a list of ede value
error_list --- a list of computed ede error
"""
data = mcf.read_data_file(ifile)
date_list = []
date_list2 = []
ede_list = []
error_list = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.is_neumeric(atemp[0]) == False:
continue
fwhm = float(atemp[2])
ferr = float(atemp[3])
ede = float(atemp[4])
date = atemp[5]
sdate = float(atemp[6])
stime = Chandra.Time.DateTime(date).secs
fyear = mcf.chandratime_to_fraq_year(stime)
date_list.append(fyear)
date_list2.append(sdate)
ede_list.append(ede)
#
#--- the error of EdE is computed using FWHM and its error value
#
error = math.sqrt(ede * ede * ((ferr * ferr) / (fwhm * fwhm)))
error_list.append(error)
return [date_list, date_list2, ede_list, error_list]
def adjust_hz43_position(date):
"""
adjust HZ43 position with proper motion
13 16 21.85329::+29 05 55.3793::-157.96::-110.23
input: date in <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
if it is not given, no corrections for the propoer motion
output: coordinate [ra, dec] in decimal format.
"""
#
#--- evt1 DATE-OBS to fractional year
#
if mcf.is_neumeric(date):
cyear = float(date)
else:
cyear = hcf.convert_time_to_fyear(date)
#
#--- set several initial values
#
pfile = house_keeping + 'hz43_pos'
with open(pfile, 'r') as f:
out = f.read()
atemp = re.split('::', out)
tra = float(atemp[0])
tdec = float(atemp[1])
pra = float(atemp[2]) / 3600.0 / 1000.0
pdec = float(atemp[3]) / 3600.0 / 1000.0
dyear = float(cyear) - 2000.0
tra += dyear * pra
tdec += dyear * pdec
#
#--- convert the foramt to sexagesimal notation
#
[ra, dec] = hcf.convert_ra_dec(tra, tdec)
return [ra, dec]
def create_data_tables(table):
"""
read input table of observations and create data tables
input: table --- a file which contains a list of observations
either obsid list or a list in which the first column is obsid
output: <data_dir>/hrc_<det/pos>_results
"""
data = mcf.read_data_file(table)
for ent in data:
if mcf.is_neumeric(ent):
obsid = ent.strip()
else:
atemp = re.split('\s+', ent)
obsid = atemp[0]
if mcf.is_neumeric(obsid) == False:
continue
print(str(obsid))
fits = hcf.run_arc5gl(0, 0, obsid=obsid, level='2', filetype='evt2')
if fits == False:
write_on_skip_file(obsid)
print("Data is not extracted")
continue
#
#--- if there are multiple output, use only first one
#
if isinstance(fits, list):
fits = fits[0]
xxx = 999
#if xxx == 999:
try:
out = extract_count_stats(fits)
#else:
except:
cmd = 'rm -f ' + fits + '*'
os.system(cmd)
write_on_skip_file(obsid)
print("Analysis Failed")
continue
if out[-1] < 0:
cmd = 'rm -f ' + fits + '*'
os.system(cmd)
write_on_skip_file(obsid)
print("No Output")
continue
line = str(obsid) + '\t'
if float(obsid) < 1000:
line = line + '\t'
line = line + str(fits) + '\t'
line = line + out[7] + '\t'
line = line + '%2.1f' % round(out[6], 1) + '\t'
line = line + '%2.2f' % round(out[5], 2) + '\t'
line = line + '%2.2f' % round(out[8], 2) + '\t'
line = line + '%2.4f' % round(out[9], 4) + '\n'
if out[-1] == 0:
outfile = data_dir + 'hrc_s_0_results'
if out[-1] == 1:
outfile = data_dir + 'hrc_s_10_results'
if out[-1] == 2:
outfile = data_dir + 'hrc_s_25_results'
if out[-1] == 3:
outfile = data_dir + 'hrc_s_m10_results'
if out[-1] == 4:
outfile = data_dir + 'hrc_s_m25_results'
if out[-1] == 10:
outfile = data_dir + 'hrc_i_0_results'
if out[-1] == 11:
outfile = data_dir + 'hrc_i_10_results'
if out[-1] == 12:
outfile = data_dir + 'hrc_i_25_results'
if out[-1] == 13:
outfile = data_dir + 'hrc_i_m10_results'
if out[-1] == 14:
outfile = data_dir + 'hrc_i_m25_results'
with open(outfile, 'a') as fo:
fo.write(line)
cmd = 'rm -f *fits*'
os.system(cmd)
def check_time_format(intime):
"""
return time in Chandra time
input: intime --- time in <yyyy>:<ddd>:<hh>:<mm>:<ss>
or <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss> or chandra time
output: year --- the year
time in chandra time (seconds from 1998.1.1)
"""
mc1 = re.search('-', intime)
mc2 = re.search(':', intime)
#
#--- it is already chandra format
#
if mcf.is_neumeric(intime):
out = Chandra.Time.DateTime(intime).date
atemp = re.split(':', out)
year = int(atemp[0])
return [year, int(float(intime))]
#
#--- time in <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
#
elif mc1 is not None:
mc2 = re.search('T', intime)
if mc2 is not None:
atemp = re.split('T', intime)
btemp = re.split('-', atemp[0])
year = int(float(btemp[0]))
mon = int(float(btemp[1]))
day = int(float(btemp[2]))
ctemp = re.split(':', atemp[1])
hrs = ctemp[0]
mins = ctemp[1]
secs = ctemp[2]
else:
btemp = re.split('-', intime)
year = int(float(btemp[0]))
mon = int(float(btemp[1]))
day = int(float(btemp[2]))
hrs = '00'
mins = '00'
secs = '00'
yday = datetime.date(year, mon, day).timetuple().tm_yday
cyday = str(yday)
if yday < 10:
cyday = '00' + cyday
elif yday < 100:
cyday = '0' + cyday
ytime = btemp[0] + ':' + cyday + ':' + hrs + ':' + mins + ':' + secs
return [year, Chandra.Time.DateTime(ytime).secs]
#
#--- time in <yyyy>:<ddd>:<hh>:<mm>:<ss>
#
elif mc2 is not None:
atemp = re.split(':', intime)
year = int(atemp[0])
return [year, Chandra.Time.DateTime(intime).secs]
def run_process(hrc):
"""
a control script to run reprocess csh scripts
input: hrc: either "hrc_i" or "hrc_s"
output: hrc_i_list --- a list of hrc i obsids which need to be re-processed
hrc_s_list --- a list of hrc s obsids which need to be re-processed
<data_dir>/<obsid> --- re-processed data direcotry
"""
#
#--- set conditions for either hrc-i or hrc s
#
if hrc == 'hrc_i':
out_list = 'hrc_i_list'
data_dir = '/data/hrc/i/'
inst = 'i'
else:
out_list = 'hrc_s_list'
data_dir = '/data/hrc/s/'
inst = 's'
#
#--- make a list of obsids
#
cmd = 'ls -d ' + data_dir + '* > ' + zspace
os.system(cmd)
data = mcf.read_data_file(zspace, remove=1)
hlist = []
for ent in data:
atemp = re.split('\/', ent)
obsid = atemp[-1]
if mcf.is_neumeric(obsid):
hlist.append(obsid)
# if hrc == 'hrc_i':
# print("HRC I : " + str(hlist))
# else:
# print("HRC S : " + str(hlist))
#
for obsid in hlist:
obsid = str(int(float(obsid)))
with open(out_list, 'w') as fo:
fo.write(str(obsid) + '\n')
cmd = 'rm -rf ' + data_dir + obsid + "analysis/*"
os.system(cmd)
#
#--- extract fits data needed for analysis
#
chk = extract_hrc_data(obsid, data_dir)
if chk == False:
print("Not all data are available")
continue
if hrc == 'hrc_i':
cmd = 'csh -f ' + bin_dir + 'repro_all_new.csh hrc_i_list'
else:
cmd = 'csh -f ' + bin_dir + 'repro_all_S_new.csh hrc_s_list'
try:
run_ciao(cmd)
cdir = data_dir + '/' + str(obsid)
if os.path.isdir(cdir):
cmd = 'chgrp -R hat ' + cdir
os.system(cmd)
cmd = 'chmod -R 775 ' + cdir
os.system(cmd)
#
#--- directory name should be 5 digit
#
test = int(float(obsid))
if test < 10000:
chk = mcf.add_leading_zero(obsid, 5)
odir = data_dir + '/' + str(chk)
if os.path.isdir(odir):
cmd = 'rm -rf ' + odir
os.system(cmd)
cmd = 'mv ' + cdir + ' ' + odir
os.system(cmd)
else:
cmd = 'mv ' + cdir + ' ' + odir
os.system(cmd)
except:
pass
mcf.rm_files(out_list)
correct_naming(obsid, inst)
def get_lines(grating):
"""
extract line statistics for a given grating
input: grating --- hetg, metg, or letg
output: acis_<grating>_<line>_data
"""
#
#--- read data file header
#
infile = house_keeping + 'data_header'
with open(infile, 'r') as f:
header = f.read()
#
#--- set which grating data to extract
#
if grating == 'hetg':
cmd = 'ls ' + gdata_dir + '/*/*/obsid_*_L1.5_S1HEGp1_linelist.txt >' + zspace
ofile = 'acis_hetg_'
l_list = h_lines
elif grating == 'metg':
cmd = 'ls ' + gdata_dir + '/*/*/obsid_*_L1.5_S1MEGp1_linelist.txt >' + zspace
ofile = 'acis_metg_'
l_list = m_lines
else:
cmd = 'ls ' + gdata_dir + '/*/*/obsid_*_L1.5_S1LEGp1_linelist.txt >' + zspace
ofile = 'hrc_letg_'
l_list = l_lines
os.system(cmd)
d_list = mcf.read_data_file(zspace, remove=1)
sdate_list = [[], [], [], [], [], [], []]
line_list = [{}, {}, {}, {}, {}, {}, {}]
lcnt = len(l_list)
#
#---- go though each files
#
for dfile in d_list:
out = find_obs_date(dfile)
if out == 'na':
continue
else:
[obsid, ltime, stime] = out
#
#--- extract line information. if energy or fwhm are either "*" or "NaN", skip
#
data = mcf.read_data_file(dfile)
for ent in data:
atemp = re.split('\s+', ent.strip())
if mcf.is_neumeric(atemp[0]):
energy = atemp[2]
fwhm = atemp[3]
if energy == 'NaN':
continue
if (fwhm == '*') or (fwhm == 'NaN'):
continue
energy = mcf.add_tailing_zero(energy, 6)
peak = float(energy)
err = atemp[4]
ede = atemp[5]
line = str(
obsid
) + '\t' + energy + '\t' + fwhm + '\t' + err + '\t' + ede + '\t'
line = line + str(ltime) + '\t' + str(int(stime)) + '\n'
#
#--- find the line value within +/-5 of the expected line center position
#
for k in range(0, lcnt):
center = l_list[k]
low = (center - 5) / 1000.0
top = (center + 5) / 1000.0
if (peak >= low) and (peak <= top):
sdate_list[k].append(stime)
line_list[k][stime] = line
#
#--- output file name
#
for k in range(0, lcnt):
#
#--- print out the data
#
slist = sdate_list[k]
slist.sort()
line = header + '\n'
for sdate in slist:
line = line + line_list[k][sdate]
val = str(l_list[k])
if len(val) < 4:
val = '0' + val
odata = data_dir + ofile + val + '_data'
with open(odata, 'w') as fo:
fo.write(line)
def find_sigma_values(fits):
"""
find 2 sigma, 3sigma, and 4 sigma values of the given data
input: fits --- image fits file name
output: (sigma1, sigma2, sigma3)
"""
#
#-- make histgram
#
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmimghist infile=' + fits
cmd2 = cmd2 + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' dmlist infile=outfile.fits outfile=' + zspace + ' opt=data'
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
data = mcf.read_data_file(zspace, remove=1)
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
vsum = 0
for ent in data:
atemp = re.split('\s+|\t+', ent)
if mcf.is_neumeric(atemp[0]):
hbin.append(float(atemp[1]))
val = int(atemp[4])
hcnt.append(val)
vsum += val
#
#--- checking one sigma and two sigma counts
#
if len(hbin) > 0:
v68 = int(0.68 * vsum)
v95 = int(0.95 * vsum)
v99 = int(0.997 * vsum)
sigma1 = -999
sigma2 = -999
sigma3 = -999
acc = 0
for i in range(0, len(hbin)):
acc += hcnt[i]
if acc > v68 and sigma1 < 0:
sigma1 = hbin[i]
elif acc > v95 and sigma2 < 0:
sigma2 = hbin[i]
elif acc > v99 and sigma3 < 0:
sigma3 = hbin[i]
break
return (sigma1, sigma2, sigma3)
else:
return (0, 0, 0)
def plot_data(data, outname, pos):
"""
plotting data
input: data --- data
outname --- output file name
pos --- indicator of position of aiming
output outname --- png plot of the data
"""
date = []
cnt = []
err = []
for ent in data:
atemp = re.split('\s+', ent)
if not mcf.is_neumeric(atemp[-1]):
continue
try:
ytime = hcf.convert_time_to_fyear(atemp[2],
tformat='%Y-%m-%dT%H:%M:%S')
exp = float(atemp[3])
val = float(atemp[4])
if val < 10:
continue
val /= exp
sig = float(atemp[5]) / exp
date.append(ytime)
cnt.append(val)
err.append(sig)
except:
continue
#
#--- fit a linear line
#
[a, b, sa, sb] = line_fit(date, cnt, err)
#
#--- set min max
#
xmin = 1999.0
ta = time.localtime()
xmax = ta.tm_year + 1
ymin = 0.0
ymax = 6
#
#--- start plotting
#
plt.close('all')
mpl.rcParams['font.size'] = 9
props = font_manager.FontProperties(size=9)
ax = plt.subplot(111)
ax.set_autoscale_on(False)
ax.set_xbound(xmin, xmax)
ax.set_xlim(xmin=xmin, xmax=xmax, auto=False)
ax.set_ylim(ymin=ymin, ymax=ymax, auto=False)
plt.errorbar(date, cnt, yerr=err, fmt='o', lw=1)
plt.xlabel('Time (year)')
plt.ylabel('Source Rate (cnt/s)')
#
#--- plot fitting line
#
start = a + b * xmin
stop = a + b * xmax
plt.plot([xmin, xmax], [start, stop], lw=1, color='blue')
#
#--- write the fitting equation
#
xdiff = xmax - xmin
ydiff = ymax - ymin
xpos = xmin + 0.1 * xdiff
ypos = ymax - 0.1 * ydiff
ac = '%1.2f' % (round(a, 2))
if abs(b) < 0.01:
bc = '%1.4f' % (round(abs(b), 4))
ec = '%1.4f' % (round(sb, 4))
else:
bc = '%1.2f' % (round(abs(b), 2))
ec = '%1.2f' % (round(sb, 2))
if b > 0:
text = '(Source Rate) = ' + ac + ' + (' + bc + '+/-' + ec + ') * Time'
else:
text = '(Source Rate) = ' + ac + ' - (' + bc + '+/-' + ec + ') * Time'
plt.text(xpos, ypos, text)
#
#--- set the size of the plotting area in inch (width: 10.0in, height 5 in)
#
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(10.0, 5.0)
#
#--- save the plot in png format
#
plt.savefig(outname, format='png', dpi=100)
def find_obsids(ifile, o_dict, m_list, mp_person):
"""
find obsids and their related information
input: ifile --- a file name
o_dict --- a dictionary to keep the information about the obsids
m_list --- a list of obsids
mp_person --- mp person responsible for the data period
output: o_dict --- updated data dictionary
m_list --- updated list of obsids
"""
data = mcf.read_data_file(ifile)
cstep = 0 #--- indicator to show which area of the file
ochk = 0 #--- how many times passed "OBS," marker
ichk = 0 #--- how many times passed "ID" marker
for ent in data:
#
#--- after 'SeqNbr' marker, there is a summary table
#
if cstep == 0:
mc1 = re.search('SeqNbr', ent)
#
#--- the summary table finishes with the following marker
#
if cstep == 1:
mc2 = re.search('OR QUICK LOOK END', ent)
if cstep == 2:
mc3 = re.search('OBS\,', ent)
mc4 = re.search('ID', ent)
mc5 = re.search('CAL', ent)
mc6 = re.search('HETG', ent)
mc7 = re.search('LETG', ent)
mc8 = re.search('ACIS', ent)
mc9 = re.search('HRC', ent)
if cstep == 0 and mc1 is not None:
cstep = 1
#
#--- here we are reading the summary table
#
elif cstep == 1:
atemp = re.split('\s+', ent)
if mcf.is_neumeric(atemp[0]):
try:
pid = int(float(atemp[1]))
except:
continue
m_list.append(pid)
#
#--- name of the target is column somewhere between 18 and 40
#
name = ent[18:40]
name.strip()
o_dict[pid] = [atemp[0], name, 'None', 'CAL', mp_person]
if mc1 is not None:
cstep = 1
#
#--- now read more information about the obsidi
#
elif cstep == 2:
if mc3 is not None:
ochk += 1
if mc4 is not None:
ent.strip()
#
#--- find which obsid information are presented next
#
atemp = re.split('\,', ent)
btemp = re.split('ID=', atemp[0])
pid = int(float(btemp[1]))
ichk += 1
#
#--- only when ochk and ichk are equal, collect needed info
#
if ochk == ichk:
if mc5 is not None:
o_dict = update_dict(o_dict, pid, 1, 'CAL')
if mc6 is not None:
o_dict = update_dict(o_dict, pid, 2, 'HETG')
if mc7 is not None:
o_dict = update_dict(o_dict, pid, 2, 'LETG')
if mc8 is not None:
o_dict = update_dict(o_dict, pid, 3, 'ACIS')
if mc9 is not None:
o_dict = update_dict(o_dict, pid, 3, 'HRC')
return [o_dict, m_list]
obsid = atemp[1]
yoffset = int(round(float(atemp[10])))
zoffset = int(round(float(atemp[11])))
odict[obsid] = ['Hrc I', yoffset, zoffset]
for ent in data2:
atemp = re.split('\s+', ent)
obsid = atemp[1]
yoffset = int(round(float(atemp[10])))
zoffset = int(round(float(atemp[11])))
odict[obsid] = ['Hrc S', yoffset, zoffset]
return odict
#--------------------------------------------------------------------
if __name__ == '__main__':
if len(sys.argv) > 1:
obsid = sys.argv[1]
obsid.strip()
if mcf.is_neumeric(obsid):
obsid = int(float(obsid))
create_profile_plot(obsid)
else:
run_for_all_profile_plot(obsid)
else:
run_for_all_profile_plot()
def acis_dose_get_data(startYear='', startMonth='', stopYear='', stopMonth=''):
"""
extract ACIS evt1 data from a month and create combined image file.
input: startYear --- year of starting time
startMonth --- month of starting time
stopYear --- year of stopping time
stopMonth --- month of stopping time
"""
if startYear == '' or startMonth == '' or stopYear == '' or stopMonth == '':
startYear = raw_input('Start Year: ')
startYear = int(float(startYear))
startMonth = raw_input('Start Month: ')
startMonth = int(float(startMonth))
stopYear = raw_input('Stop Year: ')
stopYear = int(float(stopYear))
stopMonth = raw_input('Stop Month: ')
stopMonth = int(float(stopMonth))
#
#--- start extracting the data for the year/month period
#
for year in range(startYear, stopYear + 1):
#
#--- create a list of month appropriate for the year
#
month_list = expf.make_month_list(year, startYear, stopYear,
startMonth, stopMonth)
for month in month_list:
smon = mcf.add_leading_zero(month)
start = str(year) + '-' + smon + '-01T00:00:00'
nextMonth = month + 1
nyear = year
if nextMonth > 12:
nextMonth = 1
nyear += 1
smon = mcf.add_leading_zero(nextMonth)
stop = str(nyear) + '-' + smon + '-01T00:00:00'
#
#--- using ar5gl, get a list of file names
#
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
fitsList = mcf.run_arc5gl_process(line)
#
#--- extract each evt1 file, extract the central part, and combine them into a one file
#
for fits in fitsList:
print("FITS File: " + fits)
atemp = re.split('\s+', line)
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'filename=' + fits + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
#
#--- check whether the fits file actually extracted and if so, ungip the file
#
if len(out) < 1:
continue
cmd = 'gzip -d ' + out[0]
os.system(cmd)
line = fits + '[EVENTS][bin tdetx=2800:5200:1, tdety=1650:4150:1][option type=i4]'
#
#--- create an image file
#
ichk = expf.create_image(line, 'ztemp.fits')
#
#--- combined images
#
if ichk > 0:
expf.combine_image('ztemp.fits', 'total.fits')
mcf.rm_files(fits)
mcf.rm_files('ztemp.fits')
#
#--- rename the file
#
lyear = str(startYear)
lmon = mcf.add_leading_zero(startMonth)
outfile = './ACIS_' + lmon + '_' + lyear + '_full.fits'
cmd = 'mv total.fits ' + outfile
os.system(cmd)
#
#--- trim the extreme values
#
upper = find_10th(outfile)
if mcf.is_neumeric(upper):
outfile2 = './ACIS_' + lmon + '_' + lyear + '.fits'
cmd = ' dmimgthresh infile=' + outfile + ' outfile='
cmd = cmd + outfile2 + ' cut="0:' + str(
upper) + '" value=0 clobber=yes'
expf.run_ascds(cmd)
else:
cmd = 'cp -f ' + outfile + ' ' + outfile2
os.system(cmd)
cmd = 'gzip ' + outfile
os.system(cmd)
#
#--- move full one to the data dir; keep other in <exc_dir> to be used to create cumlative files
#
cmd = 'mv ' + outfile + '* ' + mon_acis_dir + '/.'
os.system(cmd)
def create_img_html(year='', month=''):
"""
create htmls to display exposure map.
input: year --- year, if it is not given, the last month is used
month --- month, if it is not given, the last month is used
output: <web_dir>/Image/HRC<inst>/Month/HRC<inst>_<mm>_<yyyy>_<sec>.html
<web_dir>/Image/HRC<inst>/Cumulative/HRC<inst>_08_1999_<mm>_<yyyy>_<sec>.html
"""
#
#--- find the current year/month
#
atemp = time.strftime('%Y:%m', time.gmtime())
[cyear, cmonth] = re.split(':', atemp)
cyear = int(float(cyear))
cmonth = int(float(cmonth))
chk = 0
#
#--- if the year and month were not passed, set them to those of the last month
#
if mcf.is_neumeric(year):
year = int(float(year))
month = int(float(month))
if year < cyear:
chk = 1
else:
year = cyear
month = cmonth - 1
if month < 1:
month = 12
year -= 1
#
#--- set one month before and one month after
#
pyear = year
pmonth = month - 1
if pmonth < 1:
pmonth = 12
pyear -= 1
nyear = year
nmonth = month + 1
if nmonth > 12:
nmonth = 1
nyear += 1
#
#--- set link dates
#
ldate = mcf.add_leading_zero(month) + '_' + str(year)
pdate = mcf.add_leading_zero(pmonth) + '_' + str(pyear)
ndate = mcf.add_leading_zero(nmonth) + '_' + str(nyear)
monthly = read_template('mon_img_page')
for inst in ('S', 'I'):
if inst == 'S':
cstop = 10
odir = web_img_s_dir + 'Month/'
else:
cstop = 9
odir = web_img_i_dir + 'Month/'
for sec in range(0, cstop):
#
#--- create image file link
#
png = 'HRC' + inst + '_' + ldate + '_' + str(sec) + '.png'
cpath = web_dir + 'Image/HRC' + inst + '/Month/' + png
if os.path.isfile(cpath):
pnglink = './' + png
else:
pnglink = './no_data.png'
#
#--- create link paths
#
pfile = './HRC' + inst + '_' + pdate + '_' + str(sec) + '.html'
nfile = './HRC' + inst + '_' + ndate + '_' + str(sec) + '.html'
psfile = './HRC' + inst + '_' + ldate + '_' + str(sec -
1) + '.html'
nsfile = './HRC' + inst + '_' + ldate + '_' + str(sec +
1) + '.html'
#
#--- section link
#
if sec == 0:
seclink = '<a href="' + nsfile + '">Next Section</a><br />'
elif sec == cstop - 1:
seclink = '<a href="' + psfile + '">Prev Section</a><br />'
else:
seclink = '<a href="' + psfile + '">Prev Section</a> '
seclink = seclink + '<a href="' + nsfile + '">next Section</a><br />'
#
#--- time order link
#
if year == 1999 and month == 8:
tolink = '<a href="' + nfile + '">Next Month</a><br />'
elif chk == 0:
tolink = '<a href="' + pfile + '">Prev Month</a><br /> '
else:
tolink = '<a href="' + pfile + '">Prev Month</a> '
tolink = tolink + '<a href="' + nfile + '">Next Month</a><br />'
#
#--- section main link
#
sublink = '../../../Sub_html/hrc' + inst.lower() + '_' + str(
sec) + '.html'
#
#--- cumulative link
#
cumlink = '<a href="../Cumulative/HRC' + inst + '_08_1999_' + ldate
cumlink = cumlink + '_' + str(sec) + '.html">Cumulative Plot</a>'
#
#--- replace texts in the template
#
otemp = monthly
otemp = otemp.replace("#YEAR#", str(year))
otemp = otemp.replace("#MONTH#", mcf.add_leading_zero(month))
otemp = otemp.replace("#INST#", inst)
otemp = otemp.replace("#SEC#", str(sec))
otemp = otemp.replace("#PNGLINK#", pnglink)
otemp = otemp.replace("#LATEST#", ldate)
otemp = otemp.replace("#CUMLINK#", cumlink)
otemp = otemp.replace("#SECLINK#", seclink)
otemp = otemp.replace("#TOLINK#", tolink)
otemp = otemp.replace("#SUBLINK#", sublink)
#
#--- set output fine name
#
ofile = odir + 'HRC' + inst + '_' + ldate + '_' + str(
sec) + '.html'
with open(ofile, 'w') as fo:
fo.write(otemp)
#
#--- cumulatvie page
#
cumulative = read_template('cum_img_page')
for inst in ('S', 'I'):
if inst == 'S':
cstop = 10
odir = web_img_s_dir + 'Cumulative/'
else:
cstop = 9
odir = web_img_i_dir + 'Cumulative/'
for sec in range(0, cstop):
#
#--- create link paths
#
pfile = './HRC' + inst + '_08_1999_' + pdate + '_' + str(
sec) + '.html'
nfile = './HRC' + inst + '_08_1999_' + ndate + '_' + str(
sec) + '.html'
psfile = './HRC' + inst + '_08_1999_' + ldate + '_' + str(
sec - 1) + '.html'
nsfile = './HRC' + inst + '_08_1999_' + ldate + '_' + str(
sec + 1) + '.html'
#
#--- section link
#
if sec == 0:
seclink = '<a href="' + nsfile + '">next Section</a><br />'
elif sec == cstop - 1:
seclink = '<a href="' + psfile + '">Prev Section</a><br />'
else:
seclink = '<a href="' + psfile + '">Prev Section</a> '
seclink = seclink + '<a href="' + nsfile + '">next Section</a><br />'
#
#--- time order link
#
if year == 1999 and month == 8:
tolink = '<a href="' + nfile + '">Next Month</a><br />'
elif chk == 0:
tolink = '<a href="' + pfile + '">Prev Month</a><br /> '
else:
tolink = '<a href="' + pfile + '">Prev Month</a> '
tolink = tolink + '<a href="' + nfile + '">Next Month</a><br />'
#
#--- section main link
#
sublink = '../../../Sub_html/hrc' + inst.lower() + '_' + str(
sec) + '.html'
#
#--- month link
#
monlink = '<a href="../Month/HRC' + inst + '_' + ldate + '_' + str(
sec) + '.html">'
monlink = monlink + 'Month Plot</a>'
#
#--- replace texts in the template
#
otemp = cumulative
otemp = otemp.replace("#YEAR#", str(year))
otemp = otemp.replace("#MONTH#", mcf.add_leading_zero(month))
otemp = otemp.replace("#INST#", inst)
otemp = otemp.replace("#SEC#", str(sec))
otemp = otemp.replace("#LATEST#", ldate)
otemp = otemp.replace("#MONLINK#", monlink)
otemp = otemp.replace("#SECLINK#", seclink)
otemp = otemp.replace("#TOLINK#", tolink)
otemp = otemp.replace("#SUBLINK#", sublink)
#
#--- set output fine name
#
ofile = odir + 'HRC' + inst + '_08_1999_' + ldate + '_' + str(
sec) + '.html'
with open(ofile, 'w') as fo:
fo.write(otemp)
def create_data_tables(table):
"""
read input table of observations and create data tables
input: table --- a file which ontains a list of observations
either obsid list or a list in which the first column is obsid
output: <data_dir>/hrc_<det/pos>_results
"""
data = mcf.read_data_file(table)
for ent in data:
if mcf.is_neumeric(ent):
obsid = ent.strip()
else:
atemp = re.split('\s+', ent)
obsid = atemp[0]
if not mcf.is_neumeric(obsid):
continue
print(obsid)
fits = hcf.run_arc5gl(0,
0,
obsid=str(obsid),
level='1',
filetype='evt1')
out = extract_count_stats(fits)
if out[-1] < 0:
continue
line = str(obsid) + '\t'
if float(obsid) < 1000:
line = line + '\t'
line = line + str(fits) + '\t'
line = line + out[7] + '\t'
line = line + '%2.1f' % round(out[6], 1) + '\t'
line = line + '%2.2f' % round(out[5], 2) + '\t'
line = line + '%2.2f' % round(out[8], 2) + '\t'
line = line + '%2.4f' % round(out[9], 4) + '\n'
if out[-1] == 10:
outfile = data_dir + 'hrc_i_results'
if out[-1] == 0:
outfile = data_dir + 'hrc_s_0_results'
if out[-1] == 1:
outfile = data_dir + 'hrc_s_10_results'
if out[-1] == 2:
outfile = data_dir + 'hrc_s_25_results'
if out[-1] == 3:
outfile = data_dir + 'hrc_s_m10_results'
if out[-1] == 4:
outfile = data_dir + 'hrc_s_m25_results'
with open(outfile, 'a') as fo:
fo.write(line)
cmd = 'rm -f ' + fits
os.system(cmd)
def extract_sim_data():
"""
extract sim data from PRIMARYCCDM_*.*.tl
input: none but read from <dump_dir>/PRIMARYCCDM_*.*.tl
output: <data_dir>sim_data.out
"""
#
#--- find the time of the last entry from the sim_data.out
#
sfile = data_dir + 'sim_data.out'
data = mcf.read_data_file(sfile)
#
#--- cleaning up the data; drop the data which the date starts from ":" e.g. :2014
#
pdata = []
for ent in data:
if re.search('^:', ent):
continue
else:
pdata.append(ent)
#
#--- the last entiry values
#
if len(pdata) > 0:
atemp = re.split('\s+', pdata[-1])
ltime = int(float(Chandra.Time.DateTime(atemp[0]).secs))
time_2 = atemp[0]
col1_2 = atemp[1]
col2_2 = atemp[2]
col3_2 = atemp[3]
else:
ltime = 0
time_2 = 0
col1_2 = ''
col2_2 = ''
col3_2 = ''
#
#--- check whether input files exists
#
cmd = 'ls -rt ' + dump_dir + 'PRIMARYCCDM_*.*.tl >' + zspace
os.system(cmd)
data = mcf.read_data_file(zspace, remove=1)
dlen = len(data)
if dlen < 1:
exit(1)
#
#--- files exist. read the data from the last 40 files
#
if dlen > 40:
tlist = data[-40:]
else:
tlist = data
for ent in tlist:
cmd = 'cat ' + ent + ' >> ' + zspace
os.system(cmd)
data = mcf.read_data_file(zspace, remove=1)
#
#--- go though each data line
#
prev = ''
sline = ''
for ent in data:
#
#--- expect the first letter of the data line is numeric (e.g. 2014).
#
try:
val = float(ent[0])
except:
continue
#
#--- only data with "FMT" format will be used
#
mc = re.search('FMT', ent)
if mc is None:
continue
#
#--- if there are less than 20 entries, something wrong; skip it
#
atemp = re.split('\t+', ent)
if len(atemp) < 20:
continue
#
#--- convert time format
#
ttime = atemp[0]
ttime = ttime.strip()
ttime = ttime.replace(' ', ':')
ttime = ttime.replace(':::', ':00')
ttime = ttime.replace('::', ':0')
#
#--- if the time is exactly same as one before, skip it
#
if ttime == time_2:
continue
#
#--- if the time is already in the database, skip it
#
stime = int(float(Chandra.Time.DateTime(ttime).secs))
if stime <= ltime:
continue
#
#--- use only data which tscpos and fapos have numeric values
#
tscpos = atemp[4].strip()
fapos = atemp[5].strip()
if tscpos == "" or fapos == "":
continue
else:
if (mcf.is_neumeric(tscpos)) and (mcf.is_neumeric(fapos)):
tscpos = str(int(float(tscpos)))
fapos = str(int(float(fapos)))
else:
continue
mpwm = atemp[12].strip()
if mcf.is_neumeric(mpwm):
mpwm = int(float(mpwm))
mpwm = str(mpwm)
else:
mpwm = '0'
#
#--- we want to print only beginning and ending of the same data entries.
#--- skip the line if all three entiries are same as one before, except the last one
#
if (col1_2 == tscpos) and (col2_2 == fapos) and (col3_2 == mpwm):
time_2 = ttime
continue
line = ttime + '\t' + str(tscpos) + '\t' + str(fapos) + '\t' + str(
mpwm) + '\n'
if line == prev:
continue
else:
pline = time_2 + '\t' + str(col1_2) + '\t' + str(
col2_2) + '\t' + str(col3_2) + '\n'
sline = sline + pline + line
prev = line
time_2 = ttime
col1_2 = tscpos
col2_2 = fapos
col3_2 = mpwm
with open('./temp_save', 'w') as fo:
fo.write(sline)
sfile2 = sfile + '~'
cmd = 'cp ' + sfile + ' ' + sfile2
os.system(cmd)
cmd = 'cat ./temp_save >> ' + sfile
os.system(cmd)
mcf.rm_file('./temp_save')
def create_history_file(head):
"""
create count history file and the information file containing current bad entry information
input: head --- ccd, hccd, or col to indicate which data to handle
output: <head>_ccd<ccd>_cnt --- count history data:
<stime><><year:ydate><><cumlative cnt><><cnt for the day>
<head>_ccd<ccd>_information --- current information of the bad entries.
Example, list of warm pixels, flickering pixels,
totally new pixels,
and all past and current warm pixels.
"""
for ccd in range(0, 10):
#
#--- read data file head is either ccd, hccd, or col
#
ifile = data_dir + 'hist_' + head + str(ccd)
data = mcf.read_data_file(ifile)
bad_dat_list = [] #--- save all bad data as elements
bad_dat_save = [] #--- save all bad data as a list for each day
stime = []
ydate = []
dcnt = [] #--- keep discreate count history
ccnt = [] #--- keep cumulative count history
new = [] #--- keep totally new bad entries in the last 5 days
pcnt = 0
k = 0
tot = len(data)
for ent in data:
#
#--- read only data entries written in a correct format: <stime><><year>:<ydate><>:<bad_data>...
#
atemp = re.split('<>', ent)
chk1 = mcf.is_neumeric(atemp[0])
btemp = re.split(':', atemp[1])
chk2 = mcf.is_neumeric(btemp[1])
if (chk1 == True) and (int(
atemp[0]) > 0) and (chk2 == True) and (int(btemp[1]) > 0):
stime.append(atemp[0])
ydate.append(atemp[1])
#
#--- check the bad data is recorded for the given day
#
if head == 'ccd' or head == 'hccd':
m1 = re.search('\(', atemp[2])
else:
btemp = re.split(':', atemp[2]) #--- case for warm columns
if mcf.is_neumeric(btemp[len(btemp) - 1]):
m1 = 'OK'
else:
m1 = None
if m1 is not None:
btemp = re.split(':', atemp[2])
if btemp != '':
dcnt.append(len(btemp))
#
#--- for the last five days, check whether there are any totally new bad entries exists
#
if k > tot - 5:
for test in btemp:
chk = 0
for comp in bad_dat_list:
if test == comp:
chk = 1
continue
if chk == 0:
new.append(test)
bad_dat_list = bad_dat_list + btemp
out = list(set(bad_dat_list))
pcnt = len(out)
ccnt.append(pcnt)
bad_dat_save.append(btemp)
else:
dcnt.append(0)
bad_dat_save.append([])
ccnt.append(pcnt)
k += 1 #--- k is incremented to check the last 5 days
#
#--- find out which entries are warm/hot and flickering
#
[warm, flick, b_list, p_list] = find_warm_and_flickering(bad_dat_save)
#
#--- open output file to print current information
#
ofile = data_dir + '/' + head + str(ccd) + '_information'
with open(ofile, 'w') as fo:
fo.write("warm:\t")
print_data(fo, warm)
fo.write('flick:\t')
print_data(fo, flick)
fo.write('new:\t')
out = list(set(new))
print_data(fo, out)
fo.write('past:\t')
out = list(set(bad_dat_list))
print_data(fo, out)
#
#--- open output file to print out count history
#
line = ''
for i in range(0, len(stime)):
if i < 13:
line = line + stime[i] + '<>' + ydate[i] + '<>' + str(
ccnt[i]) + '<>'
line = line + str(dcnt[i]) + '<>0<>0\n'
else:
line = line + stime[i] + '<>' + ydate[i] + '<>' + str(
ccnt[i]) + '<>'
line = line + str(dcnt[i]) + '<>' + str(
b_list[i - 13]) + '<>' + str(p_list[i - 13]) + '\n'
ofile = data_dir + '' + head + str(ccd) + '_cnt'
with open(ofile, 'w') as fo:
fo.write(line)
