def get_dtf(obsid, fits):
"""
create a list of dtf for the given time periods
input: obsid --- obsid of the observations
fits --- event fits file name
output: dsave --- a list of dtf values correspoinding to the event fits data
"""
#
#--- get fits time list
#
t = pyfits.open(fits)
tdata = t[1].data
t.close()
ftime = tdata.field('time')
#
#--- get dtf time list and dtf rate list
#
dtf = hcf.run_arc5gl(0, 0, obsid=obsid, level='1', filetype='dtf')
t = pyfits.open(dtf)
tdata = t[1].data
t.close()
dtime = tdata.field('time')
drate = tdata.field('dtf')
#
#--- compare them and find dtf correspond to the time of fits data
#
dlen = len(dtime)
dsave = []
for ent in ftime:
for k in range(0, dlen - 1):
if ent >= dtime[k] and ent < dtime[k + 1]:
dsave.append(drate[k])
break
mcf.rm_files(dtf)
return dsave
def get_dead_period(obsid):
"""
extract time periods which is dtf <= 0.98
input: obsid --- obsid
output: pstart --- starting time
pstop --- stoppping time
these are +/- 2 of the center value listed on dtf fits file
"""
#
#--- get hrc dtf fits file
#
dtf = hcf.run_arc5gl(start='', stop='', obsid=obsid, operation='retrieve',\
level ='1', filetype='dtf')
#
#--- read out time and dtf column from the fits file
#
tdata = hcf.read_fits_file(dtf)
tout = hcf.extract_col_data(tdata[1], ['time', 'dtf'])
mcf.rm_files(dtf)
#
#--- find the period which match the condition (dtf<=0.98)
#
pstart = []
pstop = []
for k in range(0, len(tout)):
try:
cdtf = float(tout[k][1])
except:
continue
if cdtf <= 0.98:
time = int(float(tout[k][0]))
#
#--- take the period to be +/- 2 of the center value listed
#
pstart.append(time - 2)
pstop.append(time + 2)
return [pstart, pstop]
def get_dead_period(obsid):
"""
extract time periods which is dtf <= 0.98
input: obsid --- obsid
output: pstart --- starting time
pstop --- stoppping time
these are +/- 2 of the center value listed on dtf fits file
"""
#
#--- get hrc dtf fits file
#
dtf = hcf.run_arc5gl(start='', stop='', obsid=obsid, operation='retrieve',\
level ='1', filetype='dtf')
#
#--- read out time and dtf column from the fits file
#
t = pyfits.open(dtf)
fdata = t[1].data
tdata = fdata['time']
ddata = fdata['dtf']
t.close()
mcf.rm_files(dtf)
#
#--- find the period which match the condition (dtf<=0.98)
#
pstart = []
pstop = []
for k in range(0, len(tdata)):
if float(ddata[k]) <= 0.98:
xtime = int(float(tdata[k]))
#
#--- take the period to be +/- 2 of the center value listed
#
pstart.append(xtime - 2)
pstop.append(xtime + 2)
return [pstart, pstop]
def extract_data(obsid, inst):
"""
extract needed information for a given observation
input: obsid --- obsid
output: a list of lists:
[inst, date, start, crsv, crsu, amp_sf, pha, pi, sumamps, samp, chip_id,
tg_m, tg_mlam, tg_srcid, tg_part, tg_smap]
note: inst,date, start are strings, but all others are lists.
"""
#
#--- extract evt1 data
#
evt1 = hcf.run_arc5gl(0, 0, obsid = obsid, operation='retrieve', level ='1', filetype='evt1')
if evt1 == "":
return False
#
#--- extract evt1.5 data
#
# evta = hcf.run_arc5gl(0, 0, obsid = obsid, operation='retrieve', level ='1.5', filetype='tgevt1')
# if evta == "":
# return False
#
#--- find which instrument
#
hdr = pyfits.getheader(evt1, 1)
inst = hdr['detnam']
date = hdr['date-obs']
start = hdr['tstart']
stop = hdr['tstop']
expo = hdr['exposure']
ra_nom = float(hdr['ra_nom'])
dec_nom = float(hdr['dec_nom'])
roll_nom = float(hdr['roll_nom'])
ra_targ = float(hdr['ra_targ'])
dec_targ = float(hdr['dec_targ'])
[ra_nom2, dec_nom2] = convert_to_coordinates(ra_nom, dec_nom, evt1, 'sky')
[ra_targ2, dec_targ2] = convert_to_coordinates(ra_targ, dec_targ, evt1, 'sky')
[ra_nom3, dec_nom3] = convert_to_coordinates(ra_nom, dec_nom, evt1, 'det')
[ra_targ3, dec_targ3] = convert_to_coordinates(ra_targ, dec_targ, evt1, 'det')
hdata = [inst, date, start, stop, expo, ra_nom2, dec_nom2, ra_targ2, dec_targ2]
hdata2 = [ra_nom, dec_nom, roll_nom, ra_targ, dec_targ]
hdata3 = [ra_nom3, dec_nom3, ra_targ3, dec_targ3]
[yoffset, zoffset] = extract_offset_info(obsid, inst)
[yoffset, zoffset] = compute_y_z_offset(ra_nom, dec_nom, roll_nom, ra_targ, dec_targ)
#
#--- combine the data
#
# comb_evt1 = 'comb_evt.fits'
# col_list = ['amp_sf', 'sumamps']
# add_coldata_to_fits(evta, evt1, col_list, comb_evt1)
#
#--- clean the data
#
clean1 = 'evt1_clean.fits'
try:
fef.filter_evt_file(evt1, clean1, obsid)
except:
return False
#
#--- get the target area
#
carea = 'clean_targ.fits'
barea = 'clean_targ_bkg.fits'
chk = get_target_data(obsid, evt1, clean1, carea, barea, ra_targ, dec_targ, yoffset, zoffset, inst)
if chk:
hdata.append(chk[0])
hdata.append(chk[1])
tarea = chk[2] #--- center area in pix
sarea = chk[3] #--- background area in pix
cent_data = get_data(carea, ['x', 'y', 'pi','amp_sf', 'sumamps'])
samp = compute_samp(cent_data[4], cent_data[3], inst)
cent_data.append(samp)
cent_bkg = get_data(barea, ['x', 'y', 'pi','amp_sf', 'sumamps'])
samp = compute_samp(cent_bkg[4], cent_bkg[3], inst)
cent_bkg.append(samp)
else:
hdata.append(0.0)
hdata.append(0.0)
cent_data = []
cent_bkg = []
tarea = 0
sarea = 0
hdata = hdata + hdata2 + hdata3
hdata.append(tarea)
hdata.append(sarea)
return [hdata, cent_data, cent_bkg]
def hrc_gain_fit_voigt(candidate_list):
"""
extract hrc evt2 file, find the brightest object and create pha distribution
Input: candidate_list --- if it is obsid, use it as a input
otherwise, a list of new candidates will be create based database
Output: <header>_pha.dat --- pha distribution data
<header>_gfit.png --- a plot of pha data
fitting_results --- a table of fitted results
"""
#
#--- keep the results in save_line
#
save_line = ''
#
#--- if an obsid is provided, analyize that, else get new obsids from databases
#
if len(candidate_list) > 0:
for obsid in candidate_list:
print(str(obsid))
#hfile = extract_hrc_evt2(obsid)
hfile = hcf.run_arc5gl(0, 0, obsid, level='2', filetype='evt2')
if hfile == 'na':
sfile = house_keeping + 'skip_obsids'
with open(sfile, 'a') as fo:
fo.write(str(obsid) + '\n')
continue
#
#--- get a file name header for the later use
#
temp = re.split('N', hfile)
hname = temp[0]
#
#--- extract information from the fits file header
#
hrdr = pyfits.getheader(hfile, 1)
#
#--- current AR Lac position adjucted with proper motion
#
date = hrdr['date-obs']
[ra, dec] = aap.adjust_arlac_position(date)
#
#--- find the diffrence between real AR Lac position and nominal postion
#--- so that we can determin how much area we should include
#
ra_diff = abs(ra - float(hrdr['ra_pnt'])) * 60.0
dec_diff = abs(dec - float(hrdr['dec_pnt'])) * 60.0
rad_diff = math.sqrt(ra_diff * ra_diff + dec_diff * dec_diff)
if rad_diff < 10.0:
fit_rad = 60.0
else:
fit_rad = 200.0
#
#--- find a location of the brightest object (assume it is AR Lac) in sky coordinates
#
try:
[x, y] = find_center(hfile)
except:
sfile = house_keeping + 'skip_obsids'
with open(sfile, 'a') as fo:
fo.write(str(obsid) + '\n')
continue
#
#--- extract pha values in the given area
#
pha = extract_pha(hfile, x, y, fit_rad)
#--- create pha count distribution
#
pmax = max(pha) + 1
pha_bin = [x for x in range(0, pmax)]
pha_hist = [0 for x in range(0, pmax)]
for ent in pha:
pha_hist[ent] += 1
#
#--- print out the distirbution results
#
line = ''
for i in range(0, pmax):
line = line + str(pha_bin[i]) + '\t' + str(pha_hist[i]) + '\n'
outfile = data_dir + hname + '_pha.dat'
with open(outfile, 'w') as fo:
fo.write(line)
#
#--- gzip file
#
cmd = 'gzip -f ' + outfile
os.system(cmd)
#
#--- find median point
#
med = find_med(pha_hist)
#
#--- fit a voigt distribution on the data
#
try:
[center, width, amp, alphaD, alphaL, I, a_back, b_back] \
= voigt.fit_voigt_profile(pha_bin, pha_hist, type='voigt', plot_title=hfile)
except:
[center, width, amp, alphaD, alphaL, I, a_back, b_back] \
= [0, 0, 0, 0, 0, 0, 0, 0]
#
#--- rename a plotting file
#
outfile = plot_dir + 'Indivisual_Plots/' + hname + '_gfit.png'
cmd = 'mv out.png ' + outfile
os.system(cmd)
line = str(obsid) + '\t' + str(hrdr['date-obs']) + '\t' + str(
hrdr['tstart']) + '\t'
line = line + hrdr['detnam'] + '\t' + str(hrdr['ra_pnt']) + '\t'
line = line + str(hrdr['dec_pnt']) + '\t\t' + str(round(
ra_diff, 3)) + '\t'
line = line + str(round(dec_diff, 3)) + '\t' + str(
round(rad_diff, 3)) + '\t'
line = line + str(med) + '\t\t' + str(round(center, 3)) + '\t'
line = line + str(round(amp, 3)) + '\t' + str(round(width,
3)) + '\t'
line = line + str(hrdr['roll_pnt']) + '\t' + str(
hrdr['foc_len']) + '\t'
line = line + str(hrdr['defocus']) + '\t' + str(
hrdr['sim_x']) + '\t'
line = line + str(hrdr['sim_y']) + '\t' + str(hrdr['sim_z']) + '\t'
line = line + str(round(alphaD, 4)) + '\t' + str(round(alphaL,
4)) + '\t'
line = line + str(round(center, 3)) + '\t' + str(round(I,
2)) + '\t'
line = line + str(round(a_back, 2)) + '\t' + str(round(b_back,
2)) + '\n'
save_line = save_line + line
#
#--- remove the evt2 file
#
mcf.rm_files(hfile)
#
#--- if there is any new data, print it out
#
if save_line != '':
#
#--- print out the fitting result
#
outfile = data_dir + 'fitting_results'
copied_file = outfile + '~'
cmd = 'cp ' + outfile + ' ' + copied_file
os.system(cmd)
with open(outfile, 'a') as fo:
fo.write(save_line)
return True
else:
return False
def create_hz43_center_location_list():
"""
extract coordinates from manually process data list
input: none but read from 'hrci_hz43_zero.tar, hrcs_hz43_zero.tar
output: <house_keeping>/hrc_i_coords, <hosue_keeping>/hrc_s_coords
both has <obsid> <ra> <dec> <skyx> <skyy>
"""
for k in range(0, 2):
#
#--- find out which obsids are already processed
#
cfile = house_keeping + olist[k]
f = open(cfile, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
obs_list = []
for ent in data:
atemp = re.split('\s+', ent)
obs_list.append(atemp[0])
fo = open('./zdata', 'w')
#
#--- check which data are available
#
cmd = 'cp ' + tpath + dlist[k] + ' . '
os.system(cmd)
cmd = 'tar xf ' + dlist[k]
os.system(cmd)
cmd = 'ls src*.reg > ' + zspace
os.system(cmd)
data = read_data(zspace, remove=1)
chk = 0
for ent in data:
mc = re.search('src_zero_', ent)
if mc is not None:
cut = 'src_zero_'
else:
cut = 'src'
atemp = re.split(cut, ent)
btemp = re.split('.reg', atemp[1])
obsid = btemp[0]
#
#--- check this obsid is in the saved data list
#
if obsid in obs_list:
continue
f = open(ent, 'r')
out = f.read()
f.close()
atemp = re.split('\(', out)
btemp = re.split(',', atemp[1])
ra = btemp[0]
dec = btemp[1]
#
#--- extract evt1 data
#
evt1 = hcf.run_arc5gl(0, 0, obsid = obsid, operation='retrieve', level ='1', filetype='evt1')
if evt1 == "":
continue
#
#--- convert coordinates from cel to sky
#
cmd = 'dmcoords ' + evt1 + ' opt=cel ra=' + ra + ' dec=' + dec + ' verbose=1 > ' + zspace
run_ciao(cmd)
#
#--- extract sky coordindats
#
info = read_data(zspace, remove=1)
atemp = re.split('\s+', info[-2])
skyx = atemp[2]
skyy = atemp[3]
line = obsid + '\t' + ra + '\t' + dec + '\t' + skyx + '\t' + skyy + '\n'
fo.write(line)
chk = 1
mcf.rm_file(evt1)
fo.close()
if chk > 0:
cmd = 'cat ./zdata >> ' + cfile
os.system(cmd)
cmd = 'rm -rf *.reg *.tar ./zdata'
os.system(cmd)
def extract_data(obsid, evt1):
"""
extract needed information for a given observation
input: obsid --- obsid
evt1 --- hrc evt1 fits file name
output: a list of lists:
[inst, date, start, crsv, crsu, amp_sf, pha, pi, sumamps, samp, chip_id,
tg_m, tg_mlam, tg_srcid, tg_part, tg_smap]
note: inst,date, start are strings, but all others are lists.
"""
#
#--- evt1 data --- already extracted
#
if evt1 == "":
print("no EVT1 fits file")
return False
#
#--- extract evt1.5 data
#
evta = hcf.run_arc5gl(0,
0,
obsid=obsid,
operation='retrieve',
level='1.5',
filetype='tgevt1')
if evta == "":
print("no EVT1.5 fits file")
sfile = house_keeping + 'skip_obsids'
with open(sfile, 'a') as fo:
fo.write(str(obsid) + '\n')
return False
#
#--- find which instrument
#
hdr = pyfits.getheader(evt1, 1)
inst = hdr['detnam']
date = hdr['date-obs']
start = hdr['tstart']
stop = hdr['tstop']
expo = hdr['exposure']
hdata = [inst, date, start, stop, expo]
#
#--- we need to drop a large off axis data. set 9 arcmin
#
ra_pnt = float(hdr['ra_pnt'])
dec_pnt = float(hdr['dec_pnt'])
ra_targ = float(hdr['ra_targ'])
dec_targ = float(hdr['dec_targ'])
adiff = ra_targ - ra_pnt
bdiff = dec_targ - dec_pnt
gout = adiff * adiff + bdiff * bdiff
adiff = math.sqrt(gout)
if adiff > 0.15:
print("Offset too large (>9 amin); dropped")
return False
#
#--- combine the data
#
comb_evt1 = 'comb_evt.fits'
col_list = ['amp_sf', 'sumamps']
add_coldata_to_fits(evta, evt1, col_list, comb_evt1)
#
#--- clean the data
#
clean1 = 'evt1_clean.fits'
carea = 'clean_cent.fits'
barea = 'clean_cent_bkg.fits'
fef.filter_evt_file(comb_evt1, clean1, obsid)
#
#--- get the center part
#
chk = get_center_data(obsid, evt1, clean1, carea, barea)
if chk:
cent_data = get_data(
carea, ['tg_m', 'tg_r', 'tg_d', 'pi', 'amp_sf', 'sumamps'])
samp = compute_samp(cent_data[5], cent_data[4], inst)
cent_data.append(samp)
cent_bkg = get_data(
barea, ['tg_m', 'tg_r', 'tg_d', 'pi', 'amp_sf', 'sumamps'])
samp = compute_samp(cent_bkg[5], cent_bkg[4], inst)
cent_bkg.append(samp)
else:
cent_data = []
cent_bkg = []
#
#--- get the arm parts
#
carea = 'clean_arms.fits'
barea = 'clean_arms_bkg.fits'
get_arm_data(clean1, carea, barea)
arms_data = get_data(carea,
['tg_m', 'tg_r', 'tg_d', 'pi', 'amp_sf', 'sumamps'])
samp = compute_samp(arms_data[5], arms_data[4], inst)
arms_data.append(samp)
arms_bkg = get_data(barea,
['tg_m', 'tg_r', 'tg_d', 'pi', 'amp_sf', 'sumamps'])
samp = compute_samp(arms_bkg[5], arms_bkg[4], inst)
arms_bkg.append(samp)
return [hdata, cent_data, cent_bkg, arms_data, arms_bkg]
def create_data_tables(table):
"""
read input table of observations and create data tables
input: table --- a file which contains a list of observations
either obsid list or a list in which the first column is obsid
output: <data_dir>/hrc_<det/pos>_results
"""
data = mcf.read_data_file(table)
for ent in data:
if mcf.is_neumeric(ent):
obsid = ent.strip()
else:
atemp = re.split('\s+', ent)
obsid = atemp[0]
if mcf.is_neumeric(obsid) == False:
continue
print(str(obsid))
fits = hcf.run_arc5gl(0, 0, obsid=obsid, level='2', filetype='evt2')
if fits == False:
write_on_skip_file(obsid)
print("Data is not extracted")
continue
#
#--- if there are multiple output, use only first one
#
if isinstance(fits, list):
fits = fits[0]
xxx = 999
#if xxx == 999:
try:
out = extract_count_stats(fits)
#else:
except:
cmd = 'rm -f ' + fits + '*'
os.system(cmd)
write_on_skip_file(obsid)
print("Analysis Failed")
continue
if out[-1] < 0:
cmd = 'rm -f ' + fits + '*'
os.system(cmd)
write_on_skip_file(obsid)
print("No Output")
continue
line = str(obsid) + '\t'
if float(obsid) < 1000:
line = line + '\t'
line = line + str(fits) + '\t'
line = line + out[7] + '\t'
line = line + '%2.1f' % round(out[6], 1) + '\t'
line = line + '%2.2f' % round(out[5], 2) + '\t'
line = line + '%2.2f' % round(out[8], 2) + '\t'
line = line + '%2.4f' % round(out[9], 4) + '\n'
if out[-1] == 0:
outfile = data_dir + 'hrc_s_0_results'
if out[-1] == 1:
outfile = data_dir + 'hrc_s_10_results'
if out[-1] == 2:
outfile = data_dir + 'hrc_s_25_results'
if out[-1] == 3:
outfile = data_dir + 'hrc_s_m10_results'
if out[-1] == 4:
outfile = data_dir + 'hrc_s_m25_results'
if out[-1] == 10:
outfile = data_dir + 'hrc_i_0_results'
if out[-1] == 11:
outfile = data_dir + 'hrc_i_10_results'
if out[-1] == 12:
outfile = data_dir + 'hrc_i_25_results'
if out[-1] == 13:
outfile = data_dir + 'hrc_i_m10_results'
if out[-1] == 14:
outfile = data_dir + 'hrc_i_m25_results'
with open(outfile, 'a') as fo:
fo.write(line)
cmd = 'rm -f *fits*'
os.system(cmd)
def analyze_hz43_data(obsid='', inst='i'):
"""
extract data, analyze and update data tables
input: none but download data with arc5gl
output: <house_keeping>/hrc_i_coords, <hosue_keeping>/hrc_s_coords
both has <obsid> <ra> <dec> <detx> <dety>
<data_dir>/pi_* and samp_*
"""
#
#--- find new HZ43 observations data_set: [hrc-i, hrc-s]
#
if obsid == '':
data_set = fhd.extract_calb_hz43()
else:
if inst == 'i':
data_set = [[
obsid,
], []]
else:
data_set = [[], [
obsid,
]]
if (len(data_set[0]) == 0) and (len(data_set[1]) == 0):
print("No new HZ43 data found")
exit(1)
for k in range(0, 2):
data_list = data_set[k]
if len(data_list) == 0:
continue
if k == 0:
print("Analyzing HRC I")
else:
print("Analyzing HRC S")
zline = ''
for obsid in data_list:
print("OBSID: " + str(obsid))
#
#--- extract evt1 data
#
evt1 = hcf.run_arc5gl(0, 0, obsid = obsid, operation='retrieve',\
level ='1', filetype='evt1')
if evt1 == "":
print("No evt1 file extracted")
sfile = house_keeping + 'skip_obsids'
with open(sfile, 'a') as fo:
fo.write(str(obsid) + '\n')
continue
#
#--- get the ra/dec of HZ43 adjusted to proper motion
#
fout = pyfits.open(evt1)
fhead = fout[1].header
otime = fhead['DATE-OBS']
[ra, dec] = ahp.adjust_hz43_position(otime)
#
#--- convert coordinates from cel to det
#
cmd = 'dmcoords ' + evt1 + ' opt=cel ra=' + str(
ra) + ' dec=' + str(dec)
cmd = cmd + ' verbose=1 > ' + zspace
os.system(cmd)
#
#--- extract det coordindats
#
info = mcf.read_data_file(zspace, remove=1)
out = get_detp_coord(info)
if out == False:
continue
detx = out[0]
dety = out[1]
zout = str(obsid) + '\t' + str(ra) + '\t' + str(dec) + '\t'
zout = zout + str(detx) + '\t' + str(dety) + '\n'
zline = zline + zout
#
#--- upate the data table
#
fchk = ehs.extract_hz43_stat(obsid, evt1)
if fchk == False:
continue
if zline != "":
ofile = house_keeping + olist[k]
with open(ofile, 'a') as fo:
fo.write(zline)
#
#--- reorder data file in time order
#
order_by_time()
#
#--- clean up coord lists
#
coords_list(olist[0])
coords_list(olist[1])
def create_data_tables(table):
"""
read input table of observations and create data tables
input: table --- a file which ontains a list of observations
either obsid list or a list in which the first column is obsid
output: <data_dir>/hrc_<det/pos>_results
"""
data = mcf.read_data_file(table)
for ent in data:
if mcf.is_neumeric(ent):
obsid = ent.strip()
else:
atemp = re.split('\s+', ent)
obsid = atemp[0]
if not mcf.is_neumeric(obsid):
continue
print(obsid)
fits = hcf.run_arc5gl(0,
0,
obsid=str(obsid),
level='1',
filetype='evt1')
out = extract_count_stats(fits)
if out[-1] < 0:
continue
line = str(obsid) + '\t'
if float(obsid) < 1000:
line = line + '\t'
line = line + str(fits) + '\t'
line = line + out[7] + '\t'
line = line + '%2.1f' % round(out[6], 1) + '\t'
line = line + '%2.2f' % round(out[5], 2) + '\t'
line = line + '%2.2f' % round(out[8], 2) + '\t'
line = line + '%2.4f' % round(out[9], 4) + '\n'
if out[-1] == 10:
outfile = data_dir + 'hrc_i_results'
if out[-1] == 0:
outfile = data_dir + 'hrc_s_0_results'
if out[-1] == 1:
outfile = data_dir + 'hrc_s_10_results'
if out[-1] == 2:
outfile = data_dir + 'hrc_s_25_results'
if out[-1] == 3:
outfile = data_dir + 'hrc_s_m10_results'
if out[-1] == 4:
outfile = data_dir + 'hrc_s_m25_results'
with open(outfile, 'a') as fo:
fo.write(line)
cmd = 'rm -f ' + fits
os.system(cmd)
