def get_dtf(obsid, fits): """ create a list of dtf for the given time periods input: obsid --- obsid of the observations fits --- event fits file name output: dsave --- a list of dtf values correspoinding to the event fits data """ # #--- get fits time list # t = pyfits.open(fits) tdata = t[1].data t.close() ftime = tdata.field('time') # #--- get dtf time list and dtf rate list # dtf = hcf.run_arc5gl(0, 0, obsid=obsid, level='1', filetype='dtf') t = pyfits.open(dtf) tdata = t[1].data t.close() dtime = tdata.field('time') drate = tdata.field('dtf') # #--- compare them and find dtf correspond to the time of fits data # dlen = len(dtime) dsave = [] for ent in ftime: for k in range(0, dlen - 1): if ent >= dtime[k] and ent < dtime[k + 1]: dsave.append(drate[k]) break mcf.rm_files(dtf) return dsave
def get_dead_period(obsid): """ extract time periods which is dtf <= 0.98 input: obsid --- obsid output: pstart --- starting time pstop --- stoppping time these are +/- 2 of the center value listed on dtf fits file """ # #--- get hrc dtf fits file # dtf = hcf.run_arc5gl(start='', stop='', obsid=obsid, operation='retrieve',\ level ='1', filetype='dtf') # #--- read out time and dtf column from the fits file # tdata = hcf.read_fits_file(dtf) tout = hcf.extract_col_data(tdata[1], ['time', 'dtf']) mcf.rm_files(dtf) # #--- find the period which match the condition (dtf<=0.98) # pstart = [] pstop = [] for k in range(0, len(tout)): try: cdtf = float(tout[k][1]) except: continue if cdtf <= 0.98: time = int(float(tout[k][0])) # #--- take the period to be +/- 2 of the center value listed # pstart.append(time - 2) pstop.append(time + 2) return [pstart, pstop]
def get_dead_period(obsid): """ extract time periods which is dtf <= 0.98 input: obsid --- obsid output: pstart --- starting time pstop --- stoppping time these are +/- 2 of the center value listed on dtf fits file """ # #--- get hrc dtf fits file # dtf = hcf.run_arc5gl(start='', stop='', obsid=obsid, operation='retrieve',\ level ='1', filetype='dtf') # #--- read out time and dtf column from the fits file # t = pyfits.open(dtf) fdata = t[1].data tdata = fdata['time'] ddata = fdata['dtf'] t.close() mcf.rm_files(dtf) # #--- find the period which match the condition (dtf<=0.98) # pstart = [] pstop = [] for k in range(0, len(tdata)): if float(ddata[k]) <= 0.98: xtime = int(float(tdata[k])) # #--- take the period to be +/- 2 of the center value listed # pstart.append(xtime - 2) pstop.append(xtime + 2) return [pstart, pstop]
def extract_data(obsid, inst): """ extract needed information for a given observation input: obsid --- obsid output: a list of lists: [inst, date, start, crsv, crsu, amp_sf, pha, pi, sumamps, samp, chip_id, tg_m, tg_mlam, tg_srcid, tg_part, tg_smap] note: inst,date, start are strings, but all others are lists. """ # #--- extract evt1 data # evt1 = hcf.run_arc5gl(0, 0, obsid = obsid, operation='retrieve', level ='1', filetype='evt1') if evt1 == "": return False # #--- extract evt1.5 data # # evta = hcf.run_arc5gl(0, 0, obsid = obsid, operation='retrieve', level ='1.5', filetype='tgevt1') # if evta == "": # return False # #--- find which instrument # hdr = pyfits.getheader(evt1, 1) inst = hdr['detnam'] date = hdr['date-obs'] start = hdr['tstart'] stop = hdr['tstop'] expo = hdr['exposure'] ra_nom = float(hdr['ra_nom']) dec_nom = float(hdr['dec_nom']) roll_nom = float(hdr['roll_nom']) ra_targ = float(hdr['ra_targ']) dec_targ = float(hdr['dec_targ']) [ra_nom2, dec_nom2] = convert_to_coordinates(ra_nom, dec_nom, evt1, 'sky') [ra_targ2, dec_targ2] = convert_to_coordinates(ra_targ, dec_targ, evt1, 'sky') [ra_nom3, dec_nom3] = convert_to_coordinates(ra_nom, dec_nom, evt1, 'det') [ra_targ3, dec_targ3] = convert_to_coordinates(ra_targ, dec_targ, evt1, 'det') hdata = [inst, date, start, stop, expo, ra_nom2, dec_nom2, ra_targ2, dec_targ2] hdata2 = [ra_nom, dec_nom, roll_nom, ra_targ, dec_targ] hdata3 = [ra_nom3, dec_nom3, ra_targ3, dec_targ3] [yoffset, zoffset] = extract_offset_info(obsid, inst) [yoffset, zoffset] = compute_y_z_offset(ra_nom, dec_nom, roll_nom, ra_targ, dec_targ) # #--- combine the data # # comb_evt1 = 'comb_evt.fits' # col_list = ['amp_sf', 'sumamps'] # add_coldata_to_fits(evta, evt1, col_list, comb_evt1) # #--- clean the data # clean1 = 'evt1_clean.fits' try: fef.filter_evt_file(evt1, clean1, obsid) except: return False # #--- get the target area # carea = 'clean_targ.fits' barea = 'clean_targ_bkg.fits' chk = get_target_data(obsid, evt1, clean1, carea, barea, ra_targ, dec_targ, yoffset, zoffset, inst) if chk: hdata.append(chk[0]) hdata.append(chk[1]) tarea = chk[2] #--- center area in pix sarea = chk[3] #--- background area in pix cent_data = get_data(carea, ['x', 'y', 'pi','amp_sf', 'sumamps']) samp = compute_samp(cent_data[4], cent_data[3], inst) cent_data.append(samp) cent_bkg = get_data(barea, ['x', 'y', 'pi','amp_sf', 'sumamps']) samp = compute_samp(cent_bkg[4], cent_bkg[3], inst) cent_bkg.append(samp) else: hdata.append(0.0) hdata.append(0.0) cent_data = [] cent_bkg = [] tarea = 0 sarea = 0 hdata = hdata + hdata2 + hdata3 hdata.append(tarea) hdata.append(sarea) return [hdata, cent_data, cent_bkg]
def hrc_gain_fit_voigt(candidate_list): """ extract hrc evt2 file, find the brightest object and create pha distribution Input: candidate_list --- if it is obsid, use it as a input otherwise, a list of new candidates will be create based database Output: <header>_pha.dat --- pha distribution data <header>_gfit.png --- a plot of pha data fitting_results --- a table of fitted results """ # #--- keep the results in save_line # save_line = '' # #--- if an obsid is provided, analyize that, else get new obsids from databases # if len(candidate_list) > 0: for obsid in candidate_list: print(str(obsid)) #hfile = extract_hrc_evt2(obsid) hfile = hcf.run_arc5gl(0, 0, obsid, level='2', filetype='evt2') if hfile == 'na': sfile = house_keeping + 'skip_obsids' with open(sfile, 'a') as fo: fo.write(str(obsid) + '\n') continue # #--- get a file name header for the later use # temp = re.split('N', hfile) hname = temp[0] # #--- extract information from the fits file header # hrdr = pyfits.getheader(hfile, 1) # #--- current AR Lac position adjucted with proper motion # date = hrdr['date-obs'] [ra, dec] = aap.adjust_arlac_position(date) # #--- find the diffrence between real AR Lac position and nominal postion #--- so that we can determin how much area we should include # ra_diff = abs(ra - float(hrdr['ra_pnt'])) * 60.0 dec_diff = abs(dec - float(hrdr['dec_pnt'])) * 60.0 rad_diff = math.sqrt(ra_diff * ra_diff + dec_diff * dec_diff) if rad_diff < 10.0: fit_rad = 60.0 else: fit_rad = 200.0 # #--- find a location of the brightest object (assume it is AR Lac) in sky coordinates # try: [x, y] = find_center(hfile) except: sfile = house_keeping + 'skip_obsids' with open(sfile, 'a') as fo: fo.write(str(obsid) + '\n') continue # #--- extract pha values in the given area # pha = extract_pha(hfile, x, y, fit_rad) #--- create pha count distribution # pmax = max(pha) + 1 pha_bin = [x for x in range(0, pmax)] pha_hist = [0 for x in range(0, pmax)] for ent in pha: pha_hist[ent] += 1 # #--- print out the distirbution results # line = '' for i in range(0, pmax): line = line + str(pha_bin[i]) + '\t' + str(pha_hist[i]) + '\n' outfile = data_dir + hname + '_pha.dat' with open(outfile, 'w') as fo: fo.write(line) # #--- gzip file # cmd = 'gzip -f ' + outfile os.system(cmd) # #--- find median point # med = find_med(pha_hist) # #--- fit a voigt distribution on the data # try: [center, width, amp, alphaD, alphaL, I, a_back, b_back] \ = voigt.fit_voigt_profile(pha_bin, pha_hist, type='voigt', plot_title=hfile) except: [center, width, amp, alphaD, alphaL, I, a_back, b_back] \ = [0, 0, 0, 0, 0, 0, 0, 0] # #--- rename a plotting file # outfile = plot_dir + 'Indivisual_Plots/' + hname + '_gfit.png' cmd = 'mv out.png ' + outfile os.system(cmd) line = str(obsid) + '\t' + str(hrdr['date-obs']) + '\t' + str( hrdr['tstart']) + '\t' line = line + hrdr['detnam'] + '\t' + str(hrdr['ra_pnt']) + '\t' line = line + str(hrdr['dec_pnt']) + '\t\t' + str(round( ra_diff, 3)) + '\t' line = line + str(round(dec_diff, 3)) + '\t' + str( round(rad_diff, 3)) + '\t' line = line + str(med) + '\t\t' + str(round(center, 3)) + '\t' line = line + str(round(amp, 3)) + '\t' + str(round(width, 3)) + '\t' line = line + str(hrdr['roll_pnt']) + '\t' + str( hrdr['foc_len']) + '\t' line = line + str(hrdr['defocus']) + '\t' + str( hrdr['sim_x']) + '\t' line = line + str(hrdr['sim_y']) + '\t' + str(hrdr['sim_z']) + '\t' line = line + str(round(alphaD, 4)) + '\t' + str(round(alphaL, 4)) + '\t' line = line + str(round(center, 3)) + '\t' + str(round(I, 2)) + '\t' line = line + str(round(a_back, 2)) + '\t' + str(round(b_back, 2)) + '\n' save_line = save_line + line # #--- remove the evt2 file # mcf.rm_files(hfile) # #--- if there is any new data, print it out # if save_line != '': # #--- print out the fitting result # outfile = data_dir + 'fitting_results' copied_file = outfile + '~' cmd = 'cp ' + outfile + ' ' + copied_file os.system(cmd) with open(outfile, 'a') as fo: fo.write(save_line) return True else: return False
def create_hz43_center_location_list(): """ extract coordinates from manually process data list input: none but read from 'hrci_hz43_zero.tar, hrcs_hz43_zero.tar output: <house_keeping>/hrc_i_coords, <hosue_keeping>/hrc_s_coords both has <obsid> <ra> <dec> <skyx> <skyy> """ for k in range(0, 2): # #--- find out which obsids are already processed # cfile = house_keeping + olist[k] f = open(cfile, 'r') data = [line.strip() for line in f.readlines()] f.close() obs_list = [] for ent in data: atemp = re.split('\s+', ent) obs_list.append(atemp[0]) fo = open('./zdata', 'w') # #--- check which data are available # cmd = 'cp ' + tpath + dlist[k] + ' . ' os.system(cmd) cmd = 'tar xf ' + dlist[k] os.system(cmd) cmd = 'ls src*.reg > ' + zspace os.system(cmd) data = read_data(zspace, remove=1) chk = 0 for ent in data: mc = re.search('src_zero_', ent) if mc is not None: cut = 'src_zero_' else: cut = 'src' atemp = re.split(cut, ent) btemp = re.split('.reg', atemp[1]) obsid = btemp[0] # #--- check this obsid is in the saved data list # if obsid in obs_list: continue f = open(ent, 'r') out = f.read() f.close() atemp = re.split('\(', out) btemp = re.split(',', atemp[1]) ra = btemp[0] dec = btemp[1] # #--- extract evt1 data # evt1 = hcf.run_arc5gl(0, 0, obsid = obsid, operation='retrieve', level ='1', filetype='evt1') if evt1 == "": continue # #--- convert coordinates from cel to sky # cmd = 'dmcoords ' + evt1 + ' opt=cel ra=' + ra + ' dec=' + dec + ' verbose=1 > ' + zspace run_ciao(cmd) # #--- extract sky coordindats # info = read_data(zspace, remove=1) atemp = re.split('\s+', info[-2]) skyx = atemp[2] skyy = atemp[3] line = obsid + '\t' + ra + '\t' + dec + '\t' + skyx + '\t' + skyy + '\n' fo.write(line) chk = 1 mcf.rm_file(evt1) fo.close() if chk > 0: cmd = 'cat ./zdata >> ' + cfile os.system(cmd) cmd = 'rm -rf *.reg *.tar ./zdata' os.system(cmd)
def extract_data(obsid, evt1): """ extract needed information for a given observation input: obsid --- obsid evt1 --- hrc evt1 fits file name output: a list of lists: [inst, date, start, crsv, crsu, amp_sf, pha, pi, sumamps, samp, chip_id, tg_m, tg_mlam, tg_srcid, tg_part, tg_smap] note: inst,date, start are strings, but all others are lists. """ # #--- evt1 data --- already extracted # if evt1 == "": print("no EVT1 fits file") return False # #--- extract evt1.5 data # evta = hcf.run_arc5gl(0, 0, obsid=obsid, operation='retrieve', level='1.5', filetype='tgevt1') if evta == "": print("no EVT1.5 fits file") sfile = house_keeping + 'skip_obsids' with open(sfile, 'a') as fo: fo.write(str(obsid) + '\n') return False # #--- find which instrument # hdr = pyfits.getheader(evt1, 1) inst = hdr['detnam'] date = hdr['date-obs'] start = hdr['tstart'] stop = hdr['tstop'] expo = hdr['exposure'] hdata = [inst, date, start, stop, expo] # #--- we need to drop a large off axis data. set 9 arcmin # ra_pnt = float(hdr['ra_pnt']) dec_pnt = float(hdr['dec_pnt']) ra_targ = float(hdr['ra_targ']) dec_targ = float(hdr['dec_targ']) adiff = ra_targ - ra_pnt bdiff = dec_targ - dec_pnt gout = adiff * adiff + bdiff * bdiff adiff = math.sqrt(gout) if adiff > 0.15: print("Offset too large (>9 amin); dropped") return False # #--- combine the data # comb_evt1 = 'comb_evt.fits' col_list = ['amp_sf', 'sumamps'] add_coldata_to_fits(evta, evt1, col_list, comb_evt1) # #--- clean the data # clean1 = 'evt1_clean.fits' carea = 'clean_cent.fits' barea = 'clean_cent_bkg.fits' fef.filter_evt_file(comb_evt1, clean1, obsid) # #--- get the center part # chk = get_center_data(obsid, evt1, clean1, carea, barea) if chk: cent_data = get_data( carea, ['tg_m', 'tg_r', 'tg_d', 'pi', 'amp_sf', 'sumamps']) samp = compute_samp(cent_data[5], cent_data[4], inst) cent_data.append(samp) cent_bkg = get_data( barea, ['tg_m', 'tg_r', 'tg_d', 'pi', 'amp_sf', 'sumamps']) samp = compute_samp(cent_bkg[5], cent_bkg[4], inst) cent_bkg.append(samp) else: cent_data = [] cent_bkg = [] # #--- get the arm parts # carea = 'clean_arms.fits' barea = 'clean_arms_bkg.fits' get_arm_data(clean1, carea, barea) arms_data = get_data(carea, ['tg_m', 'tg_r', 'tg_d', 'pi', 'amp_sf', 'sumamps']) samp = compute_samp(arms_data[5], arms_data[4], inst) arms_data.append(samp) arms_bkg = get_data(barea, ['tg_m', 'tg_r', 'tg_d', 'pi', 'amp_sf', 'sumamps']) samp = compute_samp(arms_bkg[5], arms_bkg[4], inst) arms_bkg.append(samp) return [hdata, cent_data, cent_bkg, arms_data, arms_bkg]
def create_data_tables(table): """ read input table of observations and create data tables input: table --- a file which contains a list of observations either obsid list or a list in which the first column is obsid output: <data_dir>/hrc_<det/pos>_results """ data = mcf.read_data_file(table) for ent in data: if mcf.is_neumeric(ent): obsid = ent.strip() else: atemp = re.split('\s+', ent) obsid = atemp[0] if mcf.is_neumeric(obsid) == False: continue print(str(obsid)) fits = hcf.run_arc5gl(0, 0, obsid=obsid, level='2', filetype='evt2') if fits == False: write_on_skip_file(obsid) print("Data is not extracted") continue # #--- if there are multiple output, use only first one # if isinstance(fits, list): fits = fits[0] xxx = 999 #if xxx == 999: try: out = extract_count_stats(fits) #else: except: cmd = 'rm -f ' + fits + '*' os.system(cmd) write_on_skip_file(obsid) print("Analysis Failed") continue if out[-1] < 0: cmd = 'rm -f ' + fits + '*' os.system(cmd) write_on_skip_file(obsid) print("No Output") continue line = str(obsid) + '\t' if float(obsid) < 1000: line = line + '\t' line = line + str(fits) + '\t' line = line + out[7] + '\t' line = line + '%2.1f' % round(out[6], 1) + '\t' line = line + '%2.2f' % round(out[5], 2) + '\t' line = line + '%2.2f' % round(out[8], 2) + '\t' line = line + '%2.4f' % round(out[9], 4) + '\n' if out[-1] == 0: outfile = data_dir + 'hrc_s_0_results' if out[-1] == 1: outfile = data_dir + 'hrc_s_10_results' if out[-1] == 2: outfile = data_dir + 'hrc_s_25_results' if out[-1] == 3: outfile = data_dir + 'hrc_s_m10_results' if out[-1] == 4: outfile = data_dir + 'hrc_s_m25_results' if out[-1] == 10: outfile = data_dir + 'hrc_i_0_results' if out[-1] == 11: outfile = data_dir + 'hrc_i_10_results' if out[-1] == 12: outfile = data_dir + 'hrc_i_25_results' if out[-1] == 13: outfile = data_dir + 'hrc_i_m10_results' if out[-1] == 14: outfile = data_dir + 'hrc_i_m25_results' with open(outfile, 'a') as fo: fo.write(line) cmd = 'rm -f *fits*' os.system(cmd)
def analyze_hz43_data(obsid='', inst='i'): """ extract data, analyze and update data tables input: none but download data with arc5gl output: <house_keeping>/hrc_i_coords, <hosue_keeping>/hrc_s_coords both has <obsid> <ra> <dec> <detx> <dety> <data_dir>/pi_* and samp_* """ # #--- find new HZ43 observations data_set: [hrc-i, hrc-s] # if obsid == '': data_set = fhd.extract_calb_hz43() else: if inst == 'i': data_set = [[ obsid, ], []] else: data_set = [[], [ obsid, ]] if (len(data_set[0]) == 0) and (len(data_set[1]) == 0): print("No new HZ43 data found") exit(1) for k in range(0, 2): data_list = data_set[k] if len(data_list) == 0: continue if k == 0: print("Analyzing HRC I") else: print("Analyzing HRC S") zline = '' for obsid in data_list: print("OBSID: " + str(obsid)) # #--- extract evt1 data # evt1 = hcf.run_arc5gl(0, 0, obsid = obsid, operation='retrieve',\ level ='1', filetype='evt1') if evt1 == "": print("No evt1 file extracted") sfile = house_keeping + 'skip_obsids' with open(sfile, 'a') as fo: fo.write(str(obsid) + '\n') continue # #--- get the ra/dec of HZ43 adjusted to proper motion # fout = pyfits.open(evt1) fhead = fout[1].header otime = fhead['DATE-OBS'] [ra, dec] = ahp.adjust_hz43_position(otime) # #--- convert coordinates from cel to det # cmd = 'dmcoords ' + evt1 + ' opt=cel ra=' + str( ra) + ' dec=' + str(dec) cmd = cmd + ' verbose=1 > ' + zspace os.system(cmd) # #--- extract det coordindats # info = mcf.read_data_file(zspace, remove=1) out = get_detp_coord(info) if out == False: continue detx = out[0] dety = out[1] zout = str(obsid) + '\t' + str(ra) + '\t' + str(dec) + '\t' zout = zout + str(detx) + '\t' + str(dety) + '\n' zline = zline + zout # #--- upate the data table # fchk = ehs.extract_hz43_stat(obsid, evt1) if fchk == False: continue if zline != "": ofile = house_keeping + olist[k] with open(ofile, 'a') as fo: fo.write(zline) # #--- reorder data file in time order # order_by_time() # #--- clean up coord lists # coords_list(olist[0]) coords_list(olist[1])
def create_data_tables(table): """ read input table of observations and create data tables input: table --- a file which ontains a list of observations either obsid list or a list in which the first column is obsid output: <data_dir>/hrc_<det/pos>_results """ data = mcf.read_data_file(table) for ent in data: if mcf.is_neumeric(ent): obsid = ent.strip() else: atemp = re.split('\s+', ent) obsid = atemp[0] if not mcf.is_neumeric(obsid): continue print(obsid) fits = hcf.run_arc5gl(0, 0, obsid=str(obsid), level='1', filetype='evt1') out = extract_count_stats(fits) if out[-1] < 0: continue line = str(obsid) + '\t' if float(obsid) < 1000: line = line + '\t' line = line + str(fits) + '\t' line = line + out[7] + '\t' line = line + '%2.1f' % round(out[6], 1) + '\t' line = line + '%2.2f' % round(out[5], 2) + '\t' line = line + '%2.2f' % round(out[8], 2) + '\t' line = line + '%2.4f' % round(out[9], 4) + '\n' if out[-1] == 10: outfile = data_dir + 'hrc_i_results' if out[-1] == 0: outfile = data_dir + 'hrc_s_0_results' if out[-1] == 1: outfile = data_dir + 'hrc_s_10_results' if out[-1] == 2: outfile = data_dir + 'hrc_s_25_results' if out[-1] == 3: outfile = data_dir + 'hrc_s_m10_results' if out[-1] == 4: outfile = data_dir + 'hrc_s_m25_results' with open(outfile, 'a') as fo: fo.write(line) cmd = 'rm -f ' + fits os.system(cmd)