def main(): parser = optparse.OptionParser(prog="sp_pipeline..py", \ version=" Chitrang Patel (May. 12, 2015)", \ usage="%prog INFILE(PsrFits FILE, SINGLEPULSE FILES)", \ description="Create single pulse plots to show the " \ "frequency sweeps of a single pulse, " \ "DM vs time, and SNR vs DM,"\ "in psrFits data.") parser.add_option('--infile', dest='infile', type='string', \ help="Give a .inf file to read the appropriate header information.") parser.add_option('--groupsfile', dest='txtfile', type='string', \ help="Give the groups.txt file to read in the groups information.") parser.add_option('--mask', dest='maskfile', type='string', \ help="Mask file produced by rfifind. (Default: No Mask).", \ default=None) options, args = parser.parse_args() if not hasattr(options, 'infile'): raise ValueError("A .inf file must be given on the command line! ") if not hasattr(options, 'txtfile'): raise ValueError( "The groups.txt file must be given on the command line! ") files = get_textfile(options.txtfile) print_debug("Begining waterfaller... " + strftime("%Y-%m-%d %H:%M:%S")) Detrendlen = 50 if not args[0].endswith("fits"): raise ValueError("The first file must be a psrFits file! ") basename = args[0][:-5] filetype = "psrfits" inffile = options.infile topo, bary = bary_and_topo.bary_to_topo(inffile) time_shift = bary - topo inf = infodata.infodata(inffile) RA = inf.RA dec = inf.DEC MJD = inf.epoch mjd = Popen(["mjd2cal", "%f" % MJD], stdout=PIPE, stderr=PIPE) date, err = mjd.communicate() date = date.split()[2:5] telescope = inf.telescope N = inf.N Total_observed_time = inf.dt * N print_debug('getting file..') rawdatafile = psrfits.PsrfitsFile(args[0]) print "rawdatafile", memory.resident() / (1024.0**3) bin_shift = np.round(time_shift / rawdatafile.tsamp).astype('int') for group in [6, 5, 4, 3, 2]: rank = group + 1 if files[group] != "Number of rank %i groups: 0 " % rank: print_debug(files[group]) values = split_parameters(rank, options.txtfile) lis = np.where(files == '\tRank: %i.000000' % rank)[0] for ii in range(len(values)): #### Array for Plotting DM vs SNR print "DM, S/N", memory.resident() / (1024.0**3) print_debug("Making arrays for DM vs Signal to Noise...") temp_list = files[lis[ii] - 6].split() npulses = int(temp_list[2]) temp_lines = files[(lis[ii] + 3):(lis[ii] + npulses + 1)] arr = np.split(temp_lines, len(temp_lines)) dm_list = [] time_list = [] for i in range(len(arr)): dm_val = float(arr[i][0].split()[0]) time_val = float(arr[i][0].split()[2]) dm_list.append(dm_val) time_list.append(time_val) arr_2 = np.array([arr[i][0].split() for i in range(len(arr))], dtype=np.float32) dm_arr = np.array([arr_2[i][0] for i in range(len(arr))], dtype=np.float32) sigma_arr = np.array([arr_2[i][1] for i in range(len(arr))], dtype=np.float32) print "After DM, S/N", memory.resident() / (1024.0**3) #### Array for Plotting DM vs Time is in show_spplots.plot(...) #### Setting variables up for the waterfall arrays. j = ii + 1 subdm = dm = sweep_dm = values[ii][0] integrate_dm = None sigma = values[ii][1] sweep_posn = 0.0 topo_start_time = values[ii][2] - topo_timeshift( values[ii][2], time_shift, topo)[0] sample_number = values[ii][3] width_bins = values[ii][4] binratio = 50 scaleindep = False zerodm = None downsamp = np.round((values[ii][2] / sample_number / 6.54761904761905e-05)).astype('int') duration = binratio * width_bins * rawdatafile.tsamp * downsamp start = topo_start_time - (0.25 * duration) if (start < 0.0): start = 0.0 pulse_width = width_bins * downsamp * 6.54761904761905e-05 if sigma <= 10: nsub = 32 elif sigma >= 10 and sigma < 15: nsub = 64 else: nsub = 96 nbins = np.round(duration / rawdatafile.tsamp).astype('int') start_bin = np.round(start / rawdatafile.tsamp).astype('int') dmfac = 4.15e3 * np.abs(1. / rawdatafile.frequencies[0]**2 - 1. / rawdatafile.frequencies[-1]**2) nbinsextra = np.round( (duration + dmfac * dm) / rawdatafile.tsamp).astype('int') if (start_bin + nbinsextra) > N - 1: nbinsextra = N - 1 - start_bin data = rawdatafile.get_spectra(start_bin, nbinsextra) print "After rawdata", memory.resident() / (1024.0**3) data = maskdata(data, start_bin, nbinsextra, options.maskfile) #make an array to store header information for the .npz files temp_filename = basename + "_DM%.1f_%.1fs_rank_%i" % ( subdm, topo_start_time, rank) # Array for Plotting Dedispersed waterfall plot - zerodm - OFF print_debug("Running waterfaller with Zero-DM OFF...") data, Data_dedisp_nozerodm = waterfall_array( start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) print "waterfall", memory.resident() / (1024.0**3) # Add additional information to the header information array text_array = np.array([ args[0], 'Arecibo', RA, dec, MJD, rank, nsub, nbins, subdm, sigma, sample_number, duration, width_bins, pulse_width, rawdatafile.tsamp, Total_observed_time, topo_start_time, data.starttime, data.dt, data.numspectra, data.freqs.min(), data.freqs.max() ]) #### Array for plotting Dedispersed waterfall plot zerodm - ON print_debug("Running Waterfaller with Zero-DM ON...") data = rawdatafile.get_spectra(start_bin, nbinsextra) data = maskdata(data, start_bin, nbinsextra, options.maskfile) zerodm = True data, Data_dedisp_zerodm = waterfall_array( start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) print "waterfall", memory.resident() / (1024.0**3) ####Sweeped without zerodm print_debug("Running waterfaller for sweeped arrays.") start = start + (0.25 * duration) start_bin = np.round(start / rawdatafile.tsamp).astype('int') sweep_duration = 4.15e3 * np.abs( 1. / rawdatafile.frequencies[0]**2 - 1. / rawdatafile.frequencies[-1]**2) * sweep_dm nbins = np.round(sweep_duration / (rawdatafile.tsamp)).astype('int') if ((nbins + start_bin) > (N - 1)): nbins = N - 1 - start_bin data = rawdatafile.get_spectra(start_bin, nbins) data = maskdata(data, start_bin, nbins, options.maskfile) zerodm = None dm = None data, Data_nozerodm = waterfall_array( start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) print "waterfall", memory.resident() / (1024.0**3) text_array = np.append(text_array, sweep_duration) text_array = np.append(text_array, data.starttime) # Array to Construct the sweep if sweep_dm is not None: ddm = sweep_dm - data.dm delays = psr_utils.delay_from_DM(ddm, data.freqs) delays -= delays.min() delays_nozerodm = delays freqs_nozerodm = data.freqs # Sweeped with zerodm-on zerodm = True downsamp_temp = 1 data, Data_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp_temp, scaleindep, width_bins, rawdatafile, binratio, data) print "waterfall", memory.resident() / (1024.0**3) # Saving the arrays into the .spd file. with open(temp_filename + ".spd", 'wb') as f: np.savez_compressed( f, Data_dedisp_nozerodm=Data_dedisp_nozerodm.astype( np.float16), Data_dedisp_zerodm=Data_dedisp_zerodm.astype( np.float16), Data_nozerodm=Data_nozerodm.astype(np.float16), delays_nozerodm=delays_nozerodm, freqs_nozerodm=freqs_nozerodm, Data_zerodm=Data_zerodm.astype(np.float16), dm_arr=map(np.float16, dm_arr), sigma_arr=map(np.float16, sigma_arr), dm_list=map(np.float16, dm_list), time_list=map(np.float16, time_list), text_array=text_array) print_debug("Now plotting...") print "Before plot..", memory.resident() / (1024.0**3) show_spplots.plot(temp_filename + ".spd", args[1:], xwin=False, outfile=basename, tar=None) print "After plot..", memory.resident() / (1024.0**3) print_debug("Finished plot %i " % j + strftime("%Y-%m-%d %H:%M:%S")) print_debug("Finished group %i... " % rank + strftime("%Y-%m-%d %H:%M:%S")) print_debug("Finished running waterfaller... " + strftime("%Y-%m-%d %H:%M:%S"))
def main(): parser = optparse.OptionParser(prog="sp_pipeline..py", \ version=" Chitrang Patel (May. 12, 2015)", \ usage="%prog INFILE(PsrFits FILE, SINGLEPULSE FILES)", \ description="Create single pulse plots to show the " \ "frequency sweeps of a single pulse, " \ "DM vs time, and SNR vs DM,"\ "in psrFits data.") parser.add_option('--infile', dest='infile', type='string', \ help="Give a .inf file to read the appropriate header information.") parser.add_option('--groupsfile', dest='txtfile', type='string', \ help="Give the groups.txt file to read in the groups information.") parser.add_option('--mask', dest='maskfile', type='string', \ help="Mask file produced by rfifind. (Default: No Mask).", \ default=None) parser.add_option('-n', dest='maxnumcands', type='int', \ help="Maximum number of candidates to plot. (Default: 100).", \ default=100) options, args = parser.parse_args() if not hasattr(options, 'infile'): raise ValueError("A .inf file must be given on the command line! ") if not hasattr(options, 'txtfile'): raise ValueError("The groups.txt file must be given on the command line! ") files = sp_utils.spio.get_textfile(options.txtfile) print_debug("Begining waterfaller... "+strftime("%Y-%m-%d %H:%M:%S")) if not args[0].endswith("fits"): raise ValueError("The first file must be a psrFits file! ") print_debug('Maximum number of candidates to plot: %i'%options.maxnumcands) basename = args[0][:-5] filetype = "psrfits" inffile = options.infile topo, bary = bary_and_topo.bary_to_topo(inffile) time_shift = bary-topo inf = infodata.infodata(inffile) RA = inf.RA dec = inf.DEC MJD = inf.epoch mjd = Popen(["mjd2cal", "%f"%MJD], stdout=PIPE, stderr=PIPE) date, err = mjd.communicate() date = date.split()[2:5] telescope = inf.telescope N = inf.N Total_observed_time = inf.dt *N print_debug('getting file..') rawdatafile = psrfits.PsrfitsFile(args[0]) bin_shift = np.round(time_shift/rawdatafile.tsamp).astype('int') numcands = 0 # candidate counter. Use this to decide the maximum bumber of candidates to plot. loop_must_break = False # dont break the loop unless num of cands >100. for group in [6, 5, 4, 3, 2]: rank = group+1 if files[group] != "Number of rank %i groups: 0 "%rank: print_debug(files[group]) values = sp_utils.spio.split_parameters(rank, options.txtfile) lis = np.where(files == '\tRank: %i.000000'%rank)[0] for ii in range(len(values)): #### Array for Plotting DM vs SNR print_debug("Making arrays for DM vs Signal to Noise...") temp_list = files[lis[ii]-6].split() npulses = int(temp_list[2]) temp_lines = files[(lis[ii]+3):(lis[ii]+npulses+1)] arr = np.split(temp_lines, len(temp_lines)) dm_list = [] time_list = [] for i in range(len(arr)): dm_val= float(arr[i][0].split()[0]) time_val = float(arr[i][0].split()[2]) dm_list.append(dm_val) time_list.append(time_val) arr_2 = np.array([arr[i][0].split() for i in range(len(arr))], dtype = np.float32) dm_arr = np.array([arr_2[i][0] for i in range(len(arr))], dtype = np.float32) sigma_arr = np.array([arr_2[i][1] for i in range(len(arr))], dtype = np.float32) #### Array for Plotting DM vs Time is in show_spplots.plot(...) #### Setting variables up for the waterfall arrays. j = ii+1 subdm = dm = sweep_dm= values[ii][0] integrate_dm = None sigma = values[ii][1] sweep_posn = 0.0 bary_start_time = values[ii][2] topo_start_time = bary_start_time - topo_timeshift(bary_start_time, time_shift, topo)[0] sample_number = values[ii][3] width_bins = values[ii][4] binratio = 50 scaleindep = False zerodm = None downsamp = np.round((values[ii][2]/sample_number/rawdatafile.tsamp)).astype('int') duration = binratio * width_bins * rawdatafile.tsamp * downsamp start = topo_start_time - (0.25 * duration) if (start<0.0): start = 0.0 pulse_width = width_bins*downsamp*rawdatafile.tsamp if sigma < 10: nsub = 32 elif sigma >= 10 and sigma < 15: nsub = 64 else: nsub = 96 if nsub > inf.numchan: nsub = inf.numchan nbins = np.round(duration/rawdatafile.tsamp).astype('int') start_bin = np.round(start/rawdatafile.tsamp).astype('int') dmfac = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2 - 1./rawdatafile.frequencies[-1]**2) nbinsextra = np.round((duration + dmfac * dm)/rawdatafile.tsamp).astype('int') if (start_bin+nbinsextra) > N-1: nbinsextra = N-1-start_bin data = rawdatafile.get_spectra(start_bin, nbinsextra) data = maskdata(data, start_bin, nbinsextra, options.maskfile) #make an array to store header information for the .npz files temp_filename = basename+"_DM%.1f_%.1fs_rank_%i"%(subdm, topo_start_time, rank) # Array for Plotting Dedispersed waterfall plot - zerodm - OFF print_debug("Running waterfaller with Zero-DM OFF...") data, Data_dedisp_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) # Add additional information to the header information array text_array = np.array([args[0], 'Arecibo', RA, dec, MJD, rank, nsub, nbins, subdm, sigma, sample_number, duration, width_bins, pulse_width, rawdatafile.tsamp, Total_observed_time, topo_start_time, data.starttime, data.dt, data.numspectra, data.freqs.min(), data.freqs.max()]) #### Array for plotting Dedispersed waterfall plot zerodm - ON print_debug("Running Waterfaller with Zero-DM ON...") data = rawdatafile.get_spectra(start_bin, nbinsextra) data = maskdata(data, start_bin, nbinsextra, options.maskfile) zerodm = True data, Data_dedisp_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) ####Sweeped without zerodm start = start + (0.25*duration) start_bin = np.round(start/rawdatafile.tsamp).astype('int') sweep_duration = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2-1./rawdatafile.frequencies[-1]**2)*sweep_dm nbins = np.round(sweep_duration/(rawdatafile.tsamp)).astype('int') if ((nbins+start_bin)> (N-1)): nbins = N-1-start_bin data = rawdatafile.get_spectra(start_bin, nbins) data = maskdata(data, start_bin, nbins, options.maskfile) zerodm = None dm = None data, Data_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) text_array = np.append(text_array, sweep_duration) text_array = np.append(text_array, data.starttime) text_array = np.append(text_array, bary_start_time) # Array to Construct the sweep if sweep_dm is not None: ddm = sweep_dm-data.dm delays = psr_utils.delay_from_DM(ddm, data.freqs) delays -= delays.min() delays_nozerodm = delays freqs_nozerodm = data.freqs # Sweeped with zerodm-on zerodm = True downsamp_temp = 1 data, Data_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp_temp, scaleindep, width_bins, rawdatafile, binratio, data) # Saving the arrays into the .spd file. with open(temp_filename+".spd", 'wb') as f: np.savez_compressed(f, Data_dedisp_nozerodm = Data_dedisp_nozerodm.astype(np.float16), Data_dedisp_zerodm = Data_dedisp_zerodm.astype(np.float16), Data_nozerodm = Data_nozerodm.astype(np.float16), delays_nozerodm = delays_nozerodm, freqs_nozerodm = freqs_nozerodm, Data_zerodm = Data_zerodm.astype(np.float16), dm_arr= map(np.float16, dm_arr), sigma_arr = map(np.float16, sigma_arr), dm_list= map(np.float16, dm_list), time_list = map(np.float16, time_list), text_array = text_array) print_debug("Now plotting...") show_spplots.plot(temp_filename+".spd", args[1:], xwin=False, outfile = basename, tar = None) print_debug("Finished plot %i " %j+strftime("%Y-%m-%d %H:%M:%S")) numcands+= 1 print_debug('Finished sp_candidate : %i'%numcands) if numcands >= options.maxnumcands: # Max number of candidates to plot 100. loop_must_break = True break if loop_must_break: break print_debug("Finished group %i... "%rank+strftime("%Y-%m-%d %H:%M:%S")) print_debug("Finished running waterfaller... "+strftime("%Y-%m-%d %H:%M:%S"))
def main(): parser = optparse.OptionParser(prog="sp_pipeline..py", \ version=" Chitrang Patel (May. 12, 2015)", \ usage="%prog INFILE(PsrFits FILE, SINGLEPULSE FILES)", \ description="Create single pulse plots to show the " \ "frequency sweeps of a single pulse, " \ "DM vs time, and SNR vs DM,"\ "in psrFits data.") parser.add_option('--infile', dest='infile', type='string', \ help="Give a .inf file to read the appropriate header information.") parser.add_option('--groupsfile', dest='txtfile', type='string', \ help="Give the groups.txt file to read in the groups information.") parser.add_option('--mask', dest='maskfile', type='string', \ help="Mask file produced by rfifind. (Default: No Mask).", \ default=None) parser.add_option('-n', dest='maxnumcands', type='int', \ help="Maximum number of candidates to plot. (Default: 100).", \ default=100) options, args = parser.parse_args() if not hasattr(options, 'infile'): raise ValueError("A .inf file must be given on the command line! ") if not hasattr(options, 'txtfile'): raise ValueError("The groups.txt file must be given on the command line! ") files = get_textfile(options.txtfile) print_debug("Begining waterfaller... "+strftime("%Y-%m-%d %H:%M:%S")) if not args[0].endswith("fits"): raise ValueError("The first file must be a psrFits file! ") print_debug('Maximum number of candidates to plot: %i'%options.maxnumcands) basename = args[0][:-5] filetype = "psrfits" inffile = options.infile topo, bary = bary_and_topo.bary_to_topo(inffile) time_shift = bary-topo inf = infodata.infodata(inffile) RA = inf.RA dec = inf.DEC MJD = inf.epoch mjd = Popen(["mjd2cal", "%f"%MJD], stdout=PIPE, stderr=PIPE) date, err = mjd.communicate() date = date.split()[2:5] telescope = inf.telescope N = inf.N numcands=0 Total_observed_time = inf.dt *N print_debug('getting file..') values = split_parameters(options.txtfile) if len(values)> options.maxnumcands: values=sorted(values, key=itemgetter(5,1)) #sorting candidates based on ranks and snr values=values[-options.maxnumcands:] print "More than", options.maxnumcands, "candidates, making plots for", options.maxnumcands, "candidates" values = sorted(values, key=itemgetter(0)) for ii in range(len(values)): #### Array for Plotting DM vs SNR print_debug("Making arrays for DM vs Signal to Noise...") temp_list = files[values[ii][6]-6].split() npulses = int(temp_list[2]) temp_lines = files[(values[ii][6]+3):(values[ii][6]+npulses+1)] arr = np.split(temp_lines, len(temp_lines)) dm_list = [] time_list = [] for i in range(len(arr)): dm_val= float(arr[i][0].split()[0]) time_val = float(arr[i][0].split()[2]) dm_list.append(dm_val) time_list.append(time_val) arr_2 = np.array([arr[i][0].split() for i in range(len(arr))], dtype = np.float32) dm_arr = np.array([arr_2[i][0] for i in range(len(arr))], dtype = np.float32) sigma_arr = np.array([arr_2[i][1] for i in range(len(arr))], dtype = np.float32) #### Array for Plotting DM vs Time is in show_spplots.plot(...) #### Setting variables up for the waterfall arrays. j = ii+1 subdm = dm = sweep_dm= values[ii][0] sample_number = values[ii][3] rank=values[ii][5] width_bins = values[ii][4] #print "dm", dm #print "width_bins", width_bins downsamp = np.round((values[ii][2]/sample_number/inf.dt)).astype('int') #print "downsamp", downsamp pulse_width = width_bins*downsamp*inf.dt #print "pulse_width", pulse_width if ii == 0: mask_subband=rfifind.rfifind("%s_rfifind.mask"%(basename)) mask_subband.set_zap_chans(power=1000,plot=False) mask_subband.set_weights_and_offsets() mask_subband.write_weights(filename="%s_weights.txt"%(basename)) cmd="psrfits_subband -dm %.2f -nsub 128 -o %s_subband_%.2f -weights %s_weights.txt %s"%(dm,basename,dm,basename,args[0]) call(cmd, shell=True) #subband args[0] at dm and then generate a file that will be set equal to rawdatafile subband_file="%s_subband_%.2f_0001.fits" %(basename,dm) dm_prev=dm subband_prev= subband_file else: dm_diff=dm-dm_prev t_smear=8.3e3*dm_diff*(350**-3)*(np.abs(rawdatafile.frequencies[0]-rawdatafile.frequencies[-1])/128.) if (5*t_smear) > pulse_width: cmd="psrfits_subband -dm %.2f -nsub 128 -o %s_subband_%.2f -weights %s_weights.txt %s"%(dm,basename,dm,basename,args[0]) call(cmd, shell=True) #subband args[0] at dm and then generate a file that will be set equal to rawdatafile subband_file="%s_subband_%.2f_0001.fits" %(basename,dm) dm_prev=dm subband_prev=subband_file rawdatafile = psrfits.PsrfitsFile(subband_file) bin_shift = np.round(time_shift/rawdatafile.tsamp).astype('int') integrate_dm = None sigma = values[ii][1] sweep_posn = 0.0 bary_start_time = values[ii][2] topo_start_time = bary_start_time - topo_timeshift(bary_start_time, time_shift, topo)[0] binratio = 50 scaleindep = False zerodm = None duration = binratio * width_bins * rawdatafile.tsamp * downsamp start = topo_start_time - (0.25 * duration) if (start<0.0): start = 0.0 if sigma <= 7: nsub = 32 elif sigma >= 7 and sigma < 10: nsub = 64 else: nsub = 128 nbins = np.round(duration/rawdatafile.tsamp).astype('int') start_bin = np.round(start/rawdatafile.tsamp).astype('int') dmfac = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2 - 1./rawdatafile.frequencies[-1]**2) nbinsextra = np.round((duration + dmfac * dm)/rawdatafile.tsamp).astype('int') if (start_bin+nbinsextra) > N-1: nbinsextra = N-1-start_bin data = rawdatafile.get_spectra(start_bin, nbinsextra) data = maskdata(data, start_bin, nbinsextra, options.maskfile) #make an array to store header information for the .npz files temp_filename = basename+"_DM%.1f_%.1fs_rank_%i"%(subdm, topo_start_time, rank) # Array for Plotting Dedispersed waterfall plot - zerodm - OFF print_debug("Running waterfaller with Zero-DM OFF...") data, Data_dedisp_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) #Add additional information to the header information array text_array = np.array([subband_file, 'GBT', RA, dec, MJD, rank, nsub, nbins, subdm, sigma, sample_number, duration, width_bins, pulse_width, rawdatafile.tsamp, Total_observed_time, topo_start_time, data.starttime, data.dt, data.numspectra, data.freqs.min(), data.freqs.max()]) #### Array for plotting Dedispersed waterfall plot zerodm - ON print_debug("Running Waterfaller with Zero-DM ON...") #print "before get_spectra",memory.resident()/(1024.0**3) data = rawdatafile.get_spectra(start_bin, nbinsextra) #print "after get_spectra",memory.resident()/(1024.0**3) data = maskdata(data, start_bin, nbinsextra, options.maskfile) zerodm = True data, Data_dedisp_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) #print "waterfall",memory.resident()/(1024.0**3) ####Sweeped without zerodm start = start + (0.25*duration) start_bin = np.round(start/rawdatafile.tsamp).astype('int') sweep_duration = 4.15e3 * np.abs(1./rawdatafile.frequencies[0]**2-1./rawdatafile.frequencies[-1]**2)*sweep_dm nbins = np.round(sweep_duration/(rawdatafile.tsamp)).astype('int') if ((nbins+start_bin)> (N-1)): nbins = N-1-start_bin #print "before get_spectra",memory.resident()/(1024.0**3) data = rawdatafile.get_spectra(start_bin, nbins) #print "after get_spectra",memory.resident()/(1024.0**3) data = maskdata(data, start_bin, nbins, options.maskfile) zerodm = None dm = None data, Data_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) #print "waterfall",memory.resident()/(1024.0**3) text_array = np.append(text_array, sweep_duration) text_array = np.append(text_array, data.starttime) text_array = np.append(text_array, bary_start_time) # Array to Construct the sweep if sweep_dm is not None: ddm = sweep_dm-data.dm delays = psr_utils.delay_from_DM(ddm, data.freqs) delays -= delays.min() delays_nozerodm = delays freqs_nozerodm = data.freqs # Sweeped with zerodm-on zerodm = True downsamp_temp = 1 data, Data_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp_temp, scaleindep, width_bins, rawdatafile, binratio, data) #print "waterfall",memory.resident()/(1024.0**3) # Saving the arrays into the .spd file. with open(temp_filename+".spd", 'wb') as f: np.savez_compressed(f, Data_dedisp_nozerodm = Data_dedisp_nozerodm.astype(np.float16), Data_dedisp_zerodm = Data_dedisp_zerodm.astype(np.float16), Data_nozerodm = Data_nozerodm.astype(np.float16), delays_nozerodm = delays_nozerodm, freqs_nozerodm = freqs_nozerodm, Data_zerodm = Data_zerodm.astype(np.float16), dm_arr= map(np.float16, dm_arr), sigma_arr = map(np.float16, sigma_arr), dm_list= map(np.float16, dm_list), time_list = map(np.float16, time_list), text_array = text_array) print_debug("Now plotting...") #print "Before plot..",memory.resident()/(1024.0**3) show_spplots.plot(temp_filename+".spd", args[1:], xwin=False, outfile = basename, tar = None) print_debug("Finished plot %i " %j+strftime("%Y-%m-%d %H:%M:%S")) #print "After plot..",memory.resident()/(1024.0**3) numcands+=1 print_debug('Finished sp_candidate : %i'%numcands) print_debug("Finished running waterfaller... "+strftime("%Y-%m-%d %H:%M:%S"))
def main(): parser = optparse.OptionParser(prog="sp_pipeline..py", \ version=" Chitrang Patel (May. 12, 2015)", \ usage="%prog INFILE(PsrFits FILE, SINGLEPULSE FILES)", \ description="Create single pulse plots to show the " \ "frequency sweeps of a single pulse, " \ "DM vs time, and SNR vs DM,"\ "in psrFits data.") parser.add_option('--infile', dest='infile', type='string', \ help="Give a .inf file to read the appropriate header information.") parser.add_option('--groupsfile', dest='txtfile', type='string', \ help="Give the groups.txt file to read in the groups information.") parser.add_option('--mask', dest='maskfile', type='string', \ help="Mask file produced by rfifind. (Default: No Mask).", \ default=None) parser.add_option('-n', dest='maxnumcands', type='int', \ help="Maximum number of candidates to plot. (Default: 100).", \ default=100) options, args = parser.parse_args() if not hasattr(options, 'infile'): raise ValueError("A .inf file must be given on the command line! ") if not hasattr(options, 'txtfile'): raise ValueError( "The groups.txt file must be given on the command line! ") files = get_textfile(options.txtfile) print_debug("Begining waterfaller... " + strftime("%Y-%m-%d %H:%M:%S")) if not args[0].endswith("fits"): raise ValueError("The first file must be a psrFits file! ") print_debug('Maximum number of candidates to plot: %i' % options.maxnumcands) basename = args[0][:-5] filetype = "psrfits" inffile = options.infile topo, bary = bary_and_topo.bary_to_topo(inffile) time_shift = bary - topo inf = infodata.infodata(inffile) RA = inf.RA dec = inf.DEC MJD = inf.epoch mjd = Popen(["mjd2cal", "%f" % MJD], stdout=PIPE, stderr=PIPE) date, err = mjd.communicate() date = date.split()[2:5] telescope = inf.telescope N = inf.N numcands = 0 Total_observed_time = inf.dt * N print_debug('getting file..') values = split_parameters(options.txtfile) if len(values) > options.maxnumcands: values = sorted(values, key=itemgetter( 5, 1)) #sorting candidates based on ranks and snr values = values[-options.maxnumcands:] print "More than", options.maxnumcands, "candidates, making plots for", options.maxnumcands, "candidates" values = sorted(values, key=itemgetter(0)) for ii in range(len(values)): #### Array for Plotting DM vs SNR print_debug("Making arrays for DM vs Signal to Noise...") temp_list = files[values[ii][6] - 6].split() npulses = int(temp_list[2]) temp_lines = files[(values[ii][6] + 3):(values[ii][6] + npulses + 1)] arr = np.split(temp_lines, len(temp_lines)) dm_list = [] time_list = [] for i in range(len(arr)): dm_val = float(arr[i][0].split()[0]) time_val = float(arr[i][0].split()[2]) dm_list.append(dm_val) time_list.append(time_val) arr_2 = np.array([arr[i][0].split() for i in range(len(arr))], dtype=np.float32) dm_arr = np.array([arr_2[i][0] for i in range(len(arr))], dtype=np.float32) sigma_arr = np.array([arr_2[i][1] for i in range(len(arr))], dtype=np.float32) #### Array for Plotting DM vs Time is in show_spplots.plot(...) #### Setting variables up for the waterfall arrays. j = ii + 1 subdm = dm = sweep_dm = values[ii][0] sample_number = values[ii][3] rank = values[ii][5] width_bins = values[ii][4] #print "dm", dm #print "width_bins", width_bins downsamp = np.round( (values[ii][2] / sample_number / inf.dt)).astype('int') #print "downsamp", downsamp pulse_width = width_bins * downsamp * inf.dt #print "pulse_width", pulse_width if ii == 0: mask_subband = rfifind.rfifind("%s_rfifind.mask" % (basename)) mask_subband.set_zap_chans(power=1000, plot=False) mask_subband.set_weights_and_offsets() mask_subband.write_weights(filename="%s_weights.txt" % (basename)) cmd = "psrfits_subband -dm %.2f -nsub 128 -o %s_subband_%.2f -weights %s_weights.txt %s" % ( dm, basename, dm, basename, args[0]) call(cmd, shell=True) #subband args[0] at dm and then generate a file that will be set equal to rawdatafile subband_file = "%s_subband_%.2f_0001.fits" % (basename, dm) dm_prev = dm subband_prev = subband_file else: dm_diff = dm - dm_prev t_smear = 8.3e3 * dm_diff * (350**-3) * ( np.abs(rawdatafile.frequencies[0] - rawdatafile.frequencies[-1]) / 128.) if (5 * t_smear) > pulse_width: cmd = "psrfits_subband -dm %.2f -nsub 128 -o %s_subband_%.2f -weights %s_weights.txt %s" % ( dm, basename, dm, basename, args[0]) call(cmd, shell=True) #subband args[0] at dm and then generate a file that will be set equal to rawdatafile subband_file = "%s_subband_%.2f_0001.fits" % (basename, dm) dm_prev = dm subband_prev = subband_file rawdatafile = psrfits.PsrfitsFile(subband_file) bin_shift = np.round(time_shift / rawdatafile.tsamp).astype('int') integrate_dm = None sigma = values[ii][1] sweep_posn = 0.0 bary_start_time = values[ii][2] topo_start_time = bary_start_time - topo_timeshift( bary_start_time, time_shift, topo)[0] binratio = 50 scaleindep = False zerodm = None duration = binratio * width_bins * rawdatafile.tsamp * downsamp start = topo_start_time - (0.25 * duration) if (start < 0.0): start = 0.0 if sigma <= 7: nsub = 32 elif sigma >= 7 and sigma < 10: nsub = 64 else: nsub = 128 nbins = np.round(duration / rawdatafile.tsamp).astype('int') start_bin = np.round(start / rawdatafile.tsamp).astype('int') dmfac = 4.15e3 * np.abs(1. / rawdatafile.frequencies[0]**2 - 1. / rawdatafile.frequencies[-1]**2) nbinsextra = np.round( (duration + dmfac * dm) / rawdatafile.tsamp).astype('int') if (start_bin + nbinsextra) > N - 1: nbinsextra = N - 1 - start_bin data = rawdatafile.get_spectra(start_bin, nbinsextra) data = maskdata(data, start_bin, nbinsextra, options.maskfile) #make an array to store header information for the .npz files temp_filename = basename + "_DM%.1f_%.1fs_rank_%i" % ( subdm, topo_start_time, rank) # Array for Plotting Dedispersed waterfall plot - zerodm - OFF print_debug("Running waterfaller with Zero-DM OFF...") data, Data_dedisp_nozerodm = waterfall_array( start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) #Add additional information to the header information array text_array = np.array([ subband_file, 'GBT', RA, dec, MJD, rank, nsub, nbins, subdm, sigma, sample_number, duration, width_bins, pulse_width, rawdatafile.tsamp, Total_observed_time, topo_start_time, data.starttime, data.dt, data.numspectra, data.freqs.min(), data.freqs.max() ]) #### Array for plotting Dedispersed waterfall plot zerodm - ON print_debug("Running Waterfaller with Zero-DM ON...") #print "before get_spectra",memory.resident()/(1024.0**3) data = rawdatafile.get_spectra(start_bin, nbinsextra) #print "after get_spectra",memory.resident()/(1024.0**3) data = maskdata(data, start_bin, nbinsextra, options.maskfile) zerodm = True data, Data_dedisp_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) #print "waterfall",memory.resident()/(1024.0**3) ####Sweeped without zerodm start = start + (0.25 * duration) start_bin = np.round(start / rawdatafile.tsamp).astype('int') sweep_duration = 4.15e3 * np.abs( 1. / rawdatafile.frequencies[0]**2 - 1. / rawdatafile.frequencies[-1]**2) * sweep_dm nbins = np.round(sweep_duration / (rawdatafile.tsamp)).astype('int') if ((nbins + start_bin) > (N - 1)): nbins = N - 1 - start_bin #print "before get_spectra",memory.resident()/(1024.0**3) data = rawdatafile.get_spectra(start_bin, nbins) #print "after get_spectra",memory.resident()/(1024.0**3) data = maskdata(data, start_bin, nbins, options.maskfile) zerodm = None dm = None data, Data_nozerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp, scaleindep, width_bins, rawdatafile, binratio, data) #print "waterfall",memory.resident()/(1024.0**3) text_array = np.append(text_array, sweep_duration) text_array = np.append(text_array, data.starttime) text_array = np.append(text_array, bary_start_time) # Array to Construct the sweep if sweep_dm is not None: ddm = sweep_dm - data.dm delays = psr_utils.delay_from_DM(ddm, data.freqs) delays -= delays.min() delays_nozerodm = delays freqs_nozerodm = data.freqs # Sweeped with zerodm-on zerodm = True downsamp_temp = 1 data, Data_zerodm = waterfall_array(start_bin, dmfac, duration, nbins, zerodm, nsub, subdm, dm, integrate_dm, downsamp_temp, scaleindep, width_bins, rawdatafile, binratio, data) #print "waterfall",memory.resident()/(1024.0**3) # Saving the arrays into the .spd file. with open(temp_filename + ".spd", 'wb') as f: np.savez_compressed( f, Data_dedisp_nozerodm=Data_dedisp_nozerodm.astype(np.float16), Data_dedisp_zerodm=Data_dedisp_zerodm.astype(np.float16), Data_nozerodm=Data_nozerodm.astype(np.float16), delays_nozerodm=delays_nozerodm, freqs_nozerodm=freqs_nozerodm, Data_zerodm=Data_zerodm.astype(np.float16), dm_arr=map(np.float16, dm_arr), sigma_arr=map(np.float16, sigma_arr), dm_list=map(np.float16, dm_list), time_list=map(np.float16, time_list), text_array=text_array) print_debug("Now plotting...") #print "Before plot..",memory.resident()/(1024.0**3) show_spplots.plot(temp_filename + ".spd", args[1:], xwin=False, outfile=basename, tar=None) print_debug("Finished plot %i " % j + strftime("%Y-%m-%d %H:%M:%S")) #print "After plot..",memory.resident()/(1024.0**3) numcands += 1 print_debug('Finished sp_candidate : %i' % numcands) print_debug("Finished running waterfaller... " + strftime("%Y-%m-%d %H:%M:%S"))
def read_from_file(self, params, tsamp, N, lofreq, hifreq, rawdatafile, loc_pulse = 0.5, dedisp = False, \ scaleindep = None, zerodm = None, mask = None, barytime = True, \ nsub = None, bandpass_corr = False): """ Set up parameters based on input from the groups.txt file. Input: params: list of parameters (DM, S/N, Time, Sample number, downfactor) tsamp: sampling time (downsampled: 65.5 us for PALFA) N : The total number of time samples in the observation lofreq: lowest observation frequency hifreq: highest observation frequency rawdatafile: supply a PSRFITS file instance optional arguments: dedisp: Do you want to dedisperse?(Type: Boolean). scaleindep:Do you want to scale each subband independently?(Type: Boolean) zerodm:Do you want to use zero-DM filtering?(Type: Boolean) mask: Do you want to use a rfifind mask? (Type: Boolean) barytime: Is the given time(s) barycentric? bandpass_corr:Would you like to remove the bandpass? (Type: Boolean) """ self.subdm = params[0] self.sigma = params[1] topo, bary = bary_and_topo.bary_to_topo(rawdatafile.filename, rawdatafile=rawdatafile) time_shift = bary-topo if barytime: #### if the times in the groups.txt file are barycentric then get the corresponding #### topocentric times. self.bary_start_time = params[2] self.topo_start_time = self.bary_start_time - topo_timeshift(self.bary_start_time, \ time_shift, topo)[0] else: #### Vice versa. self.topo_start_time = params[2] self.bary_start_time = self.topo_start_time + topo_timeshift(self.bary_start_time, \ time_shift, topo)[0] self.sample_number = params[3] self.width_bins = params[4] sweep_posn = 0.0 self.scaleindep = scaleindep self.downsamp = np.round((params[2]/self.sample_number/tsamp)).astype('int') self.duration = self.binratio * self.width_bins * tsamp * self.downsamp self.start = self.topo_start_time - (loc_pulse * self.duration) if (self.start<0.0): self.start = 0.0 self.start_bin = np.round(self.start/tsamp).astype('int') self.pulse_width = self.width_bins*self.downsamp*tsamp if nsub is not None: self.nsub = nsub else: self.nsub = numsub(rawdatafile.nchan, self.sigma) self.zerodm = zerodm if dedisp: self.dm = self.subdm self.sweep_dm = None self.sweep_duration = 0.0 self.dmfac = 4.15e3 * np.abs(1./lofreq**2 - 1./hifreq**2) self.nbins = np.round(self.duration/tsamp).astype('int') self.nbinsextra = np.round((self.duration + self.dmfac * self.dm)/tsamp).astype('int') if (self.start_bin+self.nbinsextra) > N-1: self.nbinsextra = N-1-self.start_bin self.bandpass_corr = bandpass_corr else: self.dm = None self.sweep_dm = self.subdm self.sweep_duration = 4.15e3 * np.abs(1./lofreq**2-1./hifreq**2)*self.sweep_dm self.start = self.start + (loc_pulse*self.duration) self.start_bin = np.round(self.start/tsamp).astype('int') self.nbins = np.round(self.sweep_duration/tsamp).astype('int') self.nbinsextra = self.nbins if (self.start_bin+self.nbinsextra) > N-1: self.nbinsextra = N-1-self.start_bin self.bandpass_corr = False
def manual_params(self, subdm, dm, sweep_dm, sigma, start_time, width_bins, downsamp, \ duration, nbins, nsub, tsamp, N, lofreq, hifreq, rawdatafile, loc_pulse=0.5, dedisp = False, \ scaleindep = None, zerodm = None, mask = False, barytime = True, \ bandpass_corr = False): """ Set up parameters based on input from the groups.txt file. Input: subdm: DM to use when subbanding. dm: DM to use when dedispersing data for plot. sweep_dm: Show the frequency sweep using this DM. sigma: signal-to-noise of the pulse start_time: start time of the data to be read in for waterfalling. width_bins: Smooth each channel/subband with a boxcar width_bins wide. downsamp: Factor to downsample in time by. Default: Don't downsample. duration: duration of data to be waterfalled. nbins: Number of time bins to plot. This option overrides the duration argument. nsub: Number of subbands to use. Must be a factor of number of channels. tsamp: sampling time (downsampled: 65.5 us for PALFA) N: total number of samples in an observations lofreq: lowest observation frequency hifreq: highest observation frequency rawdatafile: supply a psrfits file instance optional arguments: dedisp: Do you want to dedisperse?(Type: Boolean). scaleindep:Do you want to scale each subband independently?(Type: Boolean) zerodm:Do you want to use zero-DM filtering?(Type: Boolean) mask: Do you want to use a rfifind mask? (Type: Boolean) barytime: Is the given time(s) barycentric? bandpass_corr:Would you like to remove the bandpass? (Type: Boolean) """ self.subdm = subdm self.mask = mask self.sigma = sigma topo, bary = bary_and_topo.bary_to_topo(rawdatafile.filename, rawdatafile=rawdatafile) time_shift = bary-topo if barytime: #### if the time is barycentric then get the corresponding topocentric time. self.bary_start_time = start_time self.topo_start_time = self.bary_start_time - topo_timeshift(self.bary_start_time, \ time_shift, topo)[0] else: #### Vice versa. self.topo_start_time = start_time self.bary_start_time = self.topo_start_time + topo_timeshift(self.bary_start_time, \ time_shift, topo)[0] self.sample_number = np.round(self.bary_start_time/tsamp).astype('int') self.width_bins = width_bins sweep_posn = 0.0 if scaleindep: self.scaleindep = True self.downsamp = downsamp if nbins: self.nbins = nbins self.duration = self.nbins*tsamp if duration: self.duration = duration self.nbins = np.round(self.duration/tsamp).astype('int') self.start = self.topo_start_time - (loc_pulse * self.duration) if (self.start<0.0): self.start = 0.0 self.start_bin = np.round(self.start/tsamp).astype('int') self.pulse_width = self.width_bins*self.downsamp*tsamp self.nsub = nsub self.zerodm = zerodm if dedisp: if dm: self.dm = dm else: self.dm = self.subdm self.sweep_dm = None self.sweep_duration = 0.0 self.dmfac = 4.15e3 * np.abs(1./lofreq**2 - 1./hifreq**2) self.nbinsextra = self.nbins + np.round((self.dmfac * self.dm)/tsamp).astype('int') if (self.start_bin+self.nbinsextra) > N-1: self.nbinsextra = N-1-self.start_bin self.bandpass_corr = bandpass_corr else: self.dm = None if not sweep_dm: self.sweep_dm = self.subdm else: self.sweep_dm = sweep_dm self.sweep_duration = 4.15e3 * np.abs(1./lofreq**2-1./hifreq**2)*self.sweep_dm self.start = self.start + (loc_pulse*self.duration) self.start_bin = np.round(self.start/tsamp).astype('int') self.nbinsextra = self.nbins if (self.start_bin+self.nbinsextra) > N-1: self.nbinsextra = N-1-self.start_bin self.bandpass_corr = False