Esempio n. 1
0
def make_spd_from_file(spdcand, rawdatafile, \
                       txtfile, maskfile, \
                       min_rank, group_rank, \
                       plot, just_waterfall, \
                       integrate_ts, integrate_spec, disp_pulse, \
                       loc_pulse, nsub, \
                       maxnumcands, \
                       basename, \
                       mask=False, bandpass_corr=True, barytime=True, \
                       man_params=None):
    """
    Makes spd files from output files of rratrap. 
    Inputs:
        spdcand: spcand parameters instance (read in spcand.params)
        rawdatafile: psrfits file instance
        txtfile: rratrap output file (groups.txt file)
        maskfile: rfifind mask file. need this file if you want to remove the bandpass 
                  or use rfifind mask information.
        min_rank: plot all groups with rank more than this. min 1, max 6
        group_rank: plot groups ranked whatever you specify
        plot: do you want to produce the plots as well? 
        just_waterfall: Do you just want to make the waterfall plots.
        integrate_ts: Do you want to display the dedispersed time series in the plot?
        integrate_spec: Do you want to display the pulse spectrum in the plot?
        disp_pulse: Do you want to see the inset dispersed pulse in the plot?
        loc_pulse: Fraction of the window length where the pulse is located.(eg. 0.25 = 1/4th of the way in.
                                                                             0.5 = middle of the plot)
        maxnumcands: What is the maximum number of candidates you would like to generate?
        basename: output basename of the file. Appended with _DM_TIME(s)_RANK.spd 
    Optional arguments:
        mask: Do you want to mask out rfi contaminated channels?
        bandpass_corr: Do you want to remove the bandpass?
        barytime: Is the given time(s) barycentric?
        man_params: Do you want to specify the parameters for waterfalling 
                    manually? If yes, I suggest using the function make_spd_from_man_params().
                    (I suggest giving it the rratrap output file)    
    Outputs:
       Binary npz file containing the necessary arrays and header information to generate the spd plots.
    """
    numcands = 0  # counter for max number of candidates
    loop_must_break = False  # dont break the loop unless num of cands >100.
    files = spio.get_textfile(options.txtfile)
    if group_rank:
        groups = [group_rank - 1]
    else:
        groups = [i for i in range(6) if (i >= min_rank)][::-1]

    for group in groups:
        rank = group + 1
        if files[group] != "Number of rank %i groups: 0 " % rank:
            values = spio.split_parameters(rank, txtfile)
            lis = np.where(files == '\tRank:             %i.000000' % rank)[0]
            for ii in range(len(values)):
                #### Arrays for Plotting DM vs SNR
                dm_list, time_list, dm_arr, sigma_arr, width_arr = spio.read_RRATrap_info(
                    txtfile, lis[ii], rank)

                # Array for Plotting Dedispersed waterfall plot - zerodm - OFF
                spdcand.read_from_file(values[ii], rawdatafile.tsamp, rawdatafile.specinfo.N, \
                                       rawdatafile.frequencies[0], rawdatafile.frequencies[-1], \
                                       rawdatafile, loc_pulse=loc_pulse, dedisp = True, \
                                       scaleindep = None, zerodm = None, mask = mask, \
                                       barytime=barytime, \
                                       nsub = nsub, bandpass_corr = bandpass_corr)

                #make an array to store header information for the spd files
                temp_filename = basename+"_DM%.1f_%.1fs_rank_%i"%(spdcand.subdm, \
                                                   spdcand.topo_start_time, rank)

                print_debug("Running waterfaller with Zero-DM OFF...")

                # Add additional information to the header information array
                data, Data_dedisp_nozerodm = waterfall_array(rawdatafile, spdcand.start, \
                                             spdcand.duration, spdcand.dm, spdcand.nbins, spdcand.nsub, \
                                             spdcand.subdm, spdcand.zerodm, spdcand.downsamp, \
                                             spdcand.scaleindep, spdcand.width_bins, \
                                             spdcand.mask, maskfile, spdcand.bandpass_corr)

                text_array = np.array([args[0], rawdatafile.specinfo.telescope, \
                                       rawdatafile.specinfo.ra_str, rawdatafile.specinfo.dec_str, \
                                       rawdatafile.specinfo.start_MJD[0], \
                                       rank, spdcand.nsub, spdcand.nbins, spdcand.subdm, \
                                       spdcand.sigma, spdcand.sample_number, spdcand.duration, \
                                       spdcand.width_bins, spdcand.pulse_width, rawdatafile.tsamp,\
                                       rawdatafile.specinfo.T, spdcand.topo_start_time, data.starttime, \
                                       data.dt,data.numspectra, data.freqs.min(), data.freqs.max()])

                #### Array for plotting Dedispersed waterfall plot zerodm - ON
                print_debug("Running Waterfaller with Zero-DM ON...")
                zerodm = True
                data, Data_dedisp_zerodm = waterfall_array(rawdatafile, spdcand.start, \
                                           spdcand.duration, spdcand.dm, spdcand.nbins, spdcand.nsub, \
                                           spdcand.subdm, zerodm, spdcand.downsamp, \
                                           spdcand.scaleindep, spdcand.width_bins, \
                                           spdcand.mask, maskfile, spdcand.bandpass_corr)
                ####Sweeped without zerodm
                spdcand.read_from_file(values[ii], rawdatafile.tsamp, rawdatafile.specinfo.N, \
                                      rawdatafile.frequencies[0], rawdatafile.frequencies[-1], \
                                      rawdatafile, loc_pulse=loc_pulse, dedisp = None, \
                                      scaleindep = None, zerodm = None, mask = mask, \
                                      barytime=barytime, \
                                      nsub = nsub, bandpass_corr = bandpass_corr)
                data, Data_nozerodm = waterfall_array(rawdatafile, spdcand.start, \
                                           spdcand.duration, spdcand.dm, spdcand.nbins, spdcand.nsub, \
                                           spdcand.subdm, spdcand.zerodm, spdcand.downsamp, \
                                           spdcand.scaleindep, spdcand.width_bins, \
                                           spdcand.mask, maskfile, spdcand.bandpass_corr)
                text_array = np.append(text_array, spdcand.sweep_duration)
                text_array = np.append(text_array, data.starttime)
                text_array = np.append(text_array, spdcand.bary_start_time)
                text_array = np.append(text_array, man_params)
                # Array to Construct the sweep
                if spdcand.sweep_dm is not None:
                    ddm = spdcand.sweep_dm - data.dm
                    delays = psr_utils.delay_from_DM(ddm, data.freqs)
                    delays -= delays.min()
                    delays_nozerodm = delays
                    freqs_nozerodm = data.freqs
                # Sweeped with zerodm-on
                zerodm = True
                #downsamp_temp = 1
                data, Data_zerodm = waterfall_array(rawdatafile, spdcand.start, \
                                           spdcand.duration, spdcand.dm, spdcand.nbins, spdcand.nsub, \
                                           spdcand.subdm, zerodm, spdcand.downsamp, \
                                           spdcand.scaleindep, spdcand.width_bins, \
                                           spdcand.mask, maskfile, spdcand.bandpass_corr)
                # Saving the arrays into the .spd file.
                with open(temp_filename + ".spd", 'wb') as f:
                    np.savez_compressed(f, \
                                        Data_dedisp_nozerodm = Data_dedisp_nozerodm.astype(np.float16),\
                                        Data_dedisp_zerodm = Data_dedisp_zerodm.astype(np.float16),\
                                        Data_nozerodm = Data_nozerodm.astype(np.float16),\
                                        delays_nozerodm = delays_nozerodm, \
                                        freqs_nozerodm = freqs_nozerodm,\
                                        Data_zerodm = Data_zerodm.astype(np.float16), \
                                        dm_arr= list(map(np.float16, dm_arr)),\
                                        sigma_arr = list(map(np.float16, sigma_arr)), \
                                        width_arr =list(map(np.uint8, width_arr)),\
                                        dm_list= list(map(np.float16, dm_list)), \
                                        time_list = list(map(np.float16, time_list)), \
                                        text_array = text_array)
                #### Arrays for Plotting DM vs Time is in plot_spd.plot(...)
                if plot:
                    print_debug("Now plotting...")
                    plot_spd.plot(temp_filename+".spd", args[1:], \
                                  spec_width=1.5, loc_pulse=loc_pulse, \
                                  xwin=False, outfile=basename, \
                                  just_waterfall=just_waterfall, \
                                  integrate_spec=integrate_spec, \
                                  integrate_ts=integrate_ts, \
                                  disp_pulse=disp_pulse, tar = None)
                    print_debug("Finished plot %i " % ii +
                                strftime("%Y-%m-%d %H:%M:%S"))
                numcands += 1
                print_debug('Finished sp_candidate : %i' % numcands)
                if numcands >= maxnumcands:  # Max number of candidates to plot 100.
                    loop_must_break = True
                    break
            if loop_must_break:
                break

        print_debug("Finished group %i... " % rank +
                    strftime("%Y-%m-%d %H:%M:%S"))
    print_debug("Finished running waterfaller... " +
                strftime("%Y-%m-%d %H:%M:%S"))
Esempio n. 2
0
def make_spd_from_file(spdcand, rawdatafile, \
                       txtfile, maskfile, \
                       min_rank, group_rank, \
                       plot, just_waterfall, \
                       integrate_ts, integrate_spec, disp_pulse, \
                       loc_pulse, nsub, \
                       maxnumcands, \
                       basename, \
                       mask=False, bandpass_corr=True, barytime=True, \
                       man_params=None):
    
    """
    Makes spd files from output files of rratrap. 
    Inputs:
        spdcand: spcand parameters instance (read in spcand.params)
        rawdatafile: psrfits file instance
        txtfile: rratrap output file (groups.txt file)
        maskfile: rfifind mask file. need this file if you want to remove the bandpass 
                  or use rfifind mask information.
        min_rank: plot all groups with rank more than this. min 1, max 6
        group_rank: plot groups ranked whatever you specify
        plot: do you want to produce the plots as well? 
        just_waterfall: Do you just want to make the waterfall plots.
        integrate_ts: Do you want to display the dedispersed time series in the plot?
        integrate_spec: Do you want to display the pulse spectrum in the plot?
        disp_pulse: Do you want to see the inset dispersed pulse in the plot?
        loc_pulse: Fraction of the window length where the pulse is located.(eg. 0.25 = 1/4th of the way in.
                                                                             0.5 = middle of the plot)
        maxnumcands: What is the maximum number of candidates you would like to generate?
        basename: output basename of the file. Appended with _DM_TIME(s)_RANK.spd 
    Optional arguments:
        mask: Do you want to mask out rfi contaminated channels?
        bandpass_corr: Do you want to remove the bandpass?
        barytime: Is the given time(s) barycentric?
        man_params: Do you want to specify the parameters for waterfalling 
                    manually? If yes, I suggest using the function make_spd_from_man_params().
                    (I suggest giving it the rratrap output file)    
    Outputs:
       Binary npz file containing the necessary arrays and header information to generate the spd plots.
    """
    numcands=0 # counter for max number of candidates
    loop_must_break = False # dont break the loop unless num of cands >100.
    files = spio.get_textfile(options.txtfile)
    if group_rank:
        groups=[group_rank-1]
    else:
        groups = [i for i in range(6) if(i>=min_rank)][::-1]
     
    for group in groups:
        rank = group+1
        if files[group] != "Number of rank %i groups: 0 "%rank:
            values = spio.split_parameters(rank, txtfile)
            lis = np.where(files == '\tRank:             %i.000000'%rank)[0]
            for ii in range(len(values)):
                #### Arrays for Plotting DM vs SNR
                dm_list, time_list, dm_arr, sigma_arr, width_arr = spio.read_RRATrap_info(txtfile, lis[ii], rank)


                # Array for Plotting Dedispersed waterfall plot - zerodm - OFF
                spdcand.read_from_file(values[ii], rawdatafile.tsamp, rawdatafile.specinfo.N, \
                                       rawdatafile.frequencies[0], rawdatafile.frequencies[-1], \
                                       rawdatafile, loc_pulse=loc_pulse, dedisp = True, \
                                       scaleindep = None, zerodm = None, mask = mask, \
                                       barytime=barytime, \
                                       nsub = nsub, bandpass_corr = bandpass_corr)

                #make an array to store header information for the spd files
                temp_filename = basename+"_DM%.1f_%.1fs_rank_%i"%(spdcand.subdm, \
                                                   spdcand.topo_start_time, rank)
                
                print_debug("Running waterfaller with Zero-DM OFF...")
                
                # Add additional information to the header information array
                data, Data_dedisp_nozerodm = waterfall_array(rawdatafile, spdcand.start, \
                                             spdcand.duration, spdcand.dm, spdcand.nbins, spdcand.nsub, \
                                             spdcand.subdm, spdcand.zerodm, spdcand.downsamp, \
                                             spdcand.scaleindep, spdcand.width_bins, \
                                             spdcand.mask, maskfile, spdcand.bandpass_corr)

                text_array = np.array([args[0], rawdatafile.specinfo.telescope, \
                                       rawdatafile.specinfo.ra_str, rawdatafile.specinfo.dec_str, \
                                       rawdatafile.specinfo.start_MJD[0], \
                                       rank, spdcand.nsub, spdcand.nbins, spdcand.subdm, \
                                       spdcand.sigma, spdcand.sample_number, spdcand.duration, \
                                       spdcand.width_bins, spdcand.pulse_width, rawdatafile.tsamp,\
                                       rawdatafile.specinfo.T, spdcand.topo_start_time, data.starttime, \
                                       data.dt,data.numspectra, data.freqs.min(), data.freqs.max()])

                #### Array for plotting Dedispersed waterfall plot zerodm - ON
                print_debug("Running Waterfaller with Zero-DM ON...")
                zerodm=True
                data, Data_dedisp_zerodm = waterfall_array(rawdatafile, spdcand.start, \
                                           spdcand.duration, spdcand.dm, spdcand.nbins, spdcand.nsub, \
                                           spdcand.subdm, zerodm, spdcand.downsamp, \
                                           spdcand.scaleindep, spdcand.width_bins, \
                                           spdcand.mask, maskfile, spdcand.bandpass_corr)
                ####Sweeped without zerodm
                spdcand.read_from_file(values[ii], rawdatafile.tsamp, rawdatafile.specinfo.N, \
                                      rawdatafile.frequencies[0], rawdatafile.frequencies[-1], \
                                      rawdatafile, loc_pulse=loc_pulse, dedisp = None, \
                                      scaleindep = None, zerodm = None, mask = mask, \
                                      barytime=barytime, \
                                      nsub = nsub, bandpass_corr = bandpass_corr)
                data, Data_nozerodm = waterfall_array(rawdatafile, spdcand.start, \
                                           spdcand.duration, spdcand.dm, spdcand.nbins, spdcand.nsub, \
                                           spdcand.subdm, spdcand.zerodm, spdcand.downsamp, \
                                           spdcand.scaleindep, spdcand.width_bins, \
                                           spdcand.mask, maskfile, spdcand.bandpass_corr)
                text_array = np.append(text_array, spdcand.sweep_duration)
                text_array = np.append(text_array, data.starttime)
                text_array = np.append(text_array, spdcand.bary_start_time)
                text_array = np.append(text_array, man_params)
                # Array to Construct the sweep
                if spdcand.sweep_dm is not None:
                    ddm = spdcand.sweep_dm-data.dm
                    delays = psr_utils.delay_from_DM(ddm, data.freqs)
                    delays -= delays.min()
                    delays_nozerodm = delays
                    freqs_nozerodm = data.freqs
                # Sweeped with zerodm-on 
                zerodm = True
                #downsamp_temp = 1
                data, Data_zerodm = waterfall_array(rawdatafile, spdcand.start, \
                                           spdcand.duration, spdcand.dm, spdcand.nbins, spdcand.nsub, \
                                           spdcand.subdm, zerodm, spdcand.downsamp, \
                                           spdcand.scaleindep, spdcand.width_bins, \
                                           spdcand.mask, maskfile, spdcand.bandpass_corr)
                # Saving the arrays into the .spd file.
                with open(temp_filename+".spd", 'wb') as f:
                    np.savez_compressed(f, \
                                        Data_dedisp_nozerodm = Data_dedisp_nozerodm.astype(np.float16),\
                                        Data_dedisp_zerodm = Data_dedisp_zerodm.astype(np.float16),\
                                        Data_nozerodm = Data_nozerodm.astype(np.float16),\
                                        delays_nozerodm = delays_nozerodm, \
                                        freqs_nozerodm = freqs_nozerodm,\
                                        Data_zerodm = Data_zerodm.astype(np.float16), \
                                        dm_arr= list(map(np.float16, dm_arr)),\
                                        sigma_arr = list(map(np.float16, sigma_arr)), \
                                        width_arr =list(map(np.uint8, width_arr)),\
                                        dm_list= list(map(np.float16, dm_list)), \
                                        time_list = list(map(np.float16, time_list)), \
                                        text_array = text_array)
                #### Arrays for Plotting DM vs Time is in plot_spd.plot(...)
                if plot:
                    print_debug("Now plotting...")
                    plot_spd.plot(temp_filename+".spd", args[1:], \
                                  spec_width=1.5, loc_pulse=loc_pulse, \
                                  xwin=False, outfile=basename, \
                                  just_waterfall=just_waterfall, \
                                  integrate_spec=integrate_spec, \
                                  integrate_ts=integrate_ts, \
                                  disp_pulse=disp_pulse, tar = None)
                    print_debug("Finished plot %i " %ii+strftime("%Y-%m-%d %H:%M:%S"))
                numcands+= 1
                print_debug('Finished sp_candidate : %i'%numcands)
                if numcands >= maxnumcands:    # Max number of candidates to plot 100.
                    loop_must_break = True
                    break
            if loop_must_break:
                break

        print_debug("Finished group %i... "%rank+strftime("%Y-%m-%d %H:%M:%S"))
    print_debug("Finished running waterfaller... "+strftime("%Y-%m-%d %H:%M:%S"))
Esempio n. 3
0
def GBNCC_wrapper(txtfile, maskfile, fitsfilenm, path_sp_files):
    """
	The pipeline should pass job.fits_filenm as argument.
    """
    files = spio.get_textfile(txtfile)
    min_rank = 3
    groups = [i for i in range(7) if (i >= min_rank)][::-1]
    numcands = 0  # counter for max number of candidates
    loop_must_break = False  # dont break the loop unless num of cands >100.

    values = []
    lis = []
    ranks = []
    for group in groups:
        rank = group + 1
        if files[group] != "Number of rank %i groups: 0 " % rank:
            add_values = spio.split_parameters(rank, txtfile)
            values.extend(add_values)
            lis = np.append(
                lis,
                np.where(files == '\tRank:             %i.000000' % rank)[0])
            add_ranks = np.ones(len(add_values)) * rank
            ranks = np.append(ranks, add_ranks)

    if len(values) > 100:
        values = values[0:100]
        lis = lis[0:100]

    #Sort candidates based on DM
    zip_list = zip(values, lis, ranks)
    zip_list = sorted(zip_list, key=itemgetter(0, 0))
    values = [x[0] for x in zip_list]
    lis = [x[1] for x in zip_list]
    ranks = [x[2] for x in zip_list]

    basename = fitsfilenm[:-5]
    #generate subbanded file at a DM of 0 to extract observation parameters
    cmd = "psrfits_subband -dm 0.0 -nsub 128 -o %s_subband_0.0 %s" % (
        basename, fitsfilenm)
    print "executing %s" % cmd
    subprocess.call(cmd, shell=True)
    subfilenm = basename + "_subband_0.0_0001.fits"
    subfile = psrfits.PsrfitsFile(subfilenm)

    for ii in range(len(values)):
        dm_list, time_list, dm_arr, sigma_arr, width_arr = spio.read_RRATrap_info(
            txtfile, lis[ii], int(ranks[ii]))
        wrapper_cand = spcand.params()
        wrapper_cand.read_from_file(values[ii], subfile.tsamp, subfile.specinfo.N, \
                                       get_obs_info(fitsfilenm)['hifreq'], get_obs_info(fitsfilenm)['lofreq'], \
                                       subfile, loc_pulse=0.5, dedisp = True,\
                                       scaleindep = None, zerodm = None, mask = None,\
                                       barytime=True, nsub = None, bandpass_corr = False)
        temp_filename = basename + "_rank_%i" % int(ranks[ii])
        if ii == 0:  #check if index is 0
            correct_rawdatafile, prevsubfile, prevDM  = pick_rawdatafile(wrapper_cand.subdm, fitsfilenm, \
                                                                get_obs_info(fitsfilenm)['hifreq'], get_obs_info(fitsfilenm)['lofreq'], \
                                                                wrapper_cand.pulse_width, prevDM = 0, prevsubfile = '', \
                                                                init_flag = 1)
        else:
            correct_rawdatafile, prevsubfile, prevDM  = pick_rawdatafile(wrapper_cand.subdm, fitsfilenm, \
                                                                get_obs_info(fitsfilenm)['hifreq'], get_obs_info(fitsfilenm)['lofreq'], \
                                                                wrapper_cand.pulse_width, prevDM = prevDM, \
                                                                prevsubfile=prevsubfile, init_flag = 0)

        path = os.path.dirname(os.path.abspath(__file__))
        cmd = "python " + path + "/make_spd.py --use_manual_params --subdm %f --nsub %d" \
                     " --dm %f -T %f -t %f --width-bins %d --downsamp %d --show-spec"\
                      " --noplot --notopo -o %s %s " %(wrapper_cand.subdm, \
                                                 wrapper_cand.nsub, wrapper_cand.dm, \
                                                 wrapper_cand.topo_start_time, wrapper_cand.duration, \
                                                 wrapper_cand.width_bins, wrapper_cand.downsamp,\
                                                 temp_filename, correct_rawdatafile)
        print "executing %s" % cmd
        subprocess.call(cmd, shell=True)
        # Add additional information to the header information array
        text_array = np.array([correct_rawdatafile, subfile.specinfo.telescope, \
                           subfile.specinfo.ra_str, subfile.specinfo.dec_str, \
                           subfile.specinfo.start_MJD[0], int(ranks[ii]), \
                           wrapper_cand.nsub, wrapper_cand.nbins, \
                           wrapper_cand.subdm, wrapper_cand.sigma, wrapper_cand.sample_number, \
                           wrapper_cand.duration, wrapper_cand.width_bins, wrapper_cand.pulse_width, \
                           subfile.tsamp, subfile.specinfo.T, wrapper_cand.topo_start_time])

        temp_filename += "_DM%.1f_%.1fs" % (wrapper_cand.subdm,
                                            wrapper_cand.topo_start_time)
        spd = read_spd.spd(temp_filename + '.spd')
        spd.man_params = None
        text_array = np.append(text_array, spd.waterfall_start_time)
        text_array = np.append(text_array, spd.waterfall_tsamp)
        text_array = np.append(text_array, spd.waterfall_prededisp_nbins)
        text_array = np.append(text_array, spd.min_freq)
        text_array = np.append(text_array, spd.max_freq)
        text_array = np.append(text_array, spd.sweep_duration)
        text_array = np.append(text_array, spd.sweep_start_time)
        text_array = np.append(text_array, spd.bary_pulse_peak_time)
        text_array = np.append(text_array, spd.man_params)

        with open(temp_filename + ".spd", 'wb') as f:
            np.savez_compressed(f, \
                                        Data_dedisp_nozerodm = spd.data_nozerodm_dedisp,\
                                        Data_dedisp_zerodm = spd.data_zerodm_dedisp,\
                                        Data_nozerodm = spd.data_nozerodm,\
                                        delays_nozerodm = spd.dmsweep_delays, \
                                        freqs_nozerodm = spd.dmsweep_freqs,\
                                        Data_zerodm = spd.data_zerodm, \
                                        dm_arr= map(np.float16, dm_arr),\
                                        sigma_arr = map(np.float16, sigma_arr), \
                                        width_arr =map(np.uint8, width_arr),\
                                        dm_list= map(np.float16, dm_list), \
                                        time_list = map(np.float16, time_list), \
                                        text_array = text_array)
        plot_spd.plot(temp_filename+".spd", glob.glob(path_sp_files+'/*.singlepulse'), maskfile, \
                              outfile=basename, just_waterfall=False, \
                              integrate_spec=True, integrate_ts=True, \
                              disp_pulse=False, bandpass_corr = True, tar = None)
        numcands += 1
        print 'Finished sp_candidate : %i' % numcands
        if numcands >= 100:  # Max number of candidates to plot 100.
            loop_must_break = True
            break
def GBNCC_wrapper(txtfile, maskfile, fitsfilenm, path_sp_files):
    """
	The pipeline should pass job.fits_filenm as argument.
    """
    files = spio.get_textfile(txtfile)
    min_rank = 3
    groups = [i for i in range(7) if(i>=min_rank)][::-1]
    numcands=0 # counter for max number of candidates
    loop_must_break = False # dont break the loop unless num of cands >100.
    
    values = []
    lis = [] 
    ranks = []
    for group in groups:
        rank = group+1
        if files[group] != "Number of rank %i groups: 0 "%rank:
            add_values = spio.split_parameters(rank, txtfile) 
            values.extend(add_values)
            lis = np.append(lis, np.where(files == '\tRank:             %i.000000'%rank)[0])
            add_ranks = np.ones(len(add_values)) * rank
            ranks = np.append(ranks, add_ranks)
     
    if len(values) > 100:
        values = values[0:100]
        lis = lis[0:100]

    #Sort candidates based on DM
    zip_list = zip(values, lis, ranks)
    zip_list = sorted(zip_list, key=itemgetter(0,0))
    values = [x[0] for x in zip_list]
    lis = [x[1] for x in zip_list]
    ranks = [x[2] for x in zip_list]
    
    basename = fitsfilenm[:-5]
    #generate subbanded file at a DM of 0 to extract observation parameters
    cmd = "psrfits_subband -dm 0.0 -nsub 128 -o %s_subband_0.0 %s"%(basename,fitsfilenm)
    print "executing %s" %cmd
    subprocess.call(cmd, shell=True)
    subfilenm = basename + "_subband_0.0_0001.fits"
    subfile = psrfits.PsrfitsFile(subfilenm)
    
    for ii in range(len(values)):
        dm_list, time_list, dm_arr, sigma_arr, width_arr = spio.read_RRATrap_info(txtfile, lis[ii], int(ranks[ii]))
        wrapper_cand = spcand.params() 
        wrapper_cand.read_from_file(values[ii], subfile.tsamp, subfile.specinfo.N, \
                                       get_obs_info(fitsfilenm)['hifreq'], get_obs_info(fitsfilenm)['lofreq'], \
                                       subfile, loc_pulse=0.5, dedisp = True,\
                                       scaleindep = None, zerodm = None, mask = None,\
                                       barytime=True, nsub = None, bandpass_corr = False) 
        temp_filename = basename+"_rank_%i"%int(ranks[ii])
        if ii == 0: #check if index is 0
            correct_rawdatafile, prevsubfile, prevDM  = pick_rawdatafile(wrapper_cand.subdm, fitsfilenm, \
                                                                get_obs_info(fitsfilenm)['hifreq'], get_obs_info(fitsfilenm)['lofreq'], \
                                                                wrapper_cand.pulse_width, prevDM = 0, prevsubfile = '', \
                                                                init_flag = 1)
        else:
            correct_rawdatafile, prevsubfile, prevDM  = pick_rawdatafile(wrapper_cand.subdm, fitsfilenm, \
                                                                get_obs_info(fitsfilenm)['hifreq'], get_obs_info(fitsfilenm)['lofreq'], \
                                                                wrapper_cand.pulse_width, prevDM = prevDM, \
                                                                prevsubfile=prevsubfile, init_flag = 0)
            
        
        path = os.path.dirname(os.path.abspath(__file__))
        cmd = "python " + path + "/make_spd.py --use_manual_params --subdm %f --nsub %d" \
              	      " --dm %f -T %f -t %f --width-bins %d --downsamp %d --show-spec"\
                      " --noplot --notopo -o %s %s " %(wrapper_cand.subdm, \
                                                 wrapper_cand.nsub, wrapper_cand.dm, \
                                                 wrapper_cand.topo_start_time, wrapper_cand.duration, \
                                                 wrapper_cand.width_bins, wrapper_cand.downsamp,\
                                                 temp_filename, correct_rawdatafile)
        print "executing %s" %cmd
        subprocess.call(cmd, shell=True)       
        # Add additional information to the header information array
        text_array = np.array([correct_rawdatafile, subfile.specinfo.telescope, \
                           subfile.specinfo.ra_str, subfile.specinfo.dec_str, \
                           subfile.specinfo.start_MJD[0], int(ranks[ii]), \
                           wrapper_cand.nsub, wrapper_cand.nbins, \
                           wrapper_cand.subdm, wrapper_cand.sigma, wrapper_cand.sample_number, \
                           wrapper_cand.duration, wrapper_cand.width_bins, wrapper_cand.pulse_width, \
                           subfile.tsamp, subfile.specinfo.T, wrapper_cand.topo_start_time])
                
        temp_filename +="_DM%.1f_%.1fs"%(wrapper_cand.subdm, wrapper_cand.topo_start_time)
        spd = read_spd.spd(temp_filename+'.spd')
        spd.man_params = None 
        text_array = np.append(text_array, spd.waterfall_start_time)
        text_array = np.append(text_array, spd.waterfall_tsamp)
        text_array = np.append(text_array, spd.waterfall_prededisp_nbins)
        text_array = np.append(text_array, spd.min_freq)
        text_array = np.append(text_array, spd.max_freq)
        text_array = np.append(text_array, spd.sweep_duration)
        text_array = np.append(text_array, spd.sweep_start_time)
        text_array = np.append(text_array, spd.bary_pulse_peak_time)
        text_array = np.append(text_array, spd.man_params)

        with open(temp_filename+".spd", 'wb') as f:
            np.savez_compressed(f, \
                                        Data_dedisp_nozerodm = spd.data_nozerodm_dedisp,\
                                        Data_dedisp_zerodm = spd.data_zerodm_dedisp,\
                                        Data_nozerodm = spd.data_nozerodm,\
                                        delays_nozerodm = spd.dmsweep_delays, \
                                        freqs_nozerodm = spd.dmsweep_freqs,\
                                        Data_zerodm = spd.data_zerodm, \
                                        dm_arr= map(np.float16, dm_arr),\
                                        sigma_arr = map(np.float16, sigma_arr), \
                                        width_arr =map(np.uint8, width_arr),\
                                        dm_list= map(np.float16, dm_list), \
                                        time_list = map(np.float16, time_list), \
                                        text_array = text_array)
        plot_spd.plot(temp_filename+".spd", glob.glob(path_sp_files+'/*.singlepulse'), maskfile, \
                              outfile=basename, just_waterfall=False, \
                              integrate_spec=True, integrate_ts=True, \
                              disp_pulse=False, bandpass_corr = True, tar = None)
        numcands+= 1
        print 'Finished sp_candidate : %i'%numcands
        if numcands >= 100:    # Max number of candidates to plot 100.
            loop_must_break = True
            break