Exemple #1
0
    def preprocess(cls, fns):
        """Given a list of filenames apply any preprocessing steps
            required. Return a list of file names of the output
            files.

            Input:
                fns: List of names of files to preprocess.

            Output:
                outfns: List of names of preprocessed files.
        """
        infiles = " ".join(fns)
        fnmatchdict = cls.fnmatch(fns[0]).groupdict()
        outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
                        fnmatchdict
        
        outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber

        # Clobber output file
        #if os.path.exists(outfile):
        #    os.remove(outfile)
        
        # Merge mock subbands
        mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
        pipeline_utils.execute(mergecmd, stdout=outbasenm+"_merge.out")

        # Remove first 7 rows from file
        rowdelcmd = "fitsdelrow %s[SUBINT] 1 7" % outfile
        pipeline_utils.execute(rowdelcmd)
        
        # Rename file to remove the '_0001' that was added
        os.rename(outfile, outbasenm+'.fits')

        return [outbasenm+'.fits']
Exemple #2
0
    def preprocess(cls, fns):
        """Given a list of filenames apply any preprocessing steps
            required. Return a list of file names of the output
            files.

            Input:
                fns: List of names of files to preprocess.

            Output:
                outfns: List of names of preprocessed files.
        """
        infiles = " ".join(fns)
        fnmatchdict = cls.fnmatch(fns[0]).groupdict()
        outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
                        fnmatchdict

        outfile = outbasenm + "_0001.fits"  # '0001' added is the filenumber

        # Clobber output file
        #if os.path.exists(outfile):
        #    os.remove(outfile)

        # Merge mock subbands
        mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
        pipeline_utils.execute(mergecmd, stdout=outbasenm + "_merge.out")

        # Remove first 7 rows from file
        rowdelcmd = "fitsdelrow %s[SUBINT] 1 7" % outfile
        pipeline_utils.execute(rowdelcmd)

        # Rename file to remove the '_0001' that was added
        os.rename(outfile, outbasenm + '.fits')

        return [outbasenm + '.fits']
Exemple #3
0
    def get_subints_with_cal(self, nsigma=20, margin_of_error=1):
        """Return a list of subint numbers with the cal turned on.

            Input:
                nsigma: The number of sigma above the median a
                    subint needs to be in order to be flagged as
                    having the cal on. (Default: 50)

                    NOTE: The median absolute deviation is used in
                        place of the standard deviation here.
                margin_of_error: For each subint with the cal on
                    also flag 'margin_of_error' subints on either
                    side. (Default: 1)

            Output:
                subints_with_cal: A sorted list of subints with the
                    cal on.
        """
        fn = self.fns[0]
        if not fn.endswith(".fits"):
            raise ValueError("Filename doesn't end with '.fits'!")
        basenm = fn[:-5]  # Chop off '.fits'
        # Make dat file
        prepdatacmd = "prepdata -noclip -nobary -dm 0 -o %s_pre_DM0.00 %s" % (
            basenm, fn)
        pipeline_utils.execute(prepdatacmd,
                               stdout=basenm + "_pre_prepdata.out")
        datfn = basenm + "_pre_DM0.00.dat"
        samp_per_rec = self.num_samples_per_record
        dat = np.memmap(datfn, mode='r', dtype='float32')
        dat = dat[:dat.size / samp_per_rec * samp_per_rec]
        dat.shape = (dat.size / samp_per_rec, samp_per_rec)
        meds = np.median(dat, axis=1)
        med_of_meds = np.median(meds)
        mad_of_meds = np.median(np.abs(meds - med_of_meds))
        #print "Median of medians:", med_of_meds
        #print "MAD of medians:", mad_of_meds
        #for ii, (med, nsig) in enumerate(zip(meds, (meds-med_of_meds)/mad_of_meds)):
        #    print "%d: %g (%g)" % (ii, med, nsig)
        has_cal = (meds - med_of_meds) / mad_of_meds > nsigma

        subints_with_cal = set()
        for isub in np.flatnonzero(has_cal):
            subints_with_cal.add(isub)
            for x in range(1, margin_of_error + 1):
                subints_with_cal.add(isub - x)
                subints_with_cal.add(isub + x)

        #print "Subints with cal: %s" % sorted(list(subints_with_cal))
        #print "Conservative list of subints to remove: %s" % sorted(list(subints_with_cal))

        # Remove dat file created
        #os.remove(datfn)

        return sorted(list(subints_with_cal))
Exemple #4
0
    def get_subints_with_cal(self, nsigma=15, margin_of_error=1):
        """Return a list of subint numbers with the cal turned on.
 
            Input:
                nsigma: The number of sigma above the median a
                    subint needs to be in order to be flagged as
                    having the cal on. (Default: 15)

                    NOTE: The median absolute deviation is used in
                        place of the standard deviation here.
                margin_of_error: For each subint with the cal on 
                    also flag 'margin_of_error' subints on either
                    side. (Default: 1)
 
            Output:
                subints_with_cal: A sorted list of subints with the
                    cal on.
        """
        fn = self.fns[0]
        if not fn.endswith(".fits"):
            raise ValueError("Filename doesn't end with '.fits'!")
        basenm = fn[:-5] # Chop off '.fits'
        # Make dat file
        prepdatacmd = "prepdata -noclip -nobary -dm 0 -o %s_pre_DM0.00 %s" % (basenm, fn)
        pipeline_utils.execute(prepdatacmd, stdout=basenm+"_pre_prepdata.out")
        datfn = basenm+"_pre_DM0.00.dat"
        samp_per_rec = self.num_samples_per_record
        dat = np.memmap(datfn, mode='r', dtype='float32')
        dat = dat[:dat.size/samp_per_rec*samp_per_rec]
        dat.shape = (dat.size/samp_per_rec, samp_per_rec)
        meds = np.median(dat, axis=1)
        med_of_meds = np.median(meds)
        mad_of_meds = np.median(np.abs(meds-med_of_meds))
        print "Median of medians:", med_of_meds
        print "MAD of medians:", mad_of_meds
        for ii, (med, nsig) in enumerate(zip(meds, (meds-med_of_meds)/mad_of_meds)):
            print "%d: %g (%g)" % (ii, med, nsig)
        has_cal = (meds-med_of_meds)/mad_of_meds > nsigma
 
        subints_with_cal = set()
        print "Subints with cal: %s" % sorted(list(subints_with_cal))
        for isub in np.flatnonzero(has_cal):
            subints_with_cal.add(isub)
            for x in range(1,margin_of_error+1):
                subints_with_cal.add(isub-x)
                subints_with_cal.add(isub+x)
 
        print "Conservative list of subints to remove: %s" % sorted(list(subints_with_cal))

        # Remove dat file created
        os.remove(datfn)
        
        return sorted(list(subints_with_cal))
Exemple #5
0
def version_num():
    """Compute version number and return it as a string.
    """
    prestohash = pipeline_utils.execute("git rev-parse HEAD", \
                            dir=os.getenv('PRESTO'))[0].strip()
    pipelinehash = pipeline_utils.execute("git rev-parse HEAD", \
                            dir=config.basic.pipelinedir)[0].strip()
    psrfits_utilshash = pipeline_utils.execute("git rev-parse HEAD", \
                            dir=config.basic.psrfits_utilsdir)[0].strip()
    vernum = 'PRESTO:%s;PIPELINE:%s;PSRFITS_UTILS:%s' % \
                            (prestohash, pipelinehash, psrfits_utilshash)
    return vernum
Exemple #6
0
def version_num():
    """Compute version number and return it as a string.
    """
    prestohash = pipeline_utils.execute("git rev-parse HEAD", \
                            dir=os.getenv('PRESTO'))[0].strip()
    pipelinehash = pipeline_utils.execute("git rev-parse HEAD", \
                            dir=config.basic.pipelinedir)[0].strip()
    psrfits_utilshash = pipeline_utils.execute("git rev-parse HEAD", \
                            dir=config.basic.psrfits_utilsdir)[0].strip()
    vernum = 'PRESTO:%s;PIPELINE:%s;PSRFITS_UTILS:%s' % \
                            (prestohash, pipelinehash, psrfits_utilshash)
    return vernum
Exemple #7
0
    def preprocess(cls, fns):
        """Given a list of filenames apply any preprocessing steps
            required. Return a list of file names of the output
            files.

            Input:
                fns: List of names of files to preprocess.

            Output:
                outfns: List of names of preprocessed files.
        """
        infiles = " ".join(fns)
        fnmatchdict = cls.fnmatch(fns[0]).groupdict()
        outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
                        fnmatchdict
        
        outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber

        # Clobber output file
        #if os.path.exists(outfile):
        #    os.remove(outfile)
        
        # Merge mock subbands
        mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
        pipeline_utils.execute(mergecmd, stdout=outbasenm+"_merge.out")

        # Anti-center commensal data taken after 2011 Nov. 4 has the
        # cal turned on at the end of the observation for 6-7 seconds.
        # This needs to be removed instead of the first 7 seconds.
        # Changes made by Ryan Lynch 2012 June 17.
        if self.galactic_longitude > 170.0 and self.galactic_longitude < 210.0 \
           and self.timestamp_mjd > 55868:
            if self.specinfo.num_subint >= 270:
                rowdelcmd = "fitsdelrow %[SUBINT] 270 %i" % \
                            (outfile,self.specinfo.num_subint)
            elif self.specinfo.num_subint >= 180 and self.specinfo.num_subint < 190:
                rowdelcmd = "fitsdelrow %[SUBINT] 180 %i" % \
                            (outfile,self.specinfo.num_subint)
        else:
            # Otherwisse, remove first 7 rows from file as usual
            rowdelcmd = "fitsdelrow %s[SUBINT] 1 7" % outfile

        pipeline_utils.execute(rowdelcmd)

        
        # Rename file to remove the '_0001' that was added
        os.rename(outfile, outbasenm+'.fits')

        return [outbasenm+'.fits']
Exemple #8
0
    def preprocess(cls, fns):
        """Given a list of filenames apply any preprocessing steps
            required. Return a list of file names of the output
            files.

            Input:
                fns: List of names of files to preprocess.

            Output:
                outfns: List of names of preprocessed files.
        """
        infiles = " ".join(fns)
        fnmatchdict = cls.fnmatch(fns[0]).groupdict()
        outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
                        fnmatchdict

        outfile = outbasenm + "_0001.fits"  # '0001' added is the filenumber

        # Clobber output file
        #if os.path.exists(outfile):
        #    os.remove(outfile)

        # Merge mock subbands
        mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
        pipeline_utils.execute(mergecmd, stdout=outbasenm + "_merge.out")

        # Anti-center commensal data taken after 2011 Nov. 4 has the
        # cal turned on at the end of the observation for 6-7 seconds.
        # This needs to be removed instead of the first 7 seconds.
        # Changes made by Ryan Lynch 2012 June 17.
        if self.galactic_longitude > 170.0 and self.galactic_longitude < 210.0 \
           and self.timestamp_mjd > 55868:
            if self.specinfo.num_subint >= 270:
                rowdelcmd = "fitsdelrow %[SUBINT] 270 %i" % \
                            (outfile,self.specinfo.num_subint)
            elif self.specinfo.num_subint >= 180 and self.specinfo.num_subint < 190:
                rowdelcmd = "fitsdelrow %[SUBINT] 180 %i" % \
                            (outfile,self.specinfo.num_subint)
        else:
            # Otherwisse, remove first 7 rows from file as usual
            rowdelcmd = "fitsdelrow %s[SUBINT] 1 7" % outfile

        pipeline_utils.execute(rowdelcmd)

        # Rename file to remove the '_0001' that was added
        os.rename(outfile, outbasenm + '.fits')

        return [outbasenm + '.fits']
Exemple #9
0
    def preprocess(cls, fns):
        """Given a list of filenames apply any preprocessing steps
            required. Return a list of file names of the output
            files.

            Input:
                fns: List of names of files to preprocess.

            Output:
                outfns: List of names of preprocessed files.
        """
        infiles = " ".join(fns)
        fnmatchdict = cls.fnmatch(fns[0]).groupdict()
        obsdata = cls(fns)
        outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
                        fnmatchdict
        
        outfile = outbasenm + "_0001.fits" # '0001' added is the filenumber

        # Clobber output file
        #if os.path.exists(outfile):
        #    os.remove(outfile)
        
        # Merge mock subbands
        mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
        pipeline_utils.execute(mergecmd, stdout=outbasenm+"_merge.out")
        
        # Rename file to remove the '_0001' that was added
        mergedfn = outbasenm+'.fits'
        os.rename(outfile, mergedfn)
        
        merged = autogen_dataobj([mergedfn])
        if not isinstance(merged, MergedMockPsrfitsData):
            raise ValueError("Preprocessing of Mock data has not produced " \
                                "a recognized merged file!")
        subints_with_cal = merged.get_subints_with_cal()
        num_subints = merged.num_samples/merged.num_samples_per_record
        subints_with_cal = [isub for isub in subints_with_cal \
                                if isub >=0 and isub < num_subints]
        if len(subints_with_cal):
            rowdelcmds = []
            startrow = subints_with_cal[0]
            numrows = 1
            numdelrows = 0 # Number of rows deleted so far
            # Add infinity to the list of subints with cal to 
            # make sure we remove the last cal-block.
            for isub in subints_with_cal[1:]+[np.inf]:
                isub -= numdelrows # Adjust based on the number of missing rows
                if isub == startrow+numrows:
                    numrows+=1
                else:
                    if startrow+1 < 0.1*num_subints:
                        print "Cal-affected region is within 10%% of start of obs " \
                                "remove all rows before cal. (cal start: %d; " \
                                "total num rows: %d)" % (startrow, num_subints)
                        numrows += startrow
                        startrow = 0
                    elif startrow+numrows > 0.9*num_subints:
                        print "Cal-affected region is within 10%% of end of obs " \
                                "remove all rows after cal. (cal start: %d; " \
                                "total num rows: %d)" % (startrow, num_subints)
                        numrows = num_subints - startrow
                    numdelrows += numrows # Keep track of number of rows deleted
                    print "Will delete %d rows starting at %d" % (numrows, startrow+1)
                    rowdelcmds.append("fitsdelrow %s[SUBINT] %d %d" % \
                                    (mergedfn, startrow+1, numrows))
                    startrow = isub-numrows # NOTE: only delete numrows because
                                            # numdelrows has already been deleted
                                            # from isub
                    num_subints -= numrows # Adjust number of subints in the file
                    print "resetting startrow to", startrow
                    numrows = 1
            for rowdelcmd in rowdelcmds:
                pipeline_utils.execute(rowdelcmd)

        # Make dat file
        prepdatacmd = "prepdata -noclip -nobary -dm 0 -o %s_post_DM0.00 %s" % (outbasenm, mergedfn)
        pipeline_utils.execute(prepdatacmd, stdout=outbasenm+"_post_prepdata.out")

        return [mergedfn]
Exemple #10
0
    def preprocess(cls, fns):
        """Given a list of filenames apply any preprocessing steps
            required. Return a list of file names of the output
            files.

            Input:
                fns: List of names of files to preprocess.

            Output:
                outfns: List of names of preprocessed files.
        """
        infiles = " ".join(fns)
        fnmatchdict = cls.fnmatch(fns[0]).groupdict()
        obsdata = cls(fns)
        outbasenm = "%(projid)s.%(date)s.%(source)s.b%(beam)s.%(scan)s" % \
                        fnmatchdict

        outfile = outbasenm + "_0001.fits"  # '0001' added is the filenumber

        # Clobber output file
        #if os.path.exists(outfile):
        #    os.remove(outfile)

        # Merge mock subbands
        mergecmd = "combine_mocks %s -o %s" % (infiles, outbasenm)
        pipeline_utils.execute(mergecmd, stdout=outbasenm + "_merge.out")

        # Rename file to remove the '_0001' that was added
        mergedfn = outbasenm + '.fits'
        os.rename(outfile, mergedfn)

        merged = autogen_dataobj([mergedfn])
        if not isinstance(merged, MergedMockPsrfitsData):
            raise ValueError("Preprocessing of Mock data has not produced " \
                                "a recognized merged file!")
        subints_with_cal = merged.get_subints_with_cal()
        num_subints = merged.num_samples / merged.num_samples_per_record
        subints_with_cal = [isub for isub in subints_with_cal \
                                if isub >=0 and isub < num_subints]

        total_removed = 0
        calrowsfile = open(outbasenm + "_calrows.txt", 'w')
        if len(subints_with_cal):
            calrowsfile.write("Subints with cal: %s\n" % \
                    ",".join(["%d" % ii for ii in sorted(subints_with_cal)]))
            rowdelcmds = []

            # Enlarge list of rows to remove if some are near
            # the beginning or end of the obs
            to_remove = []
            nearstart = [
                ii for ii in subints_with_cal if ii < 0.1 * num_subints
            ]
            nearend = [ii for ii in subints_with_cal if ii > 0.9 * num_subints]
            others = [
                ii for ii in subints_with_cal if ii not in nearstart + nearend
            ]
            if len(nearend):
                cal_start = min(nearend)
                msg = "Cal-affected region is within 10%% of end of obs.\n" \
                        "Remove all rows within cal up-to end of obs.\n" \
                        "(cal start: %d; total num rows: %d)" % \
                        (cal_start, num_subints)
                print msg
                calrowsfile.write(msg + '\n')
                numrows = num_subints - cal_start
                rowdelcmds.append("fitsdelrow %s[SUBINT] %d %d" % \
                                (mergedfn, cal_start+1, numrows))
                total_removed += numrows
            if len(nearstart):
                cal_end = max(nearstart)
                msg = "Cal-affected region is within 10%% of start of obs.\n" \
                        "remove all rows up-to and including cal.\n" \
                        "(cal end: %d; total num rows: %d)" % \
                        (cal_end, num_subints)
                print msg
                calrowsfile.write(msg + '\n')
                numrows = cal_end + 1  # Subints are counted starting with 0
                rowdelcmds.append("fitsdelrow %s[SUBINT] %d %d" % \
                                (mergedfn, 1, numrows))
                total_removed += numrows
            if len(others):
                msg = "Not removing rows in middle of obs " \
                        "(%s)." % ",".join(["%d" % ii for ii in sorted(others)])
                print msg
                calrowsfile.write(msg + '\n')
            if rowdelcmds:
                calrowsfile.write("Row-deletion commands:\n")
            else:
                calrowsfile.write("No rows to delete.\n")
            for rowdelcmd in rowdelcmds:
                calrowsfile.write(rowdelcmd + '\n')
                pipeline_utils.execute(rowdelcmd)
        msg = "Total number of rows removed: %d" % total_removed
        print msg
        calrowsfile.write(msg + '\n')
        calrowsfile.close()
        # Make dat file
        prepdatacmd = "prepdata -noclip -nobary -dm 0 -o %s_post_DM0.00 %s" % (
            outbasenm, mergedfn)
        pipeline_utils.execute(prepdatacmd,
                               stdout=outbasenm + "_post_prepdata.out")

        return [mergedfn]