def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))
        pfdpngs = [os.path.split(fn)[-1] for fn in \
                    glob.glob(os.path.join(self.directory, "*.pfd.png"))]

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % \
                                    len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])
        sigmas = []
        for c in candlist:
            base, accel = c.accelfile.split("_ACCEL_")
            pngfn = "%s_Z%s_ACCEL_Cand_%d.pfd.png" % (base, accel, c.candnum)
            if pngfn in pfdpngs:
                sigmas.append(c.sigma)
        if len(pfdpngs) > len(sigmas):
            raise DiagnosticError("Not all *.pfd.png images were found " \
                                    "in candlist! (%d > %d)" % \
                                    (len(pfdpngs), len(sigmas)))
        elif len(pfdpngs) < len(sigmas):
            raise DiagnosticError("Some *.pfd.png image match multiple " \
                                    "entries in candlist! (%d < %d)" % \
                                    (len(pfdpngs), len(sigmas)))

        self.value = min(sigmas)
Exemple #2
0
    def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))
        pfdpngs = [os.path.split(fn)[-1] for fn in \
                    glob.glob(os.path.join(self.directory, "*.pfd.png"))]

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % \
                                    len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])
        sigmas = []
        for c in candlist:
            base, accel = c.accelfile.split("_ACCEL_")
            pngfn = "%s_Z%s_ACCEL_Cand_%d.pfd.png" % (base, accel, c.candnum)
            if pngfn in pfdpngs:
                sigmas.append(c.sigma)
        if len(pfdpngs) > len(sigmas):
            raise DiagnosticError("Not all *.pfd.png images were found " \
                                    "in candlist! (%d > %d)" % \
                                    (len(pfdpngs), len(sigmas)))
        elif len(pfdpngs) < len(sigmas):
            raise DiagnosticError("Some *.pfd.png image match multiple " \
                                    "entries in candlist! (%d < %d)" % \
                                    (len(pfdpngs), len(sigmas)))

        self.value = min(sigmas)
Exemple #3
0
    def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])
        self.value = len(candlist)
Exemple #4
0
    def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % \
                                    len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])
        self.value = len(candlist)
Exemple #5
0
    def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])

        params = get_search_params(self.directory)
        self.value = len([c for c in candlist if c.sigma >= params["to_prepfold_sigma"]])
Exemple #6
0
    def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % \
                                    len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])
        
        params = get_search_params(self.directory)
        self.value = len([c for c in candlist \
                            if c.sigma >= params['to_prepfold_sigma']])
Exemple #7
0
    def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))
        pfdpngs = [os.path.split(fn)[-1] for fn in \
                    glob.glob(os.path.join(self.directory, "*_ACCEL_*.pfd.png"))]

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % \
                                    len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])
        sigmas = []
        for c in candlist:
            base, accel = c.accelfile.split("_ACCEL_")
            pngfn = "%s_Z%s_ACCEL_Cand_%d.pfd.png" % (base, accel, c.candnum)
            if pngfn in pfdpngs:
                sigmas.append(c.sigma)
        
        ffa_candlists = glob.glob(os.path.join(self.directory, "*.ffacands"))
        ffa_pfdpngs = [os.path.split(fn)[-1] for fn in \
                    glob.glob(os.path.join(self.directory, "*_ffa_*.pfd.png"))]
        if len(ffa_candlists) != 1:
            raise DiagnosticError("Wrong number of ffa candidate lists found (%d)!" % \
                                    len(ffa_candlists))
        ffa_candlist = ffacands.parse_candlist(ffa_candlists[0])
        
        #candlist.extend(ffa_candlist)
        pfdpngs.extend(ffa_pfdpngs)
        #sigmas = []
        for c in ffa_candlist:
            #base, accel = c.accelfile.split("_ACCEL_")
            #pngfn = "%s_Z%s_ACCEL_Cand_%d.pfd.png" % (base, accel, c.candnum)
            pngfn = "%s%.2fms_Cand.pfd.png"%(c.ffafile.replace("_cands.ffa","_ffa_"),c.period*1000)
            if pngfn in ffa_pfdpngs:
                sigmas.append(c.sigma)
        if len(pfdpngs) > len(sigmas):
            raise DiagnosticError("Not all *.pfd.png images were found " \
                                    "in candlist! (%d > %d)" % \
                                    (len(pfdpngs), len(sigmas)))
        elif len(pfdpngs) < len(sigmas):
            raise DiagnosticError("Some *.pfd.png image match multiple " \
                                    "entries in candlist! (%d < %d)" % \
                                    (len(pfdpngs), len(sigmas)))

        if not sigmas:
            errormsg = 'No candidates folded.'
            raise DiagnosticNonFatalError(errormsg)

        self.value = min(sigmas)
Exemple #8
0
    def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % \
                                    len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])

        # find the search_params.txt file
        paramfn = os.path.join(self.directory, 'search_params.txt')
        if os.path.exists(paramfn):
            tmp, params = {}, {}
            execfile(paramfn, tmp, params)
        else:
            raise DiagnosticError("Search parameter file doesn't exist!")
        self.value = len([c for c in candlist \
                            if c.sigma >= params['to_prepfold_sigma']])
    def get_diagnostic(self):
        # find *.accelcands file
        candlists = glob.glob(os.path.join(self.directory, "*.accelcands"))

        if len(candlists) != 1:
            raise DiagnosticError("Wrong number of candidate lists found (%d)!" % \
                                    len(candlists))
        candlist = accelcands.parse_candlist(candlists[0])
        
        # find the search_params.txt file
        paramfn = os.path.join(self.directory, 'search_params.txt')
        if os.path.exists(paramfn):
            tmp, params = {}, {}
            execfile(paramfn, tmp, params)
        else:
            raise DiagnosticError("Search parameter file doesn't exist!")
        self.value = len([c for c in candlist \
                            if c.sigma >= params['to_prepfold_sigma']])
Exemple #10
0
def get_candidates(versionnum, directory, header_id=None, timestamp_mjd=None, inst_cache=None):
    """Upload candidates to common DB.

        Inputs:
            versionnum: A combination of the githash values from 
                        PRESTO and from the pipeline. 
            directory: The directory containing results from the pipeline.
            header_id: header_id number for this beam, as returned by
                        spHeaderLoader/header.upload_header (default=None)
            timestamp_mjd: mjd timstamp for this observation (default=None).
            inst_cache: ratings2 RatingInstanceIDCache instance.

        Ouput:
            cands: List of candidates.
            tempdir: Path of temporary directory that PFDs have been untarred,
                     returned so that it can be deleted after successful PFD upload.
    """
    # find *.accelcands file    
    candlists = glob.glob(os.path.join(directory, "*.accelcands"))
    ffa_candlists = glob.glob(os.path.join(directory, "*.ffacands"))
                                                
    if len(candlists) != 1 or len(ffa_candlists) != 1:
        raise PeriodicityCandidateError("Wrong number of candidate lists found accel" \
                                        "candlists: (%d) and ffa candlists: (%d)" % \
                                            (len(candlists)), len(ffa_candlists))

    # Get list of candidates from *.accelcands file
    candlist = accelcands.parse_candlist(candlists[0])
    ffa_candlist = ffacands.parse_candlist(ffa_candlists[0])
    # find the search_params.txt file
    paramfn = os.path.join(directory, 'search_params.txt')
    if os.path.exists(paramfn):
        tmp, params = {}, {}
        execfile(paramfn, tmp, params)
    else:
        raise PeriodicityCandidateError("Search parameter file doesn't exist!")
    minsigma = params['to_prepfold_sigma']
    foldedcands = [c for c in candlist \
                    if c.sigma > params['to_prepfold_sigma']]
    foldedcands = foldedcands[:params['max_accel_cands_to_fold']]
    foldedcands.sort(reverse=True) # Sort by descending sigma

    ffa_foldedcands = [c for c in ffa_candlist \
                         if c.snr > params['to_prepfold_sigma']] 
    #                if c.snr > params['ffa_snr']]# need something for ffa
    ffa_foldedcands = ffa_foldedcands[:params['max_ffa_cands_to_fold']]
    ffa_foldedcands.sort(reverse=True) # Sort by descending snr
    # Open attribute file
    attrib_fn = os.path.join(directory, 'candidate_attributes.txt')
    attribs = np.loadtxt(attrib_fn,dtype='S')
        
    # Create temporary directory
    N = 6
    prefix = "/localscratch/PALFA_pfds_"
    suffix = "_tmp/"
    String = ''.join(random.choice(string.ascii_uppercase + string.digits + string.ascii_lowercase) for _ in range(N))
    tempdir = prefix+String+suffix
    os.makedirs(tempdir)
    #tempdir = tempname

    if foldedcands or ffa_foldedcands:

        pfd_tarfns = glob.glob(os.path.join(directory, "*_pfd.tgz"))

        if len(pfd_tarfns) != 1:
            raise PeriodicityCandidateError("Wrong number (%d) of *_pfd.tgz " \
                                             "files found in %s" % (len(pfd_tarfns), \
                                                directory))

        rating_tarfns = glob.glob(os.path.join(directory, "*_pfd_rat.tgz"))
        if len(rating_tarfns) != 1:
            raise PeriodicityCandidateError("Wrong number (%d) of *_pfd_rat.tgz " \
                                             "files found in %s" % (len(rating_tarfns), \
                                                directory))

        mjd = int(timestamp_mjd)
        remote_pfd_base = os.path.join(config.upload.pfd_ftp_dir,str(mjd)) 
        remote_pfd_dir = os.path.join(remote_pfd_base,\
                                      os.path.basename(pfd_tarfns[0]).rstrip('_pfd.tgz'))
        pfd_tarball = PFDTarball(pfd_tarfns[0],remote_pfd_base,tempdir)
        pfd_tempdir, pfd_list = pfd_tarball.extract()
        
        # extract ratings tarball 
        tar = tarfile.open(rating_tarfns[0])
        try:
            tar.extractall(path=tempdir)
        except IOError:
            if os.path.isdir(tempdir):
                shutil.rmtree(tempdir)
            raise PeriodicityCandidateError("Error while extracting pfd files " \
                                            "from tarball (%s)!" % tarfn)
        finally:
            tar.close()

    # Loop over candidates that were folded
    cands = []
    cands.append(pfd_tarball)
    counter = 0
    for ii, c in enumerate(foldedcands):
        basefn = "%s_ACCEL_Cand_%d" % (c.accelfile.replace("ACCEL_", "Z"), \
                                    c.candnum)
        pfdfn = os.path.join(pfd_tempdir, basefn+".pfd")
        pngfn = os.path.join(directory, basefn+".pfd.png")
        ratfn = os.path.join(tempdir, basefn+".pfd.rat")

        pfd = prepfold.pfd(pfdfn)
        cand_attribs = dict(attribs[attribs[:,0] == basefn+".pfd"][:,1:])
        
        try:
            cand = PeriodicityCandidate(ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma, \
                                    c.period, c.dm, cand_attribs, c.search_type, \
                                    header_id=header_id)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)
        pfd_size = dict(pfd_list)[pfdfn]
        cand.add_dependent(PeriodicityCandidatePFD(pfdfn, pfd_size, remote_pfd_dir=remote_pfd_dir))
        cand.add_dependent(PeriodicityCandidatePNG(pngfn))

        ratvals = ratings2.rating_value.read_file(ratfn)
        cand.add_dependent(PeriodicityCandidateRating(ratvals,inst_cache=inst_cache))
        cands.append(cand)
        counter +=1
        
    for ii, c in enumerate(ffa_foldedcands):
        basefn = "%s%.2fms_Cand" % (c.ffafile.replace("_cands.ffa","_ffa_"), c.period*1000)
        pfdfn = os.path.join(pfd_tempdir, basefn+".pfd")
        pngfn = os.path.join(directory, basefn+".pfd.png")
        ratfn = os.path.join(tempdir, basefn+".pfd.rat")

        pfd = prepfold.pfd(pfdfn)
        cand_attribs = dict(attribs[attribs[:,0] == basefn+".pfd"][:,1:])
        
        try:
            cand = PeriodicityCandidate(counter+ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma, \
                                    c.period, c.dm, cand_attribs, c.search_type, \
                                    header_id=header_id)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)
        pfd_size = dict(pfd_list)[pfdfn]
        cand.add_dependent(PeriodicityCandidatePFD(pfdfn, pfd_size, remote_pfd_dir=remote_pfd_dir))
        cand.add_dependent(PeriodicityCandidatePNG(pngfn))

        ratvals = ratings2.rating_value.read_file(ratfn)
        cand.add_dependent(PeriodicityCandidateRating(ratvals,inst_cache=inst_cache))
        cands.append(cand)
        
    #shutil.rmtree(tempdir)
    return cands,tempdir
def check_candidates(header_id, versionnum, directory, dbname='common-copy'):
    """Check candidates in common DB.

        Inputs:
            header_id: header_id number for this beam, as returned by
                        spHeaderLoader/header.upload_header
            versionnum: A combination of the githash values from 
                        PRESTO and from the pipeline. 
            directory: The directory containing results from the pipeline.
            dbname: Name of database to connect to, or a database
                        connection to use (Defaut: 'common-copy').
        Output:
            match: Boolean value. True if all candidates and plots match what
                    is in the DB, False otherwise.
    """
    # find *.accelcands file
    candlists = glob.glob(os.path.join(directory, "*.accelcands"))

    if len(candlists) != 1:
        raise PeriodicityCandidateError("Wrong number of candidate lists found (%d)!" % \
                                            len(candlists))

    # Get list of candidates from *.accelcands file
    candlist = accelcands.parse_candlist(candlists[0])
    minsigma = config.searching.to_prepfold_sigma
    foldedcands = [c for c in candlist if c.sigma > minsigma]
    foldedcands = foldedcands[:config.searching.max_cands_to_fold]
    foldedcands.sort(reverse=True)  # Sort by descending sigma

    # Create temporary directory
    tempdir = tempfile.mkdtemp(suffix="_tmp", prefix="PALFA_pfds_")
    tarfns = glob.glob(os.path.join(directory, "*_pfd.tgz"))
    if len(tarfns) != 1:
        raise PeriodicityCandidateError("Wrong number (%d) of *_pfd.tgz " \
                                         "files found in %s" % (len(tarfns), \
                                            directory))

    tar = tarfile.open(tarfns[0])
    try:
        tar.extractall(path=tempdir)
    except IOError:
        if os.path.isdir(tempdir):
            shutil.rmtree(tempdir)
        raise PeriodicityCandidateError("Error while extracting pfd files " \
                                        "from tarball (%s)!" % tarfns[0])
    finally:
        tar.close()
    # Loop over candidates that were folded
    matches = []
    if isinstance(dbname, database.Database):
        db = dbname
    else:
        db = database.Database(dbname)
    for ii, c in enumerate(foldedcands):
        basefn = "%s_ACCEL_Cand_%d" % (c.accelfile.replace("ACCEL_", "Z"), \
                                    c.candnum)
        pfdfn = os.path.join(tempdir, basefn + ".pfd")
        pngfn = os.path.join(directory, basefn + ".pfd.png")

        pfd = prepfold.pfd(pfdfn)

        try:
            cand = PeridocityCandidate(header_id, ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)

        matches.append(cand.compare_with_db(dbname))
        if not matches[-1]:
            break

        # Get candidate's ID number from common DB
        db.execute("SELECT pdm_cand_id " \
                   "FROM pdm_candidates AS c " \
                   "LEFT JOIN versions AS v ON v.version_id = c.version_id " \
                   "WHERE c.header_id=%d AND c.cand_num=%d " \
                        "AND v.version_number='%s'" % \
                        (header_id, cand.cand_num, versionnum))
        # For cand.compare_with_db() to return True there must be a unique
        # entry in the common DB matching this candidate
        r = db.cursor.fetchone()
        cand_id = r[0]

        pfdplot = PeriodicityCandidatePFD(cand_id, pfdfn)
        matches.append(pfdplot.compare_with_db(dbname))
        if not matches[-1]:
            break

        pngplot = PeriodicityCandidatePNG(cand_id, pngfn)
        matches.append(pngplot.compare_with_db(dbname))
        if not matches[-1]:
            break

    if type(dbname) == types.StringType:
        db.close()
    shutil.rmtree(tempdir)
    return all(matches)
def upload_candidates(header_id, versionnum, directory, verbose=False, \
                        dry_run=False, *args, **kwargs):
    """Upload candidates to common DB.

        Inputs:
            header_id: header_id number for this beam, as returned by
                        spHeaderLoader/header.upload_header
            versionnum: A combination of the githash values from 
                        PRESTO and from the pipeline. 
            directory: The directory containing results from the pipeline.
            verbose: An optional boolean value that determines if information 
                        is printed to stdout.
            dry_run: An optional boolean value. If True no connection to DB
                        will be made and DB command will not be executed.
                        (If verbose is True DB command will be printed 
                        to stdout.)

            *** NOTE: Additional arguments are passed to the uploader function.

        Ouputs:
            cand_ids: List of candidate IDs corresponding to these candidates
                        in the common DB. (Or a list of None values if
                        dry_run is True).
    """
    # find *.accelcands file
    candlists = glob.glob(os.path.join(directory, "*.accelcands"))

    if len(candlists) != 1:
        raise PeriodicityCandidateError("Wrong number of candidate lists found (%d)!" % \
                                            len(candlists))

    # Get list of candidates from *.accelcands file
    candlist = accelcands.parse_candlist(candlists[0])
    minsigma = config.searching.to_prepfold_sigma
    foldedcands = [c for c in candlist if c.sigma > minsigma]
    foldedcands = foldedcands[:config.searching.max_cands_to_fold]
    foldedcands.sort(reverse=True)  # Sort by descending sigma

    # Create temporary directory
    tempdir = tempfile.mkdtemp(suffix="_tmp", prefix="PALFA_pfds_")
    tarfns = glob.glob(os.path.join(directory, "*_pfd.tgz"))
    if len(tarfns) != 1:
        raise PeriodicityCandidateError("Wrong number (%d) of *_pfd.tgz " \
                                         "files found in %s" % (len(tarfns), \
                                            directory))

    tar = tarfile.open(tarfns[0])
    try:
        tar.extractall(path=tempdir)
    except IOError:
        if os.path.isdir(tempdir):
            shutil.rmtree(tempdir)
        raise PeriodicityCandidateError("Error while extracting pfd files " \
                                        "from tarball (%s)!" % tarfns[0])
    finally:
        tar.close()
    # Loop over candidates that were folded
    results = []
    for ii, c in enumerate(foldedcands):
        basefn = "%s_ACCEL_Cand_%d" % (c.accelfile.replace("ACCEL_", "Z"), \
                                    c.candnum)
        pfdfn = os.path.join(tempdir, basefn + ".pfd")
        pngfn = os.path.join(directory, basefn + ".pfd.png")

        pfd = prepfold.pfd(pfdfn)

        try:
            cand = PeridocityCandidate(header_id, ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)

        if dry_run:
            cand.get_upload_sproc_call()
            if verbose:
                print cand
            results.append(None)
            cand_id = -1
        else:
            cand_id = cand.upload(*args, **kwargs)

        pfdplot = PeriodicityCandidatePFD(cand_id, pfdfn)
        pngplot = PeriodicityCandidatePNG(cand_id, pngfn)
        if dry_run:
            pfdplot.get_upload_sproc_call()
            pngplot.get_upload_sproc_call()
            if verbose:
                print pfdplot
                print pngplot
        else:
            pfdplot.upload(*args, **kwargs)
            pngplot.upload(*args, **kwargs)

    shutil.rmtree(tempdir)
    return results
def upload_candidates(header_id, versionnum, directory, verbose=False, \
                        dry_run=False, *args, **kwargs):
    """Upload candidates to common DB.

        Inputs:
            header_id: header_id number for this beam, as returned by
                        spHeaderLoader/header.upload_header
            versionnum: A combination of the githash values from 
                        PRESTO and from the pipeline. 
            directory: The directory containing results from the pipeline.
            verbose: An optional boolean value that determines if information 
                        is printed to stdout.
            dry_run: An optional boolean value. If True no connection to DB
                        will be made and DB command will not be executed.
                        (If verbose is True DB command will be printed 
                        to stdout.)

            *** NOTE: Additional arguments are passed to the uploader function.

        Ouputs:
            cand_ids: List of candidate IDs corresponding to these candidates
                        in the common DB. (Or a list of None values if
                        dry_run is True).
    """
    # find *.accelcands file    
    candlists = glob.glob(os.path.join(directory, "*.accelcands"))
                                                
    if len(candlists) != 1:
        raise PeriodicityCandidateError("Wrong number of candidate lists found (%d)!" % \
                                            len(candlists))

    # Get list of candidates from *.accelcands file
    candlist = accelcands.parse_candlist(candlists[0])
    minsigma = config.searching.to_prepfold_sigma
    foldedcands = [c for c in candlist if c.sigma > minsigma]
    foldedcands = foldedcands[:config.searching.max_cands_to_fold]
    foldedcands.sort(reverse=True) # Sort by descending sigma
    
    # Create temporary directory
    tempdir = tempfile.mkdtemp(suffix="_tmp", prefix="PALFA_pfds_")
    tarfns = glob.glob(os.path.join(directory, "*_pfd.tgz"))
    if len(tarfns) != 1:
        raise PeriodicityCandidateError("Wrong number (%d) of *_pfd.tgz " \
                                         "files found in %s" % (len(tarfns), \
                                            directory))
    tar = tarfile.open(tarfns[0])
    tar.extractall(path=tempdir)
    tar.close()
    # Loop over candidates that were folded
    results = []
    for ii, c in enumerate(foldedcands):
        basefn = "%s_ACCEL_Cand_%d" % (c.accelfile.replace("ACCEL_", "Z"), \
                                    c.candnum)
        pfdfn = os.path.join(tempdir, basefn+".pfd")
        pngfn = os.path.join(directory, basefn+".pfd.png")
        
        pfd = prepfold.pfd(pfdfn)
        
        try:
            cand = PeridocityCandidate(header_id, ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)

        if dry_run:
            cand.get_upload_sproc_call()
            if verbose:
                print cand
            results.append(None)
            cand_id = -1
        else:
            cand_id = cand.upload(*args, **kwargs)
        
        pfdplot = PeriodicityCandidatePFD(cand_id, pfdfn)
        pngplot = PeriodicityCandidatePNG(cand_id, pngfn)
        if dry_run:
            pfdplot.get_upload_sproc_call()
            pngplot.get_upload_sproc_call()
            if verbose:
                print pfdplot
                print pngplot
        else:
            pfdplot.upload(*args, **kwargs)
            pngplot.upload(*args, **kwargs)
        
    shutil.rmtree(tempdir)
    return results
Exemple #14
0
def get_candidates(versionnum, directory, header_id=None, timestamp_mjd=None, inst_cache=None):
    """Upload candidates to common DB.

        Inputs:
            versionnum: A combination of the githash values from 
                        PRESTO and from the pipeline. 
            directory: The directory containing results from the pipeline.
            header_id: header_id number for this beam, as returned by
                        spHeaderLoader/header.upload_header (default=None)
            timestamp_mjd: mjd timstamp for this observation (default=None).
            inst_cache: ratings2 RatingInstanceIDCache instance.

        Ouput:
            cands: List of candidates.
            tempdir: Path of temporary directory that PFDs have been untarred,
                     returned so that it can be deleted after successful PFD upload.
    """
    # find *.accelcands file    
    candlists = glob.glob(os.path.join(directory, "*.accelcands"))
                                                
    if len(candlists) != 1:
        raise PeriodicityCandidateError("Wrong number of candidate lists found (%d)!" % \
                                            len(candlists))

    # Get list of candidates from *.accelcands file
    candlist = accelcands.parse_candlist(candlists[0])
    # find the search_params.txt file
    paramfn = os.path.join(directory, 'search_params.txt')
    if os.path.exists(paramfn):
        tmp, params = {}, {}
        execfile(paramfn, tmp, params)
    else:
        raise PeriodicityCandidateError("Search parameter file doesn't exist!")
    minsigma = params['to_prepfold_sigma']
    foldedcands = [c for c in candlist \
                    if c.sigma > params['to_prepfold_sigma']]
    foldedcands = foldedcands[:params['max_cands_to_fold']]
    foldedcands.sort(reverse=True) # Sort by descending sigma

    # Open attribute file
    attrib_fn = os.path.join(directory, 'candidate_attributes.txt')
    attribs = np.loadtxt(attrib_fn,dtype='S')
        
    # Create temporary directory
    tempdir = tempfile.mkdtemp(suffix="_tmp", prefix="pfds_", dir="/sps/hep/glast/data/survey_pulsar/")

    if foldedcands:

        pfd_tarfns = glob.glob(os.path.join(directory, "*_pfd.tgz"))
        if len(pfd_tarfns) != 1:
            raise PeriodicityCandidateError("Wrong number (%d) of *_pfd.tgz " \
                                             "files found in %s" % (len(pfd_tarfns), \
                                                directory))

        bestprof_tarfns = glob.glob(os.path.join(directory, "*_bestprof.tgz"))
        if len(bestprof_tarfns) != 1:
            raise PeriodicityCandidateError("Wrong number (%d) of *_bestprof.tgz " \
                                             "files found in %s" % (len(bestprof_tarfns), \
                                                directory))

        rating_tarfns = glob.glob(os.path.join(directory, "*_pfd_rat.tgz"))
        if len(rating_tarfns) != 1:
            raise PeriodicityCandidateError("Wrong number (%d) of *_pfd_rat.tgz " \
                                             "files found in %s" % (len(rating_tarfns), \
                                                directory))

        for tarfn in [ pfd_tarfns[0], bestprof_tarfns[0], rating_tarfns[0] ]: 
            tar = tarfile.open(tarfn)
            try:
                tar.extractall(path=tempdir)
            except IOError:
                if os.path.isdir(tempdir):
                    shutil.rmtree(tempdir)
                raise PeriodicityCandidateError("Error while extracting pfd files " \
                                                "from tarball (%s)!" % tarfn)
            finally:
                tar.close()

    # Loop over candidates that were folded
    cands = []
    for ii, c in enumerate(foldedcands):
        basefn = "%s_ACCEL_Cand_%d" % (c.accelfile.replace("ACCEL_", "Z"), \
                                    c.candnum)
        pfdfn = os.path.join(tempdir, basefn+".pfd")
        pngfn = os.path.join(directory, basefn+".pfd.png")
        ratfn = os.path.join(tempdir, basefn+".pfd.rat")

        pfd = prepfold.pfd(pfdfn)
        cand_attribs = dict(attribs[attribs[:,0] == basefn+".pfd"][:,1:])
        
        try:
            cand = PeriodicityCandidate(ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma, \
                                    c.period, c.dm, cand_attribs, header_id=header_id)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)
        cand.add_dependent(PeriodicityCandidatePFD(pfdfn, timestamp_mjd=timestamp_mjd))
        cand.add_dependent(PeriodicityCandidatePNG(pngfn))

        ratvals = ratings2.rating_value.read_file(ratfn)
        cand.add_dependent(PeriodicityCandidateRating(ratvals,inst_cache=inst_cache))
        cands.append(cand)
        
    #shutil.rmtree(tempdir)
    return cands,tempdir
Exemple #15
0
def get_candidates(versionnum, directory, header_id=None):
    """Upload candidates to common DB.

        Inputs:
            versionnum: A combination of the githash values from 
                        PRESTO and from the pipeline. 
            directory: The directory containing results from the pipeline.
            header_id: header_id number for this beam, as returned by
                        spHeaderLoader/header.upload_header (default=None)

        Ouput:
            cands: List of candidates.
    """
    # find *.accelcands file
    candlists = glob.glob(os.path.join(directory, "*.accelcands"))

    if len(candlists) != 1:
        raise PeriodicityCandidateError("Wrong number of candidate lists found (%d)!" % \
                                            len(candlists))

    # Get list of candidates from *.accelcands file
    candlist = accelcands.parse_candlist(candlists[0])
    # find the search_params.txt file
    paramfn = os.path.join(directory, 'search_params.txt')
    if os.path.exists(paramfn):
        tmp, params = {}, {}
        execfile(paramfn, tmp, params)
    else:
        raise PeriodicityCandidateError("Search parameter file doesn't exist!")
    minsigma = params['to_prepfold_sigma']
    foldedcands = [c for c in candlist \
                    if c.sigma > params['to_prepfold_sigma']]
    foldedcands = foldedcands[:params['max_cands_to_fold']]
    foldedcands.sort(reverse=True)  # Sort by descending sigma

    # Create temporary directory
    tempdir = tempfile.mkdtemp(suffix="_tmp", prefix="PALFA_pfds_")

    if foldedcands:

        tarfns = glob.glob(os.path.join(directory, "*_pfd.tgz"))
        if len(tarfns) != 1:
            raise PeriodicityCandidateError("Wrong number (%d) of *_pfd.tgz " \
                                             "files found in %s" % (len(tarfns), \
                                                directory))

        tar = tarfile.open(tarfns[0])
        try:
            tar.extractall(path=tempdir)
        except IOError:
            if os.path.isdir(tempdir):
                shutil.rmtree(tempdir)
            raise PeriodicityCandidateError("Error while extracting pfd files " \
                                            "from tarball (%s)!" % tarfns[0])
        finally:
            tar.close()

    # Loop over candidates that were folded
    cands = []
    for ii, c in enumerate(foldedcands):
        basefn = "%s_ACCEL_Cand_%d" % (c.accelfile.replace("ACCEL_", "Z"), \
                                    c.candnum)
        pfdfn = os.path.join(tempdir, basefn + ".pfd")
        pngfn = os.path.join(directory, basefn + ".pfd.png")

        pfd = prepfold.pfd(pfdfn)

        try:
            cand = PeriodicityCandidate(ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma, \
                                    header_id=header_id)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)

        cand.add_dependent(PeriodicityCandidatePFD(pfdfn))
        cand.add_dependent(PeriodicityCandidatePNG(pngfn))
        cands.append(cand)

    shutil.rmtree(tempdir)
    return cands
Exemple #16
0
def get_candidates(versionnum, directory, header_id=None):
    """Upload candidates to common DB.

        Inputs:
            versionnum: A combination of the githash values from 
                        PRESTO and from the pipeline. 
            directory: The directory containing results from the pipeline.
            header_id: header_id number for this beam, as returned by
                        spHeaderLoader/header.upload_header (default=None)

        Ouput:
            cands: List of candidates.
    """
    # find *.accelcands file    
    candlists = glob.glob(os.path.join(directory, "*.accelcands"))
                                                
    if len(candlists) != 1:
        raise PeriodicityCandidateError("Wrong number of candidate lists found (%d)!" % \
                                            len(candlists))

    # Get list of candidates from *.accelcands file
    candlist = accelcands.parse_candlist(candlists[0])
    # find the search_params.txt file
    paramfn = os.path.join(directory, 'search_params.txt')
    if os.path.exists(paramfn):
        tmp, params = {}, {}
        execfile(paramfn, tmp, params)
    else:
        raise PeriodicityCandidateError("Search parameter file doesn't exist!")
    minsigma = params['to_prepfold_sigma']
    foldedcands = [c for c in candlist \
                    if c.sigma > params['to_prepfold_sigma']]
    foldedcands = foldedcands[:params['max_cands_to_fold']]
    foldedcands.sort(reverse=True) # Sort by descending sigma

        
    # Create temporary directory
    tempdir = tempfile.mkdtemp(suffix="_tmp", prefix="PALFA_pfds_")

    if foldedcands:

        tarfns = glob.glob(os.path.join(directory, "*_pfd.tgz"))
        if len(tarfns) != 1:
            raise PeriodicityCandidateError("Wrong number (%d) of *_pfd.tgz " \
                                             "files found in %s" % (len(tarfns), \
                                                directory))
        
        tar = tarfile.open(tarfns[0])
        try:
            tar.extractall(path=tempdir)
        except IOError:
            if os.path.isdir(tempdir):
                shutil.rmtree(tempdir)
            raise PeriodicityCandidateError("Error while extracting pfd files " \
                                            "from tarball (%s)!" % tarfns[0])
        finally:
            tar.close()

    # Loop over candidates that were folded
    cands = []
    for ii, c in enumerate(foldedcands):
        basefn = "%s_ACCEL_Cand_%d" % (c.accelfile.replace("ACCEL_", "Z"), \
                                    c.candnum)
        pfdfn = os.path.join(tempdir, basefn+".pfd")
        pngfn = os.path.join(directory, basefn+".pfd.png")
        
        pfd = prepfold.pfd(pfdfn)
        
        try:
            cand = PeriodicityCandidate(ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma, \
                                    header_id=header_id)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)


        cand.add_dependent(PeriodicityCandidatePFD(pfdfn))
        cand.add_dependent(PeriodicityCandidatePNG(pngfn))
        cands.append(cand)
        
    shutil.rmtree(tempdir)
    return cands
def check_candidates(header_id, versionnum, directory, dbname='common-copy'):
    """Check candidates in common DB.

        Inputs:
            header_id: header_id number for this beam, as returned by
                        spHeaderLoader/header.upload_header
            versionnum: A combination of the githash values from 
                        PRESTO and from the pipeline. 
            directory: The directory containing results from the pipeline.
            dbname: Name of database to connect to, or a database
                        connection to use (Defaut: 'common-copy').
        Output:
            match: Boolean value. True if all candidates and plots match what
                    is in the DB, False otherwise.
    """
    # find *.accelcands file    
    candlists = glob.glob(os.path.join(directory, "*.accelcands"))
                                                
    if len(candlists) != 1:
        raise PeriodicityCandidateError("Wrong number of candidate lists found (%d)!" % \
                                            len(candlists))

    # Get list of candidates from *.accelcands file
    candlist = accelcands.parse_candlist(candlists[0])
    minsigma = config.searching.to_prepfold_sigma
    foldedcands = [c for c in candlist if c.sigma > minsigma]
    foldedcands = foldedcands[:config.searching.max_cands_to_fold]
    foldedcands.sort(reverse=True) # Sort by descending sigma
    
    # Create temporary directory
    tempdir = tempfile.mkdtemp(suffix="_tmp", prefix="PALFA_pfds_")
    tarfns = glob.glob(os.path.join(directory, "*_pfd.tgz"))
    if len(tarfns) != 1:
        raise PeriodicityCandidateError("Wrong number (%d) of *_pfd.tgz " \
                                         "files found in %s" % (len(tarfns), \
                                            directory))
    
    tar = tarfile.open(tarfns[0])
    try:
        tar.extractall(path=tempdir)
    except IOError:
        if os.path.isdir(tempdir):
            shutil.rmtree(tempdir)
        raise PeriodicityCandidateError("Error while extracting pfd files " \
                                        "from tarball (%s)!" % tarfns[0])
    finally:
        tar.close()
    # Loop over candidates that were folded
    matches = []
    if isinstance(dbname, database.Database):
        db = dbname
    else:
        db = database.Database(dbname)
    for ii, c in enumerate(foldedcands):
        basefn = "%s_ACCEL_Cand_%d" % (c.accelfile.replace("ACCEL_", "Z"), \
                                    c.candnum)
        pfdfn = os.path.join(tempdir, basefn+".pfd")
        pngfn = os.path.join(directory, basefn+".pfd.png")
        
        pfd = prepfold.pfd(pfdfn)
        
        try:
            cand = PeridocityCandidate(header_id, ii+1, pfd, c.snr, \
                                    c.cpow, c.ipow, len(c.dmhits), \
                                    c.numharm, versionnum, c.sigma)
        except Exception:
            raise PeriodicityCandidateError("PeriodicityCandidate could not be " \
                                            "created (%s)!" % pfdfn)
        
        matches.append(cand.compare_with_db(dbname))
        if not matches[-1]:
            break
        
        # Get candidate's ID number from common DB
        db.execute("SELECT pdm_cand_id " \
                   "FROM pdm_candidates AS c " \
                   "LEFT JOIN versions AS v ON v.version_id = c.version_id " \
                   "WHERE c.header_id=%d AND c.cand_num=%d " \
                        "AND v.version_number='%s'" % \
                        (header_id, cand.cand_num, versionnum))
        # For cand.compare_with_db() to return True there must be a unique
        # entry in the common DB matching this candidate
        r = db.cursor.fetchone()
        cand_id = r[0]
        
        pfdplot = PeriodicityCandidatePFD(cand_id, pfdfn)
        matches.append(pfdplot.compare_with_db(dbname))
        if not matches[-1]:
            break

        pngplot = PeriodicityCandidatePNG(cand_id, pngfn)
        matches.append(pngplot.compare_with_db(dbname))
        if not matches[-1]:
            break
        
    if type(dbname) == types.StringType:
        db.close()
    shutil.rmtree(tempdir)
    return all(matches)