Esempio n. 1
0
def cand2h5(cand_val):
    """
    TODO: Add option to use cand.resize for reshaping FT and DMT
    Generates h5 file of candidate with resized frequency-time and DM-time arrays
    :param cand_val: List of candidate parameters (filename, snr, width, dm, label, tcand(s))
    :type cand_val: Candidate
    :return: None
    """
    filename, snr, width, dm, label, tcand, kill_mask_path, args, gpu_id = cand_val
    if os.path.exists(str(kill_mask_path)):
        logger.info(f'Using mask {kill_mask_path}')
        kill_chans = np.loadtxt(kill_mask_path, dtype=np.int)
    else:
        logger.debug('No Kill Mask')

    fname, ext = os.path.splitext(filename)
    if ext == ".fits" or ext == ".sf":
        files = glob.glob(fname[:-5] + '*fits')
    elif ext == '.fil':
        files = [filename]
    else:
        raise TypeError("Can only work with list of fits file or filterbanks")

    logger.debug(f'Source file list: {files}')
    cand = Candidate(files, snr=snr, width=width, dm=dm, label=label, tcand=tcand, device=gpu_id)
    if os.path.exists(str(kill_mask_path)):
        kill_mask = np.zeros(cand.nchans, dtype=np.bool)
        kill_mask[kill_chans] = True
        cand.kill_mask = kill_mask
    cand.get_chunk()
    if cand.isfil:
        cand.fp.close()

    logger.info('Got Chunk')

    if gpu_id >= 0:
        logger.debug(f"Using the GPU {gpu_id}")
        try:
            cand = gpu_dedisp_and_dmt_crop(cand, device=gpu_id)
        except CudaAPIError:
            logger.info("Ran into a CudaAPIError, using the CPU version for this candidate")
            cand = cpu_dedisp_dmt(cand, args)
    else:
        cand = cpu_dedisp_dmt(cand, args)

    cand.resize(key='ft', size=args.frequency_size, axis=1, anti_aliasing=True, mode='constant')
    logger.info(f'Resized Frequency axis of FT to fsize: {cand.dedispersed.shape[1]}')
    cand.dmt = normalise(cand.dmt)
    cand.dedispersed = normalise(cand.dedispersed)
    fout = cand.save_h5(out_dir=args.fout)
    logger.debug(f"Filesize is {os.path.getsize(fout)}")
    if not os.path.isfile(fout):
        raise IOError(f"File with {cand.id} not written")
    if os.path.getsize(fout) < 200 * 1024:
        raise ValueError(f"File with {cand.id} has issues!")
    logger.info(fout)
    return None
Esempio n. 2
0
def cand2h5(cand_val):
    """
    TODO: Add option to use cand.resize for reshaping FT and DMT
    Generates h5 file of candidate with resized frequency-time and DM-time arrays
    :param cand_val: List of candidate parameters (filename, snr, width, dm, label, tcand(s))
    :type cand_val: Candidate
    :return: None
    """
    (
        filename,
        snr,
        width,
        dm,
        label,
        tcand,
        kill_mask_path,
        num_files,
        args,
        gpu_id,
    ) = cand_val
    if os.path.exists(str(kill_mask_path)):
        logger.info(f"Using mask {kill_mask_path}")
        kill_chans = np.loadtxt(kill_mask_path, dtype=np.int)
    else:
        logger.debug("No Kill Mask")

    fname, ext = os.path.splitext(filename)
    if ext == ".fits" or ext == ".sf":
        if num_files == 1:
            files = [filename]
        else:
            files = glob.glob(fname[:-5] + "*fits")
            if len(files) != num_files:
                raise ValueError(
                    "Number of fits files found was not equal to num_files in cand csv."
                )
    elif ext == ".fil":
        files = [filename]
    else:
        raise TypeError("Can only work with list of fits file or filterbanks")

    logger.debug(f"Source file list: {files}")
    cand = Candidate(
        files,
        snr=snr,
        width=width,
        dm=dm,
        label=label,
        tcand=tcand,
        device=gpu_id,
        spectral_kurtosis_sigma=args.spectral_kurtosis_sigma,
        savgol_frequency_window=args.savgol_frequency_window,
        savgol_sigma=args.savgol_sigma,
        flag_rfi=args.flag_rfi,
    )
    if os.path.exists(str(kill_mask_path)):
        kill_mask = np.zeros(cand.nchans, dtype=np.bool)
        kill_mask[kill_chans] = True
        cand.kill_mask = kill_mask
    cand.get_chunk(for_preprocessing=True)
    if cand.format == "fil":
        cand.fp.close()

    logger.info("Got Chunk")

    if gpu_id >= 0:
        logger.debug(f"Using the GPU {gpu_id}")
        try:
            cand = gpu_dedisp_and_dmt_crop(cand, device=gpu_id)
        except CudaAPIError:
            logger.info(
                "Ran into a CudaAPIError, using the CPU version for this candidate"
            )
            cand = cpu_dedisp_dmt(cand, args)
    else:
        cand = cpu_dedisp_dmt(cand, args)

    cand.resize(key="ft",
                size=args.frequency_size,
                axis=1,
                anti_aliasing=True,
                mode="constant")
    logger.info(
        f"Resized Frequency axis of FT to fsize: {cand.dedispersed.shape[1]}")
    cand.dmt = normalise(cand.dmt)
    cand.dedispersed = normalise(cand.dedispersed)
    fout = cand.save_h5(out_dir=args.fout)
    logger.debug(f"Filesize of {fout} is {os.path.getsize(fout)}")
    if not os.path.isfile(fout):
        raise IOError(f"File with {cand.id} not written")
    if os.path.getsize(fout) < 100 * 1024:
        raise ValueError(
            f"File with id: {cand.id} has issues! Its size is too less.")
    logger.info(fout)
    del cand
    return None