def reduce(telescope, obskey, tstart, tend, nchan, ngate, ntbin, ntw_min,
           rfi_filter_raw=None, fref=None, dedisperse=None,
           rfi_filter_power=None, do_waterfall=True, do_foldspec=True,
           verbose=True):

    comm = MPI.COMM_WORLD
    if dedisperse == 'None':
        dedisperse = None

    if verbose > 3 and comm.rank == 0:
        print(telescope, obskey, tstart, tend, nchan, ngate, ntbin, ntw_min,
              rfi_filter_raw, fref, dedisperse,
              do_waterfall, do_foldspec, verbose)

    if dedisperse is not None and fref is None:
        raise ValueError("Need to give reference frequency to dedisperse to")

    obs = obsdata()
    if verbose > 3 and comm.rank == 0:
        print(obs)
    # find nearest observation to 'date',
    # warning if > 1s off requested start time of observation
    if obskey not in obs[telescope]:
        # assume it is a date, which may be only good to nearest second
        obskey = obs[telescope].nearest_observation(obskey)
    # target of this observation
    psr = obs[telescope][obskey]['src']
    assert psr in obs['psrs'].keys()

    dm = obs['psrs'][psr]['dm']

    if verbose and comm.rank == 0:
        print("Attempting to open {0}: {1}".format(telescope, obskey))

    with obs[telescope].open(obskey, comm=comm) as fh:
        if verbose and comm.rank == 0:
            print("Opened files")

        # nchan = None means data is channelized already, so we get this
        # property directly from the file
        if nchan is None:
            nchan = fh.nchan
        # ensure requested number of channels is integer multiple of
        # existing channels
        if nchan % getattr(fh, 'nchan', 1) != 0:
            raise ValueError("Can only channelize data to an integer multiple "
                             "of the number of input channels (={0})."
                             .format(fh.nchan))

        time0 = fh.time0
        tstart = time0 if tstart is None else Time(tstart, scale='utc')
        if tend is None:
            tend = obs[telescope][obskey]['tend']

        try:
            tend = Time(tend, scale='utc')
            dt = tend - tstart
        except ValueError:
            dt = TimeDelta(float(tend), format='sec')
            tend = tstart + dt

        if verbose and comm.rank == 0:
            print("Requested time span: {0} to {1}".format(tstart.isot,
                                                           tend.isot))

        phasepol = obs[telescope][obskey].get_phasepol(time0)
        nt = fh.ntimebins(tstart, tend)
        ntint = fh.ntint(nchan if fh.nchan == 1 else fh.nchan)
        # number of samples to combine for waterfall
        ntw = min(ntw_min, nt*ntint)
        # number of records to skip

        if verbose and comm.rank == 0:
            print("Using start time {0} and phase polynomial {1}"
                  .format(time0.isot, phasepol))

        fh.seek(tstart)
        if verbose and comm.rank == 0:
            print("Skipped {0} blocks and will fold {1} blocks to cover "
                  "time span {2} to {3}"
                  .format(fh.offset/fh.blocksize, nt, fh.time().isot,
                          fh.time(fh.offset + nt*fh.blocksize).isot))

        # set the default parameters to fold
        # Note, some parameters may be in fh's HDUs, or fh.__getitem__
        # but these are overwritten if explicitly sprecified in Folder
        folder = Folder(fh, nchan=nchan,
                        nt=nt, ntint=ntint, ngate=ngate,
                        ntbin=ntbin, ntw=ntw, dm=dm, fref=fref,
                        phasepol=phasepol, dedisperse=dedisperse,
                        do_waterfall=do_waterfall, do_foldspec=do_foldspec,
                        verbose=verbose, progress_interval=1,
                        rfi_filter_raw=rfi_filter_raw,
                        rfi_filter_power=rfi_filter_power)
        myfoldspec, myicount, mywaterfall = folder(fh, comm=comm)
    # end with

    print("Rank {0} exited with statement".format(comm.rank))

    savepref = "{0}{1}_{2}chan{3}ntbin".format(telescope, psr, nchan, ntbin)
    if do_waterfall:
        waterfall = np.zeros_like(mywaterfall)
        comm.Reduce(mywaterfall, waterfall, op=MPI.SUM, root=0)
        if comm.rank == 0:
            # waterfall = normalize_counts(waterfall)
            np.save("{0}waterfall_{1}+{2:08}sec.npy"
                    .format(savepref, tstart.isot, dt.sec), waterfall)

    if do_foldspec:
        foldspec = np.zeros_like(myfoldspec) if comm.rank == 0 else None
        print("Rank {0} is entering comm.Reduce".format(comm.rank))
        comm.Reduce(myfoldspec, foldspec, op=MPI.SUM, root=0)
        del myfoldspec  # save memory on node 0
        icount = np.zeros_like(myicount) if comm.rank == 0 else None
        comm.Reduce(myicount, icount, op=MPI.SUM, root=0)
        del myicount  # save memory on node 0
        if comm.rank == 0:
            fname = ("{0}foldspec_{1}+{2:08}sec.npy")
            np.save(fname.format(savepref, tstart.isot, dt.sec), foldspec)
            iname = ("{0}icount_{1}+{2:08}sec.npy")
            np.save(iname.format(savepref, tstart.isot, dt.sec), icount)

    if comm.rank == 0:
        if do_foldspec and foldspec.ndim == 3:
            # sum over time slices -> pulse profile vs channel
            foldspec1 = normalize_counts(foldspec.sum(0).astype(np.float64),
                                         icount.sum(0).astype(np.int64))
            # sum over channels -> pulse profile vs time
            foldspec3 = normalize_counts(foldspec.sum(1).astype(np.float64),
                                         icount.sum(1).astype(np.int64))
            fluxes = foldspec1.sum(axis=0)
            with open('{0}flux_{1}+{2:08}sec.dat'
                      .format(savepref, tstart.isot, dt.sec), 'w') as f:
                for i, flux in enumerate(fluxes):
                    f.write('{0:12d} {1:12.9g}\n'.format(i+1, flux))
            # ratio'd flux only if file will not be ridiculously large
            if ntbin*ngate < 10000:
                foldspec2 = normalize_counts(foldspec, icount)

        plots = True
        if plots and do_waterfall and waterfall.ndim == 2:  # no polarizations
            w = waterfall.copy()
            pmap('{0}waterfall_{1}+{2:08}sec.pgm'
                 .format(savepref, tstart.isot, dt.sec), w, 1, verbose=True)
        if plots and do_foldspec and foldspec.ndim == 3:
            pmap('{0}folded_{1}+{2:08}sec.pgm'
                 .format(savepref, tstart.isot, dt.sec), foldspec1, 0, verbose)
            pmap('{0}folded3_{1}+{2:08}sec.pgm'
                 .format(savepref, tstart.isot, dt.sec),
                 foldspec3.T, 0, verbose)
            # ratio'd flux only if file will not be ridiculously large
            if ntbin*ngate < 10000:
                pmap('{0}foldedbin_{1}+{2:08}sec.pgm'
                     .format(savepref, tstart.isot, dt.sec),
                     foldspec2.transpose(1,2,0).reshape(nchan,-1), 1, verbose)
Example #2
0
def main_correlate(tel1, date1, tel2, date2, nchan, tstart, tend, dedisperse,
                   do_foldspec, ntbin, ngate,
                   do_waterfall, ntw_min,
                   save_xcorr, verbose=0):
    """
    the main cross-correlation routine
    """
    comm = MPI.COMM_WORLD
    if comm.size > 1 and save_xcorr:
        if comm.rank == 0:
	    print("Warning, h5py mpio is sometimes slow. Consider disabling save_xcorr")
	# save_xcorr = False
    # observing parameters
    t0 = Time(tstart, scale='utc')
    t1 = Time(tend, scale='utc')

    Obs = obsdata()
    obskey1 = Obs[tel1].nearest_observation(date1)
    obskey2 = Obs[tel2].nearest_observation(date2)
    psr1 = Obs[tel1][obskey1]['src']
    psr2 = Obs[tel2][obskey2]['src']
    files1 = Obs[tel1].file_list(obskey1)
    files2 = Obs[tel2].file_list(obskey2)

    assert psr1 == psr2
    if comm.rank == 0:
        print("forming visibilities from (telescope, observation_key) = \n"
              "\t ({0}, {1}) and ({2}, {3}), source {4}".format(tel1, obskey1, tel2, obskey2, psr1))
    dm = Obs['psrs'][psr1]['dm']
    with LOFARdata_Pcombined(*files1, comm=comm) as fh1,\
            GMRTdata(*files2, comm=comm) as fh2:
        phasepol1 = Obs['lofar'][obskey1].get_phasepol(fh1.time0, rphase=None)
        phasepol2 = Obs['gmrt'][obskey2].get_phasepol(fh2.time0, rphase=None)
        nt = min(fh1.ntimebins(t0, t1), fh2.ntimebins(t0, t1))
        # out = (foldspec, icount, waterfall)
        out = correlate.correlate(fh1, fh2, dm=dm, nchan=nchan, ngate=ngate,
                                  ntbin=ntbin, nt=nt, ntw=ntw_min,
                                  t0=t0, t1=t1, dedisperse=dedisperse,
                                  phasepol=(phasepol1, phasepol2),
                                  do_waterfall=do_waterfall,
                                  do_foldspec=do_foldspec,
                                  save_xcorr=save_xcorr,
                                  comm=comm)
        myfoldspec = out[0]
        myicount = out[1]
        mywaterfall = out[2]

    savepref = "{0}{1}_{2}chan{3}ntbin".format(tel1[0], tel2[0], nchan, ntbin)
    dt = t1 - t0
    if do_waterfall:
        waterfall = np.zeros_like(mywaterfall)
        comm.Reduce(mywaterfall, waterfall, op=MPI.SUM, root=0)
        if comm.rank == 0:
            # waterfall = normalize_counts(waterfall)
            np.save("{0}waterfall_{1}+{2:08}sec.npy"
                    .format(savepref, t0, dt.sec), waterfall)

    if do_foldspec:
        foldspec = np.zeros_like(myfoldspec)
        icount = np.zeros_like(myicount)
        comm.Reduce(myfoldspec, foldspec, op=MPI.SUM, root=0)
        comm.Reduce(myicount, icount, op=MPI.SUM, root=0)
        if comm.rank == 0:
            fname = ("{0}foldspec_{1}+{2:08}sec.npy")
            iname = ("{0}icount_{1}+{2:08}sec.npy")
            np.save(fname.format(savepref, t0, dt.sec), foldspec)
            np.save(iname.format(savepref, t0, dt.sec), icount)

            # get normalized flux in each bin (where any were added)
            f2 = normalize_counts(foldspec, icount)
            foldspec1 = f2.sum(axis=2)
            fluxes = foldspec1.sum(axis=0)
            foldspec3 = f2.sum(axis=0)

            with open('{0}flux_{1}+{2:08}sec.dat'
                      .format(savepref, t0, dt.sec), 'w') as f:
                for i, flux in enumerate(fluxes):
                    f.write('{0:12d} {1:12.9g}\n'.format(i + 1, flux))

    plots = True
    if plots and comm.rank == 0:
        if do_waterfall:
            w = waterfall.copy()
            try:
                pmap('{0}waterfall_{1}+{2:08}sec.pgm'
                     .format(savepref, t0, dt.sec), w, 1, verbose=True)
            except:
                pass
        if do_foldspec:
            pmap('{0}folded_{1}+{2:08}sec.pgm'
                 .format(savepref, t0, dt.sec), foldspec1, 0, verbose)
            # TODO: Note, I (aaron) don't think this works for LOFAR data
            # since nchan=20, but we concatenate several subband files
            # together, so f2.nchan = N_concat * nchan
            # It should work for my "new" LOFAR_Pconcate file class
            pmap('{0}foldedbin_{1}+{2:08}sec.pgm'
                 .format(savepref, t0, dt.sec),
                 f2.transpose(0, 2, 1).reshape(nchan, -1), 1, verbose)
            pmap('{0}folded3_{1}+{2:08}sec.pgm'
                 .format(savepref, t0, dt.sec), foldspec3, 0, verbose)
Example #3
0
def reduce(telescope, obskey, tstart, tend, nchan, ngate, ntbin, ntw_min,
           rfi_filter_raw=None, fref=None, dedisperse=None,
           rfi_filter_power=None, do_waterfall=True, do_foldspec=True,
           verbose=True):

    comm = MPI.COMM_WORLD
    if dedisperse == 'None':
        dedisperse = None

    if verbose > 3 and comm.rank == 0:
        print(telescope, obskey, tstart, tend, nchan, ngate, ntbin, ntw_min,
              rfi_filter_raw, fref, dedisperse,
              do_waterfall, do_foldspec, verbose)

    if dedisperse is not None and fref is None:
        raise ValueError("Need to give reference frequency to dedisperse to")

    obs = obsdata()
    if verbose > 3 and comm.rank == 0:
        print(obs)
    # find nearest observation to 'date',
    # warning if > 1s off requested start time of observation
    if obskey not in obs[telescope]:
        # assume it is a date, which may be only good to nearest second
        obskey = obs[telescope].nearest_observation(obskey)
    # target of this observation
    psr = obs[telescope][obskey]['src']
    assert psr in obs['psrs'].keys()

    dm = obs['psrs'][psr]['dm']

    if verbose and comm.rank == 0:
        print("Attempting to open {0}: {1}".format(telescope, obskey))

    with obs[telescope].open(obskey, comm=comm) as fh:
        if verbose and comm.rank == 0:
            print("Opened files")

        # nchan = None means data is channelized already, so we get this
        # property directly from the file
        if nchan is None:
            nchan = fh.nchan
        # ensure requested number of channels is integer multiple of
        # existing channels
        if nchan % getattr(fh, 'nchan', 1) != 0:
            raise ValueError("Can only channelize data to an integer multiple "
                             "of the number of input channels (={0})."
                             .format(fh.nchan))

        time0 = fh.time0
        tstart = time0 if tstart is None else Time(tstart, scale='utc')
        if tstart < time0:
            raise ValueError("Cowardly refusing to analyse with requested "
                             "time {0} before start time {1}."
                             .format(tstart.iso, time0.iso))
        if tend is None:
            tend = obs[telescope][obskey]['tend']

        try:
            tend = Time(tend, scale='utc')
            dt = tend - tstart
        except ValueError:
            dt = TimeDelta(float(tend), format='sec')
            tend = tstart + dt

        if verbose and comm.rank == 0:
            print("Requested time span: {0} to {1}".format(tstart.isot,
                                                           tend.isot))

        phasepol = obs[telescope][obskey].get_phasepol(time0)
        nt = fh.ntimebins(tstart, tend)
        ntint = fh.ntint(nchan if fh.nchan == 1 else fh.nchan)
        # number of samples to combine for waterfall
        ntw = min(ntw_min, nt*ntint)
        # number of records to skip

        if verbose and comm.rank == 0:
            print("Using start time {0} and phase polynomial {1}"
                  .format(time0.isot, phasepol))

        fh.seek(tstart)
        if verbose and comm.rank == 0:
            print("Skipped {0} blocks and will fold {1} blocks to cover "
                  "time span {2} to {3}"
                  .format(fh.offset/fh.blocksize, nt, fh.time().isot,
                          fh.time(fh.offset + nt*fh.blocksize).isot))

        # set the default parameters to fold
        # Note, some parameters may be in fh's HDUs, or fh.__getitem__
        # but these are overwritten if explicitly sprecified in Folder
        folder = Folder(fh, nchan=nchan,
                        nt=nt, ntint=ntint, ngate=ngate,
                        ntbin=ntbin, ntw=ntw, dm=dm, fref=fref,
                        phasepol=phasepol, dedisperse=dedisperse,
                        do_waterfall=do_waterfall, do_foldspec=do_foldspec,
                        verbose=verbose, progress_interval=1,
                        rfi_filter_raw=rfi_filter_raw,
                        rfi_filter_power=rfi_filter_power)
        myfoldspec, myicount, mywaterfall = folder(fh, comm=comm)
    # end with

    print("Rank {0} exited with statement".format(comm.rank))

    savepref = "{0}{1}_{2}chan{3}ntbin".format(telescope, psr, nchan, ntbin)
    if do_waterfall:
        waterfall = np.zeros_like(mywaterfall)
        comm.Reduce(mywaterfall, waterfall, op=MPI.SUM, root=0)
        if comm.rank == 0:
            # waterfall = normalize_counts(waterfall)
            np.save("{0}waterfall_{1}+{2:08}sec.npy"
                    .format(savepref, tstart.isot, dt.sec), waterfall)

    if do_foldspec:
        foldspec = np.zeros_like(myfoldspec) if comm.rank == 0 else None
        print("Rank {0} is entering comm.Reduce".format(comm.rank))
        comm.Reduce(myfoldspec, foldspec, op=MPI.SUM, root=0)
        del myfoldspec  # save memory on node 0
        icount = np.zeros_like(myicount) if comm.rank == 0 else None
        comm.Reduce(myicount, icount, op=MPI.SUM, root=0)
        del myicount  # save memory on node 0
        if comm.rank == 0:
            fname = ("{0}foldspec_{1}+{2:08}sec.npy")
            np.save(fname.format(savepref, tstart.isot, dt.sec), foldspec)
            iname = ("{0}icount_{1}+{2:08}sec.npy")
            np.save(iname.format(savepref, tstart.isot, dt.sec), icount)

    if comm.rank == 0:
        if do_foldspec and foldspec.ndim == 3:
            # sum over time slices -> pulse profile vs channel
            foldspec1 = normalize_counts(foldspec.sum(0).astype(np.float64),
                                         icount.sum(0).astype(np.int64))
            # sum over channels -> pulse profile vs time
            foldspec3 = normalize_counts(foldspec.sum(1).astype(np.float64),
                                         icount.sum(1).astype(np.int64))
            fluxes = foldspec1.sum(axis=0)
            with open('{0}flux_{1}+{2:08}sec.dat'
                      .format(savepref, tstart.isot, dt.sec), 'w') as f:
                for i, flux in enumerate(fluxes):
                    f.write('{0:12d} {1:12.9g}\n'.format(i+1, flux))
            # ratio'd flux only if file will not be ridiculously large
            if ntbin*ngate < 10000:
                foldspec2 = normalize_counts(foldspec, icount)

        plots = True
        if plots and do_waterfall and waterfall.ndim == 2:  # no polarizations
            w = waterfall.copy()
            pmap('{0}waterfall_{1}+{2:08}sec.pgm'
                 .format(savepref, tstart.isot, dt.sec), w, 1, verbose=True)
        if plots and do_foldspec and foldspec.ndim == 3:
            pmap('{0}folded_{1}+{2:08}sec.pgm'
                 .format(savepref, tstart.isot, dt.sec), foldspec1, 0, verbose)
            pmap('{0}folded3_{1}+{2:08}sec.pgm'
                 .format(savepref, tstart.isot, dt.sec),
                 foldspec3.T, 0, verbose)
            # ratio'd flux only if file will not be ridiculously large
            if ntbin*ngate < 10000:
                pmap('{0}foldedbin_{1}+{2:08}sec.pgm'
                     .format(savepref, tstart.isot, dt.sec),
                     foldspec2.transpose(1,2,0).reshape(nchan,-1), 1, verbose)
Example #4
0
def reduce(telescope,
           obsdate,
           tstart,
           tend,
           nchan,
           ngate,
           ntbin,
           ntw_min=10200,
           fref=_fref,
           rfi_filter_raw=None,
           do_waterfall=True,
           do_foldspec=True,
           dedisperse=None,
           verbose=True):
    comm = MPI.COMM_WORLD
    Obs = obsdata()
    # find nearest observation to 'date',
    # warning if > 1s off requested start time of observation
    obskey = Obs[telescope].nearest_observation(obsdate)
    # target of this observation
    psr = Obs[telescope][obskey]['src']
    assert psr in Obs['psrs'].keys()

    dm = Obs['psrs'][psr]['dm']

    files = Obs[telescope].file_list(obskey)
    if telescope == 'aro':
        GenericOpen = AROdata
    elif telescope == 'lofar':
        GenericOpen = LOFARdata
        GenericOpen = LOFARdata_Pcombined
    elif telescope == 'gmrt':
        GenericOpen = GMRTdata

    with GenericOpen(*files, comm=comm) as fh:
        # nchan = None means data is channelized already, so we get this
        # property directly from the file
        if nchan is None or telescope == 'lofar':
            if comm.rank == 0:
                print("LOFAR data already channelized: overriding nchan to {0}"
                      "\n\t as configured in observations.conf".format(
                          fh.nchan))
            nchan = fh.nchan
        time0 = fh.time0
        phasepol = Obs[telescope][obskey].get_phasepol(time0)
        nt = fh.ntimebins(tstart, tend)
        ntint = fh.ntint(nchan)
        # number of samples to combine for waterfall
        ntw = min(ntw_min, nt * ntint)
        # number of records to skip
        tstart = Time(tstart, scale='utc')
        tend = Time(tend, scale='utc')
        dt = tend - tstart

        fh.seek(tstart)
        if verbose and comm.rank == 0:
            print("Using start time {0} and phase polynomial {1}".format(
                time0, phasepol))
            print("Skipping {0} blocks and folding {1} blocks to cover "
                  "time span {2} to {3}".format(
                      fh.offset / fh.blocksize, nt, fh.time(),
                      fh.time(fh.offset + nt * fh.blocksize)))
        # set the default parameters to fold
        # Note, some parameters may be in fh's HDUs, or fh.__getitem__
        # but these are overwritten if explicitly sprecified in Folder
        folder = Folder(fh,
                        nchan=nchan,
                        nt=nt,
                        ntint=ntint,
                        ngate=ngate,
                        ntbin=ntbin,
                        ntw=ntw,
                        dm=dm,
                        fref=fref,
                        phasepol=phasepol,
                        dedisperse=dedisperse,
                        do_waterfall=do_waterfall,
                        do_foldspec=do_foldspec,
                        verbose=verbose,
                        progress_interval=1,
                        rfi_filter_raw=rfi_filter_raw,
                        rfi_filter_power=None)
        myfoldspec, myicount, mywaterfall, subint_table = folder(fh, comm=comm)
    # end with

    savepref = "{0}{1}_{2}chan{3}ntbin".format(telescope, psr, nchan, ntbin)
    if do_waterfall:
        waterfall = np.zeros_like(mywaterfall)
        comm.Reduce(mywaterfall, waterfall, op=MPI.SUM, root=0)
        if comm.rank == 0:
            # waterfall = normalize_counts(waterfall)
            np.save(
                "{0}waterfall_{1}+{2:08}sec.npy".format(
                    savepref, tstart, dt.sec), waterfall)

    if do_foldspec:
        foldspec = np.zeros_like(myfoldspec)
        icount = np.zeros_like(myicount)
        comm.Reduce(myfoldspec, foldspec, op=MPI.SUM, root=0)
        comm.Reduce(myicount, icount, op=MPI.SUM, root=0)
        if comm.rank == 0:
            fname = ("{0}foldspec_{1}+{2:08}sec.npy")
            iname = ("{0}icount_{1}+{2:08}sec.npy")
            np.save(fname.format(savepref, tstart, dt.sec), foldspec)
            np.save(iname.format(savepref, tstart, dt.sec), icount)

            # get normalized flux in each bin (where any were added)
            f2 = normalize_counts(foldspec, icount)
            foldspec1 = f2.sum(axis=2)
            fluxes = foldspec1.sum(axis=0)
            foldspec3 = f2.sum(axis=0)

            with open(
                    '{0}flux_{1}+{2:08}sec.dat'.format(savepref, tstart,
                                                       dt.sec), 'w') as f:
                for i, flux in enumerate(fluxes):
                    f.write('{0:12d} {1:12.9g}\n'.format(i + 1, flux))

    plots = True
    if plots and comm.rank == 0:
        if do_waterfall:
            w = waterfall.copy()
            pmap('{0}waterfall_{1}+{2:08}sec.pgm'.format(
                savepref, tstart, dt.sec),
                 w,
                 1,
                 verbose=True)
        if do_foldspec:
            pmap(
                '{0}folded_{1}+{2:08}sec.pgm'.format(savepref, tstart, dt.sec),
                foldspec1, 0, verbose)
            # TODO: Note, I (aaron) don't think this works for LOFAR data
            # since nchan=20, but we concatenate several subband files
            # together, so f2.nchan = N_concat * nchan
            # It should work for my "new" LOFAR_Pconcate file class
            pmap(
                '{0}foldedbin_{1}+{2:08}sec.pgm'.format(
                    savepref, tstart, dt.sec),
                f2.transpose(0, 2, 1).reshape(nchan, -1), 1, verbose)
            pmap(
                '{0}folded3_{1}+{2:08}sec.pgm'.format(savepref, tstart,
                                                      dt.sec), foldspec3, 0,
                verbose)

    savefits = False
    if savefits and comm.rank == 0:
        print("Saving FITS files...")
        # assign the folded data ( [f2] = [nchan, ngate, ntbin]
        #                          want [ntbin, npol, nchan, ngate]

        # TODO: figure out lofar and aro data, which concatenates the channels
        # so f2 = len(range(P))*nchan
        if telescope == 'lofar':
            f2 = f2[-nchan:]
        elif telescope == 'aro':
            f2 = f2[0:nchan]
        nchan, ngate, ntbin = f2.shape
        f2 = f2.transpose(2, 0, 1)
        f2 = f2.reshape(ntbin, np.newaxis, nchan, ngate)
        std = f2.std(axis=0)
        subint_table[1].data.field('DATA')[:] = (2**16 * f2 / std).astype(
            np.int16)
        fout = '{0}folded3_{1}+{2:08}sec.fits'.format(savepref, tstart, dt.sec)
        # Note: output_verify != 'ignore' resets the cards for some reason
        try:
            subint_table.writeto(fout, output_verify='ignore', clobber=True)
        except ValueError:
            print("FITS writings is a work in progress: "
                  "need to come to terms with array shapes")
            print("f2.shape, subint_table[1].data.field('DATA')[:].shape = ",
                  f2.shape, subint_table[1].data.field('DATA')[:].shape)
Example #5
0
def main_correlate(tel1,
                   date1,
                   tel2,
                   date2,
                   nchan,
                   tstart,
                   tend,
                   dedisperse,
                   do_foldspec,
                   ntbin,
                   ngate,
                   do_waterfall,
                   ntw_min,
                   save_xcorr,
                   verbose=0):
    """
    the main cross-correlation routine
    """
    comm = MPI.COMM_WORLD
    if comm.size > 1 and save_xcorr:
        if comm.rank == 0:
            print(
                "Warning, h5py mpio is sometimes slow. Consider disabling save_xcorr"
            )

# save_xcorr = False
# observing parameters
    t0 = Time(tstart, scale='utc')
    t1 = Time(tend, scale='utc')

    Obs = obsdata()
    obskey1 = Obs[tel1].nearest_observation(date1)
    obskey2 = Obs[tel2].nearest_observation(date2)
    psr1 = Obs[tel1][obskey1]['src']
    psr2 = Obs[tel2][obskey2]['src']
    files1 = Obs[tel1].file_list(obskey1)
    files2 = Obs[tel2].file_list(obskey2)

    assert psr1 == psr2
    if comm.rank == 0:
        print("forming visibilities from (telescope, observation_key) = \n"
              "\t ({0}, {1}) and ({2}, {3}), source {4}".format(
                  tel1, obskey1, tel2, obskey2, psr1))
    dm = Obs['psrs'][psr1]['dm']
    with LOFARdata_Pcombined(*files1, comm=comm) as fh1,\
            GMRTdata(*files2, comm=comm) as fh2:
        phasepol1 = Obs['lofar'][obskey1].get_phasepol(fh1.time0, rphase=None)
        phasepol2 = Obs['gmrt'][obskey2].get_phasepol(fh2.time0, rphase=None)
        nt = min(fh1.ntimebins(t0, t1), fh2.ntimebins(t0, t1))
        # out = (foldspec, icount, waterfall)
        out = correlate.correlate(fh1,
                                  fh2,
                                  dm=dm,
                                  nchan=nchan,
                                  ngate=ngate,
                                  ntbin=ntbin,
                                  nt=nt,
                                  ntw=ntw_min,
                                  t0=t0,
                                  t1=t1,
                                  dedisperse=dedisperse,
                                  phasepol=(phasepol1, phasepol2),
                                  do_waterfall=do_waterfall,
                                  do_foldspec=do_foldspec,
                                  save_xcorr=save_xcorr,
                                  comm=comm)
        myfoldspec = out[0]
        myicount = out[1]
        mywaterfall = out[2]

    savepref = "{0}{1}_{2}chan{3}ntbin".format(tel1[0], tel2[0], nchan, ntbin)
    dt = t1 - t0
    if do_waterfall:
        waterfall = np.zeros_like(mywaterfall)
        comm.Reduce(mywaterfall, waterfall, op=MPI.SUM, root=0)
        if comm.rank == 0:
            # waterfall = normalize_counts(waterfall)
            np.save(
                "{0}waterfall_{1}+{2:08}sec.npy".format(savepref, t0, dt.sec),
                waterfall)

    if do_foldspec:
        foldspec = np.zeros_like(myfoldspec)
        icount = np.zeros_like(myicount)
        comm.Reduce(myfoldspec, foldspec, op=MPI.SUM, root=0)
        comm.Reduce(myicount, icount, op=MPI.SUM, root=0)
        if comm.rank == 0:
            fname = ("{0}foldspec_{1}+{2:08}sec.npy")
            iname = ("{0}icount_{1}+{2:08}sec.npy")
            np.save(fname.format(savepref, t0, dt.sec), foldspec)
            np.save(iname.format(savepref, t0, dt.sec), icount)

            # get normalized flux in each bin (where any were added)
            f2 = normalize_counts(foldspec, icount)
            foldspec1 = f2.sum(axis=2)
            fluxes = foldspec1.sum(axis=0)
            foldspec3 = f2.sum(axis=0)

            with open('{0}flux_{1}+{2:08}sec.dat'.format(savepref, t0, dt.sec),
                      'w') as f:
                for i, flux in enumerate(fluxes):
                    f.write('{0:12d} {1:12.9g}\n'.format(i + 1, flux))

    plots = True
    if plots and comm.rank == 0:
        if do_waterfall:
            w = waterfall.copy()
            try:
                pmap('{0}waterfall_{1}+{2:08}sec.pgm'.format(
                    savepref, t0, dt.sec),
                     w,
                     1,
                     verbose=True)
            except:
                pass
        if do_foldspec:
            pmap('{0}folded_{1}+{2:08}sec.pgm'.format(savepref, t0, dt.sec),
                 foldspec1, 0, verbose)
            # TODO: Note, I (aaron) don't think this works for LOFAR data
            # since nchan=20, but we concatenate several subband files
            # together, so f2.nchan = N_concat * nchan
            # It should work for my "new" LOFAR_Pconcate file class
            pmap('{0}foldedbin_{1}+{2:08}sec.pgm'.format(savepref, t0, dt.sec),
                 f2.transpose(0, 2, 1).reshape(nchan, -1), 1, verbose)
            pmap('{0}folded3_{1}+{2:08}sec.pgm'.format(savepref, t0, dt.sec),
                 foldspec3, 0, verbose)