Exemple #1
0
def main():

  # Ignore divide by zero and related warnings caused by 3-position switch 
  # correction applied to full bandwidth (including <50 MHz)
  np.seterr(divide='ignore', invalid='ignore')
  
  # Parse command line arguments
  parser = argparse.ArgumentParser(description='Plots contents of ACQ file.');
  parser.add_argument('inputFile', help='Options for plotting contents of ACQ file.');
  parser.add_argument('-o', '--output',   
                      nargs=1, default=['.'],
                      help='Output directory.');
  parser.add_argument('-b', '--start',   
                      nargs=1, type=int, default=[0],
                      help='Start reading on specified line in the file (first line is 0).  Negative values count back from end of file.  There are two lines per spectrum and six lines per switch cycle.');
  parser.add_argument('-e', '--stop',   
                      nargs=1, type=int, default=[9999999],
                      help='Stop reading before specified line in the file.  Negative values count back from end of file.  There are two lines per spectrum and six lines per switch cycle.');                      
  parser.add_argument('-l', '--fmin',   
                      nargs=1, type=float, default=[50],
                      help='Minimum frequency to plot.');
  parser.add_argument('-u', '--fmax',   
                      nargs=1, type=float, default=[199],
                      help='Maximum frequency to plot.');  
  parser.add_argument('-t', '--thin',   
                      nargs=1, type=int, default=[1],
                      help='Reduce the number of spectra used in calculations and plots by the specified factor.');
  parser.add_argument('-r', '--raw',   
                      action='store_true', 
                      help='Plot the raw spectra for each switch position.');
  parser.add_argument('-c', '--corrected',   
                      action='store_true', 
                      help='Plot the individual 3-position switch corrected spectra overlaid on the same plot.');                      
  parser.add_argument('-i', '--integrate',   
                      action='store_true', 
                      help='Plot the integration (mean) of the 3-position corrected spectra.');                      
  parser.add_argument('-w', '--waterfall', 
                      action='store_true', 
                      help='Plot a waterfall of spectra.  Used with--corrected and/or --raw.');
  parser.add_argument('-f', '--fit',   
                      action='store_true',
                      help='Fit the integrated (mean) spectrum and plot the residuals.  Specify model details using --model and --nterms.');
  parser.add_argument('-m', '--model',   
                      nargs=1, default=['linpoly'],
                      help='Model to use if --fit is set.  Specify one of: linpoly, linphys, linlog.');      
  parser.add_argument('-n', '--nterms',   
                      nargs=1, type=int, default=[5],
                      help='Number of terms to use in model fit.');  
  parser.add_argument('-g', '--flag', 
                      action='store_true', 
                      help='Flag RFI in corrected integrations and residual plots.');  
  parser.add_argument('-p', '--power', 
                      nargs=2, type=float, default=[100, 2500], 
                      help='Provide bounds for allowed minimum and maximum average corrected spectrum power for use when flagging.');                         
  parser.add_argument('-d', '--threshold', 
                      nargs=6, type=float, default=[3, 0.001, 20, 11, 75, 0], 
                      help='Provide sigma, tolerance, maxiter, and polynomial nterms, center freq, and beta for use when flagging channels.');                         
                                          
  args = parser.parse_args();
  print(args);
  inputFile = args.inputFile;
  outputDir = args.output[0];
  start = args.start[0];
  stop = args.stop[0];
  fmin = args.fmin[0];
  fmax = args.fmax[0];
  thin = args.thin[0];
  model = args.model[0].lower();
  nterms = args.nterms[0];
  power = args.power;
  threshold = args.threshold;
  bFlagRFI = args.flag;
  bPlotFit = args.fit;
  bPlotRaw = args.raw;
  bPlotCorrected = args.corrected;
  bPlotIntegration = args.integrate;
  bPlotWaterfall = args.waterfall;
  
  # Hard coded parameters -- should be added to commandline args at some point
  vc = 75;
  beta = -2.5;
  nkernel = 64;

  inputDir = os.path.dirname(inputFile);
  inputFilenameBase = os.path.splitext(os.path.basename(inputFile))[0];
  outputFullBase = outputDir + '/' + inputFilenameBase;

  # Read the input file
  spec, anc, freqs = analysis.readAcq(inputFile, start=start, stop=stop, verbose=1, seek=True);
  nspec = len(spec);
  nfreqs = len(freqs);
  channelSize = anc[0,10];

  nsize = nspec - (nspec % 3);
  p0 = spec[0:nsize:int(3*thin)];
  p1 = spec[1:nsize:int(3*thin)];
  p2 = spec[2:nsize:int(3*thin)];
  

  # Make the requested plots
  fig = 1;
  cmap = 'jet';
  lw = 0.75;  
  
  if bPlotRaw:
    
    print('Plotting raw spectra summary...');
    
    plt.figure(fig);
    fig = fig + 1;
    plt.clf();
    
    plt.plot(freqs, np.max(10*np.log10(p0.transpose()), axis=1), color=[0.75,0.75,0.75], linewidth=lw);
    h0, = plt.plot(freqs, np.mean(10*np.log10(p0.transpose()), axis=1), 'k-', linewidth=lw, label='p0 (antenna)');
    plt.plot(freqs, np.min(10*np.log10(p0.transpose()), axis=1), color=[0.5,0.5,0.5], linewidth=lw);
   
    
    plt.plot(freqs, np.max(10*np.log10(p1.transpose()), axis=1), color=[0.75,0.75,1], linewidth=lw);
    h1, = plt.plot(freqs, np.mean(10*np.log10(p1.transpose()), axis=1), 'b-', linewidth=lw, label='p1 (antenna)');
    plt.plot(freqs, np.min(10*np.log10(p1.transpose()), axis=1), color=[0.5,0.5,1], linewidth=lw);
       
    plt.plot(freqs, np.max(10*np.log10(p2.transpose()), axis=1), color=[1,0.75,0.75], linewidth=lw);
    h2, = plt.plot(freqs, np.mean(10*np.log10(p2.transpose()), axis=1), 'r-', linewidth=lw, label='p2 (antenna)'); 
    plt.plot(freqs, np.min(10*np.log10(p2.transpose()), axis=1), color=[1,0.5,0.5], linewidth=lw);
    
    plt.xlabel("Frequency [MHz]");
    plt.ylabel("p [dB]");  
    plt.ylim([-90, -50]);
    plt.legend(handles=[h0, h1, h2]);
    plt.savefig(outputFullBase + '_raw.png');
      
    if bPlotWaterfall:
      
      print('Plotting raw spectra waterfalls...');
      clim = [-90, -75];
      plt.figure(fig);
      fig = fig + 1;
      plt.clf();
      plt.imshow(10*np.log10(p0), extent=[freqs[0], freqs[len(freqs)-1], len(p0)*thin, 0], cmap=cmap, aspect='auto', origin='upper');
      plt.clim(clim);
      plt.colorbar(label='Arbitrary power [dB]');     
      plt.xlabel("Frequency [MHz]");
      plt.ylabel("Spectrum # (p0)");
      plt.savefig(outputFullBase + '_raw_waterfall_p0.png');                 

      plt.figure(fig);
      fig = fig + 1;
      plt.clf();
      plt.imshow(10*np.log10(p1), extent=[freqs[0], freqs[len(freqs)-1], len(p0)*thin, 0], cmap=cmap, aspect='auto', origin='upper');
      plt.clim(clim);
      plt.colorbar(label='Arbitrary power [dB]');     
      plt.xlabel("Frequency [MHz]");
      plt.ylabel("Spectrum # (p1)");
      plt.savefig(outputFullBase + '_raw_waterfall_p1.png');  
      
      plt.figure(fig);
      fig = fig + 1;
      plt.clf();
      plt.imshow(10*np.log10(p2), extent=[freqs[0], freqs[len(freqs)-1], len(p0)*thin, 0], cmap=cmap, aspect='auto', origin='upper'); 
      plt.clim(clim);
      plt.colorbar(label='Arbitrary power [dB]');     
      plt.xlabel("Frequency [MHz]");
      plt.ylabel("Spectrum # (p2)");   
      plt.savefig(outputFullBase + '_raw_waterfall_p2.png');  
         
      
      
  if bPlotCorrected or bPlotIntegration or bPlotFit:
        
    # Apply the 3-position correction
    print('Applying 3-position correction...');
    cor = analysis.correct(p0, p1, p2);
    
    # Reduce the frequency range to the specified bounds
    freqssub = freqs[(freqs>=fmin) & (freqs<=fmax)];
    corsub = cor[:, (freqs>=fmin) & (freqs<=fmax)];
    print('Frequency range reducted to {}-{} MHz (keeping {} channels)'.format(fmin, fmax, corsub.shape[1]));

    # Integrate without filtering
    if bFlagRFI:
      
      # Integrate after flagging bad spectra
      rowWeights = analysis.flagAveragePower(corsub, maxpwr=power[1], minpwr=power[0]);
      corsubmean = np.sum((corsub.transpose()*rowWeights).transpose(), axis=0) / np.sum(rowWeights);
      print('Number of spectra flagged: {} out of {}'.format(corsub.shape[0] - np.sum(rowWeights), corsub.shape[0]));      
      
      flagComponents = models.linearPolynomialComponents(freqssub, vc=threshold[4], nterms=threshold[3], beta=threshold[5]);
      channelWeights, flagrms = analysis.flagChannels(corsubmean, flagComponents, sigma=threshold[0], tol=threshold[1], maxiter=threshold[2]);  
      print('Number of channels flagged: {} out of {}'.format(corsub.shape[1] - np.sum(channelWeights), corsub.shape[1]));
          
    else:
      
      corsubmean = np.mean(corsub, axis=0);  
      channelWeights = np.ones(corsubmean.shape);
        
     
  if bPlotCorrected:
    
    print('Plotting 3-position corrected spectra summary...');
      
    plt.figure(fig);
    fig = fig + 1;
    plt.clf();
    plt.plot(freqssub, np.max(corsub.transpose(), axis=1), 'r-', linewidth=lw);
    plt.plot(freqssub, np.min(corsub.transpose(), axis=1), 'b-', linewidth=lw);
    plt.plot(freqssub, np.mean(corsub.transpose(), axis=1), 'k-', linewidth=lw);   
    plt.xlabel("Frequency [MHz]");
    plt.ylabel("$T_{3pos}$ [K]");
    plt.legend(['Max hold', 'Min hold', 'Mean']);
    plt.xlim([fmin, fmax]);
    
    ymax = np.nanmax(np.where(~np.isinf(corsub), corsub, np.nan));
    ymin = np.nanmin(np.where(~np.isinf(corsub), corsub, np.nan));  
    if ymax>10000:
      ymax = np.nanmedian(corsub[:,0])*1.5;
    ydiff = ymax - ymin;
    ypad = ydiff * 0.1;  
    
    plt.ylim([ymin-ypad, ymax+ypad]);
    plt.savefig(outputFullBase + '_3pos.png');                   
   
    if bPlotWaterfall:
      
      print('Plot 3-position corrected spectra waterfall...');
      
      plt.figure(fig);
      fig = fig + 1;
      plt.clf();
      plt.imshow(corsub, extent=[freqssub[0], freqssub[len(freqssub)-1], len(corsub)*thin, 0], cmap=cmap, aspect='auto', origin='upper');
      plt.clim([ymin, ymax]);
      plt.colorbar(label='$T_{3pos}$ [K]');     
      plt.xlabel("Frequency [MHz]");
      plt.ylabel("Spectrum #");
      plt.savefig(outputFullBase + '_3pos_waterfall.png');                   
      
      
  if bPlotIntegration:
  
    print('Plotting 3-position corrected integrated spectrum...');

    ind = [i for i in range(len(channelWeights)) if channelWeights[i]==0];

    plotdata = corsubmean.copy();
    plotdata[ind] = np.nan;
    plt.figure(fig);
    fig = fig + 1;
    plt.clf();
    plt.plot(freqssub, plotdata, linewidth=lw);
    plt.xlabel("Frequency [MHz]");
    plt.ylabel("$T_{3pos}$ [K]");
    plt.xlim([fmin, fmax]);
  
    ymax = np.nanmax(np.where(~np.isinf(corsubmean), corsubmean, np.nan));
    ymin = np.nanmin(np.where(~np.isinf(corsubmean), corsubmean, np.nan));
    if ymax>10000:
      ymax = np.nanmedian(corsub[:,0])*1.5;
    ydiff = ymax - ymin;
    ypad = ydiff * 0.1;  
  
    plt.ylim([ymin-ypad, ymax+ypad]);
    plt.savefig(outputFullBase + '_3pos_integration.png');  
  

  if bPlotFit:

    print('Plotting residuals to model fit...');
    
    # Fit with model and get residuals
    if model  == 'linpoly':
      components = models.linearPolynomialComponents(freqssub, vc, nterms, beta=beta);
    elif model == 'linphys':
      if nterms is not 5:
        print("Using 5 terms as requried by 'linphys' model.");
        nterms = 5;
      components = models.linearPhysicalComponents(freqssub, vc);
    elif model == 'linlog':
      components = models.linearLogExpansionComponents(freqssub, vc, nterms, beta=beta);
    else:
      print("Specified model '{}' was not recognized.  Using 'linpoly'.".format(model) );
      components = models.linearPolynomialComponents(freqssub, vc, nterms, beta=beta);
          
    # Do the fit
    ind = [i for i in range(len(channelWeights)) if ((channelWeights[i]==1) & (not np.isnan(corsubmean[i])))]; 
    fit, rms = models.fitLinear(corsubmean[ind], components[ind,:]);
    residuals = channelWeights * (corsubmean - np.dot(components, fit));
    rms = np.nanstd(residuals[ind]);
        
    # Smooth residuals with boxcar kernel
    kernel = np.ones(nkernel) / nkernel;
    smoothres = np.convolve(residuals, kernel, 'same');
    smoothrms = np.nanstd(smoothres);

    print('RMS: {:.4f} K'.format(rms));
    print('RMS smoothed ({:}): {:.4f} K'.format(nkernel, smoothrms));

    ind = [i for i in range(len(channelWeights)) if channelWeights[i]==0];     
    plotres = residuals.copy();
    plotres[ind] = np.nan;
        
    plt.figure(fig);
    fig = fig + 1;
    plt.clf();
    plt.plot(freqssub, plotres, 'b-', linewidth=lw);
    plt.plot(freqssub, smoothres, 'k-', linewidth=lw);
    plt.xlabel("Frequency [MHz]");
    plt.ylabel("$T_{res}$ [K]");
    plt.xlim([fmin, fmax]);
    plt.legend(['{:.1f} kHz'.format(1e3*channelSize), '{:.1f} kHz (smoothed)'.format(1e3*nkernel*channelSize)]);
  
    ymax = np.nanmax(np.where(~np.isinf(residuals), residuals, np.nan));
    ymin = np.nanmin(np.where(~np.isinf(residuals), residuals, np.nan));
    if ymax>10000:
      ymax = np.nanmedian(residuals)*1.5;
    ydiff = ymax - ymin;
    ypad = ydiff * 0.1;  
  
    plt.ylim([ymin-ypad, ymax+ypad]);
    plt.savefig(outputFullBase + '_3pos_residuals_{}_nterms{}.png'.format(model, nterms));     
Exemple #2
0
def main():

    # Define the parameters for the run

    #  outputBase = 'G:/My Drive/EDGES/mcmc/run2';
    #  dataFile = 'G:/My Drive/Publications/Papers/2017_EDGES_Low/PublicData/figure1_plotdata.csv';

    outputBase = '/home/loco/analysis/run3'
    dataFile = '/home/loco/analysis/figure1_plotdata.csv'
    nprocs = 2
    vc = 75
    beta = -2.5
    nterms = 5

    # Load public EDGS data from Bowman et al. 2018 Figure 1
    data = np.genfromtxt(dataFile, delimiter=',', skip_header=1)
    f = data[:, 0]
    w = data[:, 1]
    d = data[:, 2]

    # Extract frequency coordinates from the data file for small and large subbands
    v = [
        data[(data[:, 1] > 0), 0], data[(data[:, 0] > 63) & (data[:, 0] < 99),
                                        0]
    ]

    # Extract the data for the same ranges
    d = [
        data[(data[:, 1] > 0), 2], data[(data[:, 0] > 63) & (data[:, 0] < 99),
                                        2]
    ]

    # Estimated thermal noise;
    n = [0.025 * np.ones(d[i].size) for i in range(len(v))]

    # --------------------------------------------------------------------------- #
    # Make a set of simulated foreground realizations
    # --------------------------------------------------------------------------- #

    # Sample the model parameters in a multi-dimensional grid with points at the
    # following parameter values:
    steps = [[1250, 1500, 1750], [-2.6, -2.55, -2.5, -2.45, -2.4],
             [-0.1, -0.05, 0, 0.05, 0.1], [-0.1, -0.05, 0, 0.05, 0.1],
             [-0.1, -0.05, 0, 0.05, 0.1], [0, 0.05, 0.1], [0, 500, 750, 1000]]

    steps_mini = [[1500, 1750], [-2.6, -2.5, -2.4], [-0.1, 0, 0.1],
                  [-0.1, 0, 0.1], [-0.1, 0, 0.1], [0, 0.1], [0, 1000]]

    sig = [80, 20, 7, -0.5]
    steps_sig = [
        np.linspace(55, 100, 10),
        np.linspace(5, 50, 10),
        np.linspace(1, 9, 5),
        np.linspace(-2, 2, 21)
    ]

    # Generate all parmeter grid points
    params = models.populateGrid(steps)
    params_mini = models.populateGrid(steps_mini)
    params_sig = models.populateGrid(steps_sig)

    # Generate the realizations
    set = [models.fullSet(x, vc, params[:-2, :], params[-2:, :]) for x in v]
    set_mini = [
        models.fullSet(x, vc, params_mini[:-2, :], params_mini[-2:, :])
        for x in v
    ]

    # Make a version of the realizations with a simulated signal feature
    set_sig = [
        np.add(set_mini[i].transpose(),
               models.flattenedGaussian(v[i], sig)).transpose()
        for i in range(len(v))
    ]

    # Append the actual data to the end of the foreground-only realization set
    set = [
        np.hstack([set[i], d[i].reshape([v[i].size, 1])])
        for i in range(len(v))
    ]
    set_mini = [
        np.hstack([set_mini[i], d[i].reshape([v[i].size, 1])])
        for i in range(len(v))
    ]

    # Generate the vector components of the models we'll fit
    foregroundComponents = []
    foregroundComponents.append([
        models.linearPolynomialComponents(x, vc, nterms, beta=beta) for x in v
    ])
    foregroundComponents.append(
        [models.linearPhysicalComponents(x, vc) for x in v])

    # --------------------------------------------------------------------------- #
    # Fit the simulated/real data
    # --------------------------------------------------------------------------- #

    fits = [
        models.fitLinear(x, y) for x, y in zip(set, foregroundComponents[0])
    ]
    estimatesPoly = [x[0] for x in fits]
    rmsPoly = [x[1] for x in fits]
    diffPoly = [
        set[i] - foregroundComponents[0][i].dot(estimatesPoly[i])
        for i in range(0, len(set))
    ]

    fits = [
        models.fitLinear(x, y) for x, y in zip(set, foregroundComponents[1])
    ]
    estimatesPhys = [x[0] for x in fits]
    rmsPhys = [x[1] for x in fits]
    diffPhys = [
        set[i] - foregroundComponents[1][i].dot(estimatesPhys[i])
        for i in range(0, len(set))
    ]

    # Estimate the equivalent RMS with EDGES thermal noise
    rmsNoise = 0.025
    rmsPolyNoise = [np.sqrt(x**2 + rmsNoise**2) for x in rmsPoly]
    rmsPhysNoise = [np.sqrt(x**2 + rmsNoise**2) for x in rmsPhys]

    # --------------------------------------------------------------------------- #
    # Fit the simulated data that includes the fake signal feature
    # --------------------------------------------------------------------------- #
    noiseCov = [np.diag(x * x) for x in n]
    rcvSig = [[
        np.zeros([params_sig.shape[0], set_sig[i].shape[1]])
        for i in range(len(v))
    ] for j in range(len(foregroundComponents))]
    rcvRMS = [[np.zeros([1, set_sig[i].shape[1]]) for i in range(len(v))]
              for j in range(len(foregroundComponents))]

    tempFolder = tempfile.mkdtemp()
    tempFrequency = os.path.join(tempFolder, 'frequency')
    tempComponents = os.path.join(tempFolder, 'components')
    tempNoiseCov = os.path.join(tempFolder, 'noisecov')
    tempSignalParams = os.path.join(tempFolder, 'signalparams')
    tempData = os.path.join(tempFolder, 'data')
    tempRecoveredSignal = os.path.join(tempFolder, 'recoveredsignal')
    tempRecoveredRMS = os.path.join(tempFolder, 'recoveredrms')

    #dump(v, tempFrequency);
    #dump(foregroundComponents, tempComponents);
    #dump(noiseCov, tempNoiseCov);
    #dump(params_sig, tempSignalParams);
    #dump(set_sig, tempData);

    nFrequencySets = len(v)
    nComponentSets = len(foregroundComponents)
    nSignalParams = params_sig.shape[0]
    nSpectra = params_mini.shape[1]

    recoveredSignal = np.memmap(tempRecoveredSignal,
                                dtype='float64',
                                shape=(nFrequencySets, nComponentSets,
                                       nSignalParams, nSpectra),
                                mode="w+")

    recoveredRMS = np.memmap(tempRecoveredRMS,
                             dtype='float64',
                             shape=(nFrequencySets, nComponentSets, nSpectra),
                             mode="w+")

    for k in range(len(v)):
        for j in range(len(foregroundComponents)):

            # Filename for CSV output
            #filename = "{}_gridsearch_freq{}_model{}.txt".format(outputBase, k, j);
            #print("Writing: {}".format(filename));

            # Delete any existing file
            #try:
            #  os.remove(filename)
            #except OSError:
            #  print("No existing file.  Continuing.");
            #  pass

            # Start the parallel jobs
            results = Parallel(n_jobs=nprocs, verbose=1000, max_nbytes=100)(
                delayed(processJob)
                (i, j, k, v, set_sig, noiseCov, foregroundComponents, models.
                 flattenedGaussian, params_sig, recoveredSignal, recoveredRMS)
                for i in range(nSpectra))

            # Start a parallel processing session
            #m = multiprocessing.Manager();
            #q = m.Queue();
            #pool = multiprocessing.Pool(processes=nprocs);

            # Start the listener that writes rows whenever a job finishes
            #pool.apply_async(listener, (q,));

            # Populate all of the jobs for parallel processing
            #pool.starmap(processJob, [(q, v[k], set_sig[k],
            #       noiseCov[k],
            #       foregroundComponents[j][k],
            #       models.flattenedGaussian,
            #       params_sig,
            #       filename,
            #       i) for i in range(params_mini.shape[1])] );

            # Kill the queue and close the pool
            #q.put('kill');
            #pool.close();

    # --------------------------------------------------------------------------- #
    # Read back in simulated signal fit results
    # --------------------------------------------------------------------------- #
    for k in range(len(v)):
        for j in range(len(foregroundComponents)):

            # Filename for CSV input
            filename = "{}_gridsearch_freq{}_model{}.txt".format(
                outputBase, k, j)
            print("Reading: {}".format(filename))

            with open(filename, "r") as csvFile:
                csvReader = csv.reader(csvFile)
                nrow = 0
                for row in csvReader:
                    recoveredSig[j][k][:, nrow] = row[-4:]
                    recoveredRMS[j][k][nrow] = row[1]
                    nrow = nrow + 1

    # --------------------------------------------------------------------------- #
    # Plot the results
    # --------------------------------------------------------------------------- #

    plt.figure(1)
    plt.clf()
    lw1 = 2
    lw2 = 2
    plt.plot(rmsPhys[0][-1] * np.array([1.0, 1.0]),
             np.array([1500, 8000]),
             'k--',
             linewidth=lw1)
    plt.plot(rmsPhys[1][-1] * np.array([1.0, 1.0]),
             np.array([1500, 8000]),
             'g:',
             linewidth=lw1)
    plt.plot(rmsPoly[0][-1] * np.array([1.0, 1.0]),
             np.array([1500, 8000]),
             'r-',
             linewidth=lw1)
    plt.plot(rmsPoly[1][-1] * np.array([1.0, 1.0]),
             np.array([1500, 8000]),
             'b-',
             linewidth=lw1)
    plt.hist(rmsPhysNoise[0],
             bins=np.logspace(-2, -1, 50),
             density=False,
             histtype='step',
             edgecolor='black',
             linewidth=lw2,
             linestyle='dashed',
             alpha=1)
    plt.hist(rmsPhysNoise[1],
             bins=np.logspace(-2, -1, 50),
             density=False,
             histtype='step',
             edgecolor='green',
             linewidth=lw2,
             linestyle='dotted',
             alpha=1)
    plt.hist(rmsPolyNoise[0],
             bins=np.logspace(-2, 0, 100),
             density=False,
             histtype='stepfilled',
             facecolor='red',
             linewidth=lw2,
             alpha=0.5)
    plt.hist(rmsPolyNoise[1],
             bins=np.logspace(-2, -1, 50),
             density=False,
             histtype='stepfilled',
             facecolor='blue',
             linewidth=lw2,
             alpha=0.3)
    ax = plt.gca()
    ax.set_xscale('log')
    ax.set_yscale('log')
    plt.ylim([3, 1e5])
    plt.xlim([1e-2, 4e-1])
    plt.xlabel('RMS residual [K]')
    plt.ylabel('Samples')
    plt.legend([
        '50-100 MHz linphys', '63-99 MHz linphys', '50-100 MHz poly',
        '63-99 MHz poly'
    ],
               loc='upper right')
    plt.title(
        'Histogram of RMS residual \nto simulated foregrounds plus noise')
    plt.show()

    plt.figure(2)
    plt.clf()
    lw1 = 2
    lw2 = 2
    plt.plot(np.sqrt(rmsPhys[0][-1]**2 - 0.025**2) * np.array([1.0, 1.0]),
             np.array([4500, 5500]),
             'k--',
             linewidth=lw1)
    plt.plot(np.sqrt(rmsPhys[1][-1]**2 - 0.025**2) * np.array([1.0, 1.0]),
             np.array([4500, 5500]),
             'g:',
             linewidth=lw1)
    plt.plot(np.sqrt(rmsPoly[0][-1]**2 - 0.025**2) * np.array([1.0, 1.0]),
             np.array([4500, 5500]),
             'r-',
             linewidth=lw1)
    plt.plot(np.sqrt(rmsPoly[1][-1]**2 - 0.025**2) * np.array([1.0, 1.0]),
             np.array([4500, 5500]),
             'b-',
             linewidth=lw1)
    plt.hist(rmsPhys[0],
             bins=np.logspace(-5, 0, 50),
             density=False,
             histtype='step',
             edgecolor='black',
             linewidth=lw2,
             linestyle='dashed',
             alpha=1)
    plt.hist(rmsPhys[1],
             bins=np.logspace(-5, 0, 50),
             density=False,
             histtype='step',
             edgecolor='green',
             linewidth=lw2,
             linestyle='dotted',
             alpha=1)
    plt.hist(rmsPoly[0],
             bins=np.logspace(-4, 0, 40),
             density=False,
             histtype='stepfilled',
             facecolor='red',
             linewidth=lw2,
             alpha=0.5)
    plt.hist(rmsPoly[1],
             bins=np.logspace(-5, 0, 50),
             density=False,
             histtype='stepfilled',
             facecolor='blue',
             linewidth=lw2,
             alpha=0.5)
    ax = plt.gca()
    ax.set_xscale('log')
    #ax.set_yscale('log');
    plt.ylim([0, 6000])
    plt.xlim([3e-6, 1])
    plt.xlabel('RMS residual [K]')
    plt.ylabel('Samples')
    plt.legend([
        '50-100 MHz linphys', '63-99 MHz linphys', '50-100 MHz poly',
        '63-99 MHz poly'
    ],
               loc='upper left')
    plt.title('Histogram of RMS residual to simulated foregrounds')
    plt.show()

    plt.figure(3)
    plt.clf()
    plt.plot(v[0], diffPhys[0][:, -1])
    plt.plot(v[1], diffPhys[1][:, -1])
    plt.plot(v[0], diffPoly[0][:, -1])
    plt.plot(v[1], diffPoly[1][:, -1])
    print(rmsPhys[0][-1])
    print(rmsPhys[1][-1])
    print(rmsPoly[0][-1])
    print(rmsPoly[1][-1])
    plt.show()

    plt.figure(4)
    plt.clf()
    plt.hist(rcvSig[0][1][3, 0:648],
             bins=np.linspace(-2.1, 2.1, 22),
             density=False,
             histtype='stepfilled',
             facecolor='blue',
             linewidth=lw2,
             alpha=0.5)
    plt.hist(rcvSig[0][0][3, 0:648],
             bins=np.linspace(-2.1, 2.1, 22),
             density=False,
             histtype='stepfilled',
             facecolor='red',
             linewidth=lw2,
             alpha=0.75)
    plt.plot([-0.5, -0.5], [0, 500], 'k:')
    plt.xlabel("Best fit amplitude [K]")
    plt.ylabel("Samples")
    plt.legend(["True (-0.5 K)", "63-99 MHz poly", "50-100 MHz poly"])
    plt.ylim([0, 450])
    plt.savefig("{}_hist_recovered_amp_poly.png".format(outputBase))

    plt.figure(5)
    plt.clf()
    plt.hist(rcvSig[1][1][3, 0:648],
             bins=np.linspace(-2.1, 2.1, 22),
             density=False,
             histtype='stepfilled',
             facecolor='blue',
             linewidth=lw2,
             alpha=0.5)
    plt.hist(rcvSig[1][0][3, 0:648],
             bins=np.linspace(-2.1, 2.1, 22),
             density=False,
             histtype='stepfilled',
             facecolor='red',
             linewidth=lw2,
             alpha=0.75)
    plt.plot([-0.5, -0.5], [0, 500], 'k:')
    plt.xlabel("Best fit amplitude [K]")
    plt.ylabel("Samples")
    plt.legend(["True (-0.5 K)", "63-99 MHz linphys", "50-100 MHz linphys"])
    plt.ylim([0, 450])
    plt.savefig("{}_hist_recovered_amp_phys.png".format(outputBase))
Exemple #3
0
#  nta2 = ta2.shape[0];

print('Integrating...')
ta1Sum = np.sum(ta1, axis=0)
#ta2Sum = np.sum(ta2, axis=0);
taMean = (ta1Sum) / (nta1)

print('Select frequency range...')
indices = np.array(range(nfreqsfull))[(freqsfull >= fmin)
                                      & (freqsfull <= fmax)]
taMean = taMean[indices]
freqs = freqsfull[indices]
nfreqs = len(freqs)

print('Fit polynomial...')
components = models.linearPolynomialComponents(freqs, 75, nterms, beta=beta)
fit, rms = models.fitLinear(taMean, components)
residuals = taMean - models.linearPolynomial(freqs, 75, fit, beta=beta)
kernel = np.ones(60) / 60
smooth = np.convolve(residuals, kernel, 'same')
print('RMS: {}'.format(rms))

# Make the requested plots
print('Plotting...')
fig = 1
cmap = 'jet'

if bPlotResiduals:

    ymax = np.max(residuals)
    ymin = np.min(residuals)
Exemple #4
0
def main():

    # Parse command line arguments
    parser = argparse.ArgumentParser(description='Fit simulated data.')
    parser.add_argument('infile',
                        help='Input file containing foreground simulations')
    parser.add_argument('-m',
                        '--model',
                        nargs=1,
                        default=['linpoly'],
                        help='linpoly, linphys, or linlog')
    parser.add_argument(
        '-s',
        '--addsig',
        action='store_true',
        help='Adds a simulated signal to every foreground realization.')
    parser.add_argument(
        '-n',
        '--addnoise',
        action='store_true',
        help='Adds simulated thermal noise to every foreground realization.')
    parser.add_argument(
        '-g',
        '--grid',
        action='store_true',
        help="Perform grid search for a signal in each simulation.")
    parser.add_argument('-f',
                        '--fit',
                        action='store_true',
                        help="Fit a linear model to each simulation.")
    parser.add_argument(
        '-c',
        '--cov',
        action='store_true',
        help=
        "Use an estimated covariance matrix in the grid search.  ONLY used in grid search.  Currently this is placeholder diagonal matrix based on thermal noise."
    )
    parser.add_argument('-t', '--nterms', nargs=1, type=int, default=[5])
    parser.add_argument('-b', '--beta', nargs=1, type=int, default=[-2.5])
    parser.add_argument(
        '-r',
        '--thermal',
        nargs=1,
        type=float,
        default=[0.025],
        help='Specify the thermal noise per frequency bin in Kelvin')
    parser.add_argument('-o',
                        '--output',
                        nargs=1,
                        default='./',
                        help='Path to directory for output files.')

    args = parser.parse_args()
    print(args)

    outputDir = args.output[0]
    #'/home/loco/analysis/';
    inputFile = args.infile
    #'/home/loco/analysis/foreground_mini_simulations_freq0.csv';
    bIncludeSignal = args.addsig
    #False;
    bIncludeNoise = args.addnoise
    #False;
    bDoGridSearch = args.grid
    #False;
    bDoForegroundOnlyFit = args.fit
    #False;
    bUseCov = args.cov
    model = args.model[0]
    #'linpoly';
    thermalNoise = args.thermal[0]
    #0.025;
    beta = args.beta[0]
    #-2.5;
    nterms = args.nterms[0]
    #5;
    vc = 75.0
    nInputParams = 7
    signalFunction = models.flattenedGaussian
    signalArgs = [80, 20, 7, -0.5]

    # --------------------------------------------------------------------------- #
    # Read in the input foreground simulation file
    # --------------------------------------------------------------------------- #
    nrows = models.numRows(inputFile)
    set, freqs, params, paramLabels = models.readFile(inputFile)
    nfreqs = len(freqs)

    # --------------------------------------------------------------------------- #
    # Add any requested additions
    # --------------------------------------------------------------------------- #
    if bIncludeSignal is True:
        set = (set.transpose() +
               signalFunction(freqs, signalArgs).transpose()).transpose()

    if bIncludeNoise is True:
        set = set + thermalNoise * np.random.standard_normal(size=set.shape)

    # Make sure things are float32
    freqs = freqs
    set = set

    # --------------------------------------------------------------------------- #
    # Generate the foreground components of the models we'll fit
    # --------------------------------------------------------------------------- #
    if model == 'linpoly':
        foregroundComponents = models.linearPolynomialComponents(freqs,
                                                                 vc,
                                                                 nterms,
                                                                 beta=beta)
    elif model == 'linphys':
        foregroundComponents = models.linearPhysicalComponents(freqs, vc)
    elif model == 'linlog':
        foregroundComponents = models.linearLogExpansionComponents(freqs,
                                                                   vc,
                                                                   nterms,
                                                                   beta=beta)
    else:
        print("Failed.  Can't find model: {}".format(model))
        return

    # --------------------------------------------------------------------------- #
    # Fit the simulated foregrounds
    # --------------------------------------------------------------------------- #
    if bDoForegroundOnlyFit is True:

        fits = [
            models.fitLinear(set[:, x], foregroundComponents)
            for x in range(set.shape[1])
        ]

        fo_recovParams = np.array([x[0] for x in fits]).transpose()
        fo_recovRMS = np.array([x[1] for x in fits])
        fo_recovResiduals = set - foregroundComponents.dot(fo_recovParams)

        # Write results to file
        filename = "{}_fits_{}_sig{}_noise{}_terms{}_beta{}.csv".format(
            inputFile[:-4], model, bIncludeSignal, bIncludeNoise, nterms, beta)
        with open(filename, "w") as csvFile:
            csvWriter = csv.writer(csvFile)

            # Write header
            csvWriter.writerow(["f{}".format(n)
                                for n in range(nterms)] + ["RMS [K]"] +
                               ["{0:8.6f}".format(n) for n in freqs])

            # Write data
            for j in range(set.shape[1]):
                csvWriter.writerow(
                    np.concatenate((np.array(fo_recovParams[:, j]),
                                    fo_recovRMS[j], fo_recovResiduals[:, j])))

    # --------------------------------------------------------------------------- #
    # Do grid search for signal
    # --------------------------------------------------------------------------- #
    if bDoGridSearch is True:

        # Generate the signal grid for searching (20500 points)
        steps_sig = [
            np.linspace(55, 100, 10),
            np.linspace(5, 50, 10),
            np.linspace(1, 9, 5),
            np.linspace(-2, 2, 41)
        ]

        # Generate the signal grid for searching (1004400 points)
        steps_sig = [
            np.linspace(60, 90, 31),
            np.linspace(1, 40, 40),
            np.linspace(1, 10, 10),
            np.linspace(-2, 2, 81)
        ]

        # Generate all parmeter grid points
        params_sig = models.populateGrid(steps_sig)

        # Generate realizations of the signal at each parameter grid point
        realizations_sig = np.array([
            models.flattenedGaussian(freqs, p) for p in params_sig.transpose()
        ]).transpose()

        # Generate noise covariance matrix
        if bUseCov:
            noiseCov = np.diag(thermalNoise**2 * np.ones([nfreqs]))
        else:
            noiseCov = None

        # Allocate output arrays
        gs_recovParams = np.zeros([nterms, set.shape[1]])
        gs_recovSig = np.zeros([params_sig.shape[0], set.shape[1]])
        gs_recovRMS = np.zeros(set.shape[1])
        gs_recovResiduals = np.zeros([nfreqs, set.shape[1]])

        # Write results to file on the fly
        filename = "{}_gridsearch{}_{}_sig{}_noise{}_cov{}_terms{}_beta{}.csv".format(
            inputFile[:-4], params_sig.shape[1], model, bIncludeSignal,
            bIncludeNoise, bUseCov, nterms, beta)

        with open(filename, "w") as csvFile:
            csvWriter = csv.writer(csvFile)

            # Write header
            csvWriter.writerow(
                ["f{}".format(n) for n in range(nterms)] +
                ["s{}".format(n) for n in range(params_sig.shape[0])] +
                ["RMS [K]"] + ["{0:8.6f}".format(n) for n in freqs])

            # Calculate foreground model compontens inversion once
            XTX = np.linalg.inv(
                np.dot(foregroundComponents.T, foregroundComponents))
            XT = foregroundComponents.T

            # Loop over all simulated spectra and perform the grid search on each
            tic = time.time()
            for i in range(set.shape[1]):

                #fits, rmss = models.searchSignalTrials(set[:,i], foregroundComponents,
                #                                 freqs, signalFunction, params_sig, dataCov=noiseCov);

                #fits, rmss = models.searchSignalRealizations(set[:,i], foregroundComponents,
                #                                 freqs, realizations_sig, dataCov=noiseCov);

                fits, rmss = models.searchSignalRealizationsFast(
                    set[:, i], XTX, XT, realizations_sig)

                index = np.argmin(rmss)
                gs_recovParams[:, i] = fits[:, index]
                gs_recovSig[:, i] = params_sig[:, index]
                gs_recovRMS[i] = rmss[index]
                gs_recovResiduals[:, i] = set[:, i] - foregroundComponents.dot(
                    gs_recovParams[:, i]) - signalFunction(
                        freqs, gs_recovSig[:, i])
                #gs_recovResiduals[:,i] = set[:,i] - foregroundComponents.dot(gs_recovParams[:,i]) - realizations_sig[:,i];

                # Write row
                csvWriter.writerow(
                    list(gs_recovParams[:, i]) + list(gs_recovSig[:, i]) +
                    [gs_recovRMS[i]] + list(gs_recovResiduals[:, i]))

                # Print update to stdout
                print(
                    "{0:8.2f} s ({1} of {2}): RMS {3:6.3} K -- {4:6.3f}, {5:6.3f}, {6:6.3f}, {7:6.3f}"
                    .format(time.time() - tic, i, set.shape[1], gs_recovRMS[i],
                            gs_recovSig[0, i], gs_recovSig[1, i],
                            gs_recovSig[2, i], gs_recovSig[3, i]))