def main():
# Ignore divide by zero and related warnings caused by 3-position switch
# correction applied to full bandwidth (including <50 MHz)
np.seterr(divide='ignore', invalid='ignore')
# Parse command line arguments
parser = argparse.ArgumentParser(description='Plots contents of ACQ file.');
parser.add_argument('inputFile', help='Options for plotting contents of ACQ file.');
parser.add_argument('-o', '--output',
nargs=1, default=['.'],
help='Output directory.');
parser.add_argument('-b', '--start',
nargs=1, type=int, default=[0],
help='Start reading on specified line in the file (first line is 0). Negative values count back from end of file. There are two lines per spectrum and six lines per switch cycle.');
parser.add_argument('-e', '--stop',
nargs=1, type=int, default=[9999999],
help='Stop reading before specified line in the file. Negative values count back from end of file. There are two lines per spectrum and six lines per switch cycle.');
parser.add_argument('-l', '--fmin',
nargs=1, type=float, default=[50],
help='Minimum frequency to plot.');
parser.add_argument('-u', '--fmax',
nargs=1, type=float, default=[199],
help='Maximum frequency to plot.');
parser.add_argument('-t', '--thin',
nargs=1, type=int, default=[1],
help='Reduce the number of spectra used in calculations and plots by the specified factor.');
parser.add_argument('-r', '--raw',
action='store_true',
help='Plot the raw spectra for each switch position.');
parser.add_argument('-c', '--corrected',
action='store_true',
help='Plot the individual 3-position switch corrected spectra overlaid on the same plot.');
parser.add_argument('-i', '--integrate',
action='store_true',
help='Plot the integration (mean) of the 3-position corrected spectra.');
parser.add_argument('-w', '--waterfall',
action='store_true',
help='Plot a waterfall of spectra. Used with--corrected and/or --raw.');
parser.add_argument('-f', '--fit',
action='store_true',
help='Fit the integrated (mean) spectrum and plot the residuals. Specify model details using --model and --nterms.');
parser.add_argument('-m', '--model',
nargs=1, default=['linpoly'],
help='Model to use if --fit is set. Specify one of: linpoly, linphys, linlog.');
parser.add_argument('-n', '--nterms',
nargs=1, type=int, default=[5],
help='Number of terms to use in model fit.');
parser.add_argument('-g', '--flag',
action='store_true',
help='Flag RFI in corrected integrations and residual plots.');
parser.add_argument('-p', '--power',
nargs=2, type=float, default=[100, 2500],
help='Provide bounds for allowed minimum and maximum average corrected spectrum power for use when flagging.');
parser.add_argument('-d', '--threshold',
nargs=6, type=float, default=[3, 0.001, 20, 11, 75, 0],
help='Provide sigma, tolerance, maxiter, and polynomial nterms, center freq, and beta for use when flagging channels.');
args = parser.parse_args();
print(args);
inputFile = args.inputFile;
outputDir = args.output[0];
start = args.start[0];
stop = args.stop[0];
fmin = args.fmin[0];
fmax = args.fmax[0];
thin = args.thin[0];
model = args.model[0].lower();
nterms = args.nterms[0];
power = args.power;
threshold = args.threshold;
bFlagRFI = args.flag;
bPlotFit = args.fit;
bPlotRaw = args.raw;
bPlotCorrected = args.corrected;
bPlotIntegration = args.integrate;
bPlotWaterfall = args.waterfall;
# Hard coded parameters -- should be added to commandline args at some point
vc = 75;
beta = -2.5;
nkernel = 64;
inputDir = os.path.dirname(inputFile);
inputFilenameBase = os.path.splitext(os.path.basename(inputFile))[0];
outputFullBase = outputDir + '/' + inputFilenameBase;
# Read the input file
spec, anc, freqs = analysis.readAcq(inputFile, start=start, stop=stop, verbose=1, seek=True);
nspec = len(spec);
nfreqs = len(freqs);
channelSize = anc[0,10];
nsize = nspec - (nspec % 3);
p0 = spec[0:nsize:int(3*thin)];
p1 = spec[1:nsize:int(3*thin)];
p2 = spec[2:nsize:int(3*thin)];
# Make the requested plots
fig = 1;
cmap = 'jet';
lw = 0.75;
if bPlotRaw:
print('Plotting raw spectra summary...');
plt.figure(fig);
fig = fig + 1;
plt.clf();
plt.plot(freqs, np.max(10*np.log10(p0.transpose()), axis=1), color=[0.75,0.75,0.75], linewidth=lw);
h0, = plt.plot(freqs, np.mean(10*np.log10(p0.transpose()), axis=1), 'k-', linewidth=lw, label='p0 (antenna)');
plt.plot(freqs, np.min(10*np.log10(p0.transpose()), axis=1), color=[0.5,0.5,0.5], linewidth=lw);
plt.plot(freqs, np.max(10*np.log10(p1.transpose()), axis=1), color=[0.75,0.75,1], linewidth=lw);
h1, = plt.plot(freqs, np.mean(10*np.log10(p1.transpose()), axis=1), 'b-', linewidth=lw, label='p1 (antenna)');
plt.plot(freqs, np.min(10*np.log10(p1.transpose()), axis=1), color=[0.5,0.5,1], linewidth=lw);
plt.plot(freqs, np.max(10*np.log10(p2.transpose()), axis=1), color=[1,0.75,0.75], linewidth=lw);
h2, = plt.plot(freqs, np.mean(10*np.log10(p2.transpose()), axis=1), 'r-', linewidth=lw, label='p2 (antenna)');
plt.plot(freqs, np.min(10*np.log10(p2.transpose()), axis=1), color=[1,0.5,0.5], linewidth=lw);
plt.xlabel("Frequency [MHz]");
plt.ylabel("p [dB]");
plt.ylim([-90, -50]);
plt.legend(handles=[h0, h1, h2]);
plt.savefig(outputFullBase + '_raw.png');
if bPlotWaterfall:
print('Plotting raw spectra waterfalls...');
clim = [-90, -75];
plt.figure(fig);
fig = fig + 1;
plt.clf();
plt.imshow(10*np.log10(p0), extent=[freqs[0], freqs[len(freqs)-1], len(p0)*thin, 0], cmap=cmap, aspect='auto', origin='upper');
plt.clim(clim);
plt.colorbar(label='Arbitrary power [dB]');
plt.xlabel("Frequency [MHz]");
plt.ylabel("Spectrum # (p0)");
plt.savefig(outputFullBase + '_raw_waterfall_p0.png');
plt.figure(fig);
fig = fig + 1;
plt.clf();
plt.imshow(10*np.log10(p1), extent=[freqs[0], freqs[len(freqs)-1], len(p0)*thin, 0], cmap=cmap, aspect='auto', origin='upper');
plt.clim(clim);
plt.colorbar(label='Arbitrary power [dB]');
plt.xlabel("Frequency [MHz]");
plt.ylabel("Spectrum # (p1)");
plt.savefig(outputFullBase + '_raw_waterfall_p1.png');
plt.figure(fig);
fig = fig + 1;
plt.clf();
plt.imshow(10*np.log10(p2), extent=[freqs[0], freqs[len(freqs)-1], len(p0)*thin, 0], cmap=cmap, aspect='auto', origin='upper');
plt.clim(clim);
plt.colorbar(label='Arbitrary power [dB]');
plt.xlabel("Frequency [MHz]");
plt.ylabel("Spectrum # (p2)");
plt.savefig(outputFullBase + '_raw_waterfall_p2.png');
if bPlotCorrected or bPlotIntegration or bPlotFit:
# Apply the 3-position correction
print('Applying 3-position correction...');
cor = analysis.correct(p0, p1, p2);
# Reduce the frequency range to the specified bounds
freqssub = freqs[(freqs>=fmin) & (freqs<=fmax)];
corsub = cor[:, (freqs>=fmin) & (freqs<=fmax)];
print('Frequency range reducted to {}-{} MHz (keeping {} channels)'.format(fmin, fmax, corsub.shape[1]));
# Integrate without filtering
if bFlagRFI:
# Integrate after flagging bad spectra
rowWeights = analysis.flagAveragePower(corsub, maxpwr=power[1], minpwr=power[0]);
corsubmean = np.sum((corsub.transpose()*rowWeights).transpose(), axis=0) / np.sum(rowWeights);
print('Number of spectra flagged: {} out of {}'.format(corsub.shape[0] - np.sum(rowWeights), corsub.shape[0]));
flagComponents = models.linearPolynomialComponents(freqssub, vc=threshold[4], nterms=threshold[3], beta=threshold[5]);
channelWeights, flagrms = analysis.flagChannels(corsubmean, flagComponents, sigma=threshold[0], tol=threshold[1], maxiter=threshold[2]);
print('Number of channels flagged: {} out of {}'.format(corsub.shape[1] - np.sum(channelWeights), corsub.shape[1]));
else:
corsubmean = np.mean(corsub, axis=0);
channelWeights = np.ones(corsubmean.shape);
if bPlotCorrected:
print('Plotting 3-position corrected spectra summary...');
plt.figure(fig);
fig = fig + 1;
plt.clf();
plt.plot(freqssub, np.max(corsub.transpose(), axis=1), 'r-', linewidth=lw);
plt.plot(freqssub, np.min(corsub.transpose(), axis=1), 'b-', linewidth=lw);
plt.plot(freqssub, np.mean(corsub.transpose(), axis=1), 'k-', linewidth=lw);
plt.xlabel("Frequency [MHz]");
plt.ylabel("$T_{3pos}$ [K]");
plt.legend(['Max hold', 'Min hold', 'Mean']);
plt.xlim([fmin, fmax]);
ymax = np.nanmax(np.where(~np.isinf(corsub), corsub, np.nan));
ymin = np.nanmin(np.where(~np.isinf(corsub), corsub, np.nan));
if ymax>10000:
ymax = np.nanmedian(corsub[:,0])*1.5;
ydiff = ymax - ymin;
ypad = ydiff * 0.1;
plt.ylim([ymin-ypad, ymax+ypad]);
plt.savefig(outputFullBase + '_3pos.png');
if bPlotWaterfall:
print('Plot 3-position corrected spectra waterfall...');
plt.figure(fig);
fig = fig + 1;
plt.clf();
plt.imshow(corsub, extent=[freqssub[0], freqssub[len(freqssub)-1], len(corsub)*thin, 0], cmap=cmap, aspect='auto', origin='upper');
plt.clim([ymin, ymax]);
plt.colorbar(label='$T_{3pos}$ [K]');
plt.xlabel("Frequency [MHz]");
plt.ylabel("Spectrum #");
plt.savefig(outputFullBase + '_3pos_waterfall.png');
if bPlotIntegration:
print('Plotting 3-position corrected integrated spectrum...');
ind = [i for i in range(len(channelWeights)) if channelWeights[i]==0];
plotdata = corsubmean.copy();
plotdata[ind] = np.nan;
plt.figure(fig);
fig = fig + 1;
plt.clf();
plt.plot(freqssub, plotdata, linewidth=lw);
plt.xlabel("Frequency [MHz]");
plt.ylabel("$T_{3pos}$ [K]");
plt.xlim([fmin, fmax]);
ymax = np.nanmax(np.where(~np.isinf(corsubmean), corsubmean, np.nan));
ymin = np.nanmin(np.where(~np.isinf(corsubmean), corsubmean, np.nan));
if ymax>10000:
ymax = np.nanmedian(corsub[:,0])*1.5;
ydiff = ymax - ymin;
ypad = ydiff * 0.1;
plt.ylim([ymin-ypad, ymax+ypad]);
plt.savefig(outputFullBase + '_3pos_integration.png');
if bPlotFit:
print('Plotting residuals to model fit...');
# Fit with model and get residuals
if model == 'linpoly':
components = models.linearPolynomialComponents(freqssub, vc, nterms, beta=beta);
elif model == 'linphys':
if nterms is not 5:
print("Using 5 terms as requried by 'linphys' model.");
nterms = 5;
components = models.linearPhysicalComponents(freqssub, vc);
elif model == 'linlog':
components = models.linearLogExpansionComponents(freqssub, vc, nterms, beta=beta);
else:
print("Specified model '{}' was not recognized. Using 'linpoly'.".format(model) );
components = models.linearPolynomialComponents(freqssub, vc, nterms, beta=beta);
# Do the fit
ind = [i for i in range(len(channelWeights)) if ((channelWeights[i]==1) & (not np.isnan(corsubmean[i])))];
fit, rms = models.fitLinear(corsubmean[ind], components[ind,:]);
residuals = channelWeights * (corsubmean - np.dot(components, fit));
rms = np.nanstd(residuals[ind]);
# Smooth residuals with boxcar kernel
kernel = np.ones(nkernel) / nkernel;
smoothres = np.convolve(residuals, kernel, 'same');
smoothrms = np.nanstd(smoothres);
print('RMS: {:.4f} K'.format(rms));
print('RMS smoothed ({:}): {:.4f} K'.format(nkernel, smoothrms));
ind = [i for i in range(len(channelWeights)) if channelWeights[i]==0];
plotres = residuals.copy();
plotres[ind] = np.nan;
plt.figure(fig);
fig = fig + 1;
plt.clf();
plt.plot(freqssub, plotres, 'b-', linewidth=lw);
plt.plot(freqssub, smoothres, 'k-', linewidth=lw);
plt.xlabel("Frequency [MHz]");
plt.ylabel("$T_{res}$ [K]");
plt.xlim([fmin, fmax]);
plt.legend(['{:.1f} kHz'.format(1e3*channelSize), '{:.1f} kHz (smoothed)'.format(1e3*nkernel*channelSize)]);
ymax = np.nanmax(np.where(~np.isinf(residuals), residuals, np.nan));
ymin = np.nanmin(np.where(~np.isinf(residuals), residuals, np.nan));
if ymax>10000:
ymax = np.nanmedian(residuals)*1.5;
ydiff = ymax - ymin;
ypad = ydiff * 0.1;
plt.ylim([ymin-ypad, ymax+ypad]);
plt.savefig(outputFullBase + '_3pos_residuals_{}_nterms{}.png'.format(model, nterms));
def main():
# Define the parameters for the run
# outputBase = 'G:/My Drive/EDGES/mcmc/run2';
# dataFile = 'G:/My Drive/Publications/Papers/2017_EDGES_Low/PublicData/figure1_plotdata.csv';
outputBase = '/home/loco/analysis/run3'
dataFile = '/home/loco/analysis/figure1_plotdata.csv'
nprocs = 2
vc = 75
beta = -2.5
nterms = 5
# Load public EDGS data from Bowman et al. 2018 Figure 1
data = np.genfromtxt(dataFile, delimiter=',', skip_header=1)
f = data[:, 0]
w = data[:, 1]
d = data[:, 2]
# Extract frequency coordinates from the data file for small and large subbands
v = [
data[(data[:, 1] > 0), 0], data[(data[:, 0] > 63) & (data[:, 0] < 99),
0]
]
# Extract the data for the same ranges
d = [
data[(data[:, 1] > 0), 2], data[(data[:, 0] > 63) & (data[:, 0] < 99),
2]
]
# Estimated thermal noise;
n = [0.025 * np.ones(d[i].size) for i in range(len(v))]
# --------------------------------------------------------------------------- #
# Make a set of simulated foreground realizations
# --------------------------------------------------------------------------- #
# Sample the model parameters in a multi-dimensional grid with points at the
# following parameter values:
steps = [[1250, 1500, 1750], [-2.6, -2.55, -2.5, -2.45, -2.4],
[-0.1, -0.05, 0, 0.05, 0.1], [-0.1, -0.05, 0, 0.05, 0.1],
[-0.1, -0.05, 0, 0.05, 0.1], [0, 0.05, 0.1], [0, 500, 750, 1000]]
steps_mini = [[1500, 1750], [-2.6, -2.5, -2.4], [-0.1, 0, 0.1],
[-0.1, 0, 0.1], [-0.1, 0, 0.1], [0, 0.1], [0, 1000]]
sig = [80, 20, 7, -0.5]
steps_sig = [
np.linspace(55, 100, 10),
np.linspace(5, 50, 10),
np.linspace(1, 9, 5),
np.linspace(-2, 2, 21)
]
# Generate all parmeter grid points
params = models.populateGrid(steps)
params_mini = models.populateGrid(steps_mini)
params_sig = models.populateGrid(steps_sig)
# Generate the realizations
set = [models.fullSet(x, vc, params[:-2, :], params[-2:, :]) for x in v]
set_mini = [
models.fullSet(x, vc, params_mini[:-2, :], params_mini[-2:, :])
for x in v
]
# Make a version of the realizations with a simulated signal feature
set_sig = [
np.add(set_mini[i].transpose(),
models.flattenedGaussian(v[i], sig)).transpose()
for i in range(len(v))
]
# Append the actual data to the end of the foreground-only realization set
set = [
np.hstack([set[i], d[i].reshape([v[i].size, 1])])
for i in range(len(v))
]
set_mini = [
np.hstack([set_mini[i], d[i].reshape([v[i].size, 1])])
for i in range(len(v))
]
# Generate the vector components of the models we'll fit
foregroundComponents = []
foregroundComponents.append([
models.linearPolynomialComponents(x, vc, nterms, beta=beta) for x in v
])
foregroundComponents.append(
[models.linearPhysicalComponents(x, vc) for x in v])
# --------------------------------------------------------------------------- #
# Fit the simulated/real data
# --------------------------------------------------------------------------- #
fits = [
models.fitLinear(x, y) for x, y in zip(set, foregroundComponents[0])
]
estimatesPoly = [x[0] for x in fits]
rmsPoly = [x[1] for x in fits]
diffPoly = [
set[i] - foregroundComponents[0][i].dot(estimatesPoly[i])
for i in range(0, len(set))
]
fits = [
models.fitLinear(x, y) for x, y in zip(set, foregroundComponents[1])
]
estimatesPhys = [x[0] for x in fits]
rmsPhys = [x[1] for x in fits]
diffPhys = [
set[i] - foregroundComponents[1][i].dot(estimatesPhys[i])
for i in range(0, len(set))
]
# Estimate the equivalent RMS with EDGES thermal noise
rmsNoise = 0.025
rmsPolyNoise = [np.sqrt(x**2 + rmsNoise**2) for x in rmsPoly]
rmsPhysNoise = [np.sqrt(x**2 + rmsNoise**2) for x in rmsPhys]
# --------------------------------------------------------------------------- #
# Fit the simulated data that includes the fake signal feature
# --------------------------------------------------------------------------- #
noiseCov = [np.diag(x * x) for x in n]
rcvSig = [[
np.zeros([params_sig.shape[0], set_sig[i].shape[1]])
for i in range(len(v))
] for j in range(len(foregroundComponents))]
rcvRMS = [[np.zeros([1, set_sig[i].shape[1]]) for i in range(len(v))]
for j in range(len(foregroundComponents))]
tempFolder = tempfile.mkdtemp()
tempFrequency = os.path.join(tempFolder, 'frequency')
tempComponents = os.path.join(tempFolder, 'components')
tempNoiseCov = os.path.join(tempFolder, 'noisecov')
tempSignalParams = os.path.join(tempFolder, 'signalparams')
tempData = os.path.join(tempFolder, 'data')
tempRecoveredSignal = os.path.join(tempFolder, 'recoveredsignal')
tempRecoveredRMS = os.path.join(tempFolder, 'recoveredrms')
#dump(v, tempFrequency);
#dump(foregroundComponents, tempComponents);
#dump(noiseCov, tempNoiseCov);
#dump(params_sig, tempSignalParams);
#dump(set_sig, tempData);
nFrequencySets = len(v)
nComponentSets = len(foregroundComponents)
nSignalParams = params_sig.shape[0]
nSpectra = params_mini.shape[1]
recoveredSignal = np.memmap(tempRecoveredSignal,
dtype='float64',
shape=(nFrequencySets, nComponentSets,
nSignalParams, nSpectra),
mode="w+")
recoveredRMS = np.memmap(tempRecoveredRMS,
dtype='float64',
shape=(nFrequencySets, nComponentSets, nSpectra),
mode="w+")
for k in range(len(v)):
for j in range(len(foregroundComponents)):
# Filename for CSV output
#filename = "{}_gridsearch_freq{}_model{}.txt".format(outputBase, k, j);
#print("Writing: {}".format(filename));
# Delete any existing file
#try:
# os.remove(filename)
#except OSError:
# print("No existing file. Continuing.");
# pass
# Start the parallel jobs
results = Parallel(n_jobs=nprocs, verbose=1000, max_nbytes=100)(
delayed(processJob)
(i, j, k, v, set_sig, noiseCov, foregroundComponents, models.
flattenedGaussian, params_sig, recoveredSignal, recoveredRMS)
for i in range(nSpectra))
# Start a parallel processing session
#m = multiprocessing.Manager();
#q = m.Queue();
#pool = multiprocessing.Pool(processes=nprocs);
# Start the listener that writes rows whenever a job finishes
#pool.apply_async(listener, (q,));
# Populate all of the jobs for parallel processing
#pool.starmap(processJob, [(q, v[k], set_sig[k],
# noiseCov[k],
# foregroundComponents[j][k],
# models.flattenedGaussian,
# params_sig,
# filename,
# i) for i in range(params_mini.shape[1])] );
# Kill the queue and close the pool
#q.put('kill');
#pool.close();
# --------------------------------------------------------------------------- #
# Read back in simulated signal fit results
# --------------------------------------------------------------------------- #
for k in range(len(v)):
for j in range(len(foregroundComponents)):
# Filename for CSV input
filename = "{}_gridsearch_freq{}_model{}.txt".format(
outputBase, k, j)
print("Reading: {}".format(filename))
with open(filename, "r") as csvFile:
csvReader = csv.reader(csvFile)
nrow = 0
for row in csvReader:
recoveredSig[j][k][:, nrow] = row[-4:]
recoveredRMS[j][k][nrow] = row[1]
nrow = nrow + 1
# --------------------------------------------------------------------------- #
# Plot the results
# --------------------------------------------------------------------------- #
plt.figure(1)
plt.clf()
lw1 = 2
lw2 = 2
plt.plot(rmsPhys[0][-1] * np.array([1.0, 1.0]),
np.array([1500, 8000]),
'k--',
linewidth=lw1)
plt.plot(rmsPhys[1][-1] * np.array([1.0, 1.0]),
np.array([1500, 8000]),
'g:',
linewidth=lw1)
plt.plot(rmsPoly[0][-1] * np.array([1.0, 1.0]),
np.array([1500, 8000]),
'r-',
linewidth=lw1)
plt.plot(rmsPoly[1][-1] * np.array([1.0, 1.0]),
np.array([1500, 8000]),
'b-',
linewidth=lw1)
plt.hist(rmsPhysNoise[0],
bins=np.logspace(-2, -1, 50),
density=False,
histtype='step',
edgecolor='black',
linewidth=lw2,
linestyle='dashed',
alpha=1)
plt.hist(rmsPhysNoise[1],
bins=np.logspace(-2, -1, 50),
density=False,
histtype='step',
edgecolor='green',
linewidth=lw2,
linestyle='dotted',
alpha=1)
plt.hist(rmsPolyNoise[0],
bins=np.logspace(-2, 0, 100),
density=False,
histtype='stepfilled',
facecolor='red',
linewidth=lw2,
alpha=0.5)
plt.hist(rmsPolyNoise[1],
bins=np.logspace(-2, -1, 50),
density=False,
histtype='stepfilled',
facecolor='blue',
linewidth=lw2,
alpha=0.3)
ax = plt.gca()
ax.set_xscale('log')
ax.set_yscale('log')
plt.ylim([3, 1e5])
plt.xlim([1e-2, 4e-1])
plt.xlabel('RMS residual [K]')
plt.ylabel('Samples')
plt.legend([
'50-100 MHz linphys', '63-99 MHz linphys', '50-100 MHz poly',
'63-99 MHz poly'
],
loc='upper right')
plt.title(
'Histogram of RMS residual \nto simulated foregrounds plus noise')
plt.show()
plt.figure(2)
plt.clf()
lw1 = 2
lw2 = 2
plt.plot(np.sqrt(rmsPhys[0][-1]**2 - 0.025**2) * np.array([1.0, 1.0]),
np.array([4500, 5500]),
'k--',
linewidth=lw1)
plt.plot(np.sqrt(rmsPhys[1][-1]**2 - 0.025**2) * np.array([1.0, 1.0]),
np.array([4500, 5500]),
'g:',
linewidth=lw1)
plt.plot(np.sqrt(rmsPoly[0][-1]**2 - 0.025**2) * np.array([1.0, 1.0]),
np.array([4500, 5500]),
'r-',
linewidth=lw1)
plt.plot(np.sqrt(rmsPoly[1][-1]**2 - 0.025**2) * np.array([1.0, 1.0]),
np.array([4500, 5500]),
'b-',
linewidth=lw1)
plt.hist(rmsPhys[0],
bins=np.logspace(-5, 0, 50),
density=False,
histtype='step',
edgecolor='black',
linewidth=lw2,
linestyle='dashed',
alpha=1)
plt.hist(rmsPhys[1],
bins=np.logspace(-5, 0, 50),
density=False,
histtype='step',
edgecolor='green',
linewidth=lw2,
linestyle='dotted',
alpha=1)
plt.hist(rmsPoly[0],
bins=np.logspace(-4, 0, 40),
density=False,
histtype='stepfilled',
facecolor='red',
linewidth=lw2,
alpha=0.5)
plt.hist(rmsPoly[1],
bins=np.logspace(-5, 0, 50),
density=False,
histtype='stepfilled',
facecolor='blue',
linewidth=lw2,
alpha=0.5)
ax = plt.gca()
ax.set_xscale('log')
#ax.set_yscale('log');
plt.ylim([0, 6000])
plt.xlim([3e-6, 1])
plt.xlabel('RMS residual [K]')
plt.ylabel('Samples')
plt.legend([
'50-100 MHz linphys', '63-99 MHz linphys', '50-100 MHz poly',
'63-99 MHz poly'
],
loc='upper left')
plt.title('Histogram of RMS residual to simulated foregrounds')
plt.show()
plt.figure(3)
plt.clf()
plt.plot(v[0], diffPhys[0][:, -1])
plt.plot(v[1], diffPhys[1][:, -1])
plt.plot(v[0], diffPoly[0][:, -1])
plt.plot(v[1], diffPoly[1][:, -1])
print(rmsPhys[0][-1])
print(rmsPhys[1][-1])
print(rmsPoly[0][-1])
print(rmsPoly[1][-1])
plt.show()
plt.figure(4)
plt.clf()
plt.hist(rcvSig[0][1][3, 0:648],
bins=np.linspace(-2.1, 2.1, 22),
density=False,
histtype='stepfilled',
facecolor='blue',
linewidth=lw2,
alpha=0.5)
plt.hist(rcvSig[0][0][3, 0:648],
bins=np.linspace(-2.1, 2.1, 22),
density=False,
histtype='stepfilled',
facecolor='red',
linewidth=lw2,
alpha=0.75)
plt.plot([-0.5, -0.5], [0, 500], 'k:')
plt.xlabel("Best fit amplitude [K]")
plt.ylabel("Samples")
plt.legend(["True (-0.5 K)", "63-99 MHz poly", "50-100 MHz poly"])
plt.ylim([0, 450])
plt.savefig("{}_hist_recovered_amp_poly.png".format(outputBase))
plt.figure(5)
plt.clf()
plt.hist(rcvSig[1][1][3, 0:648],
bins=np.linspace(-2.1, 2.1, 22),
density=False,
histtype='stepfilled',
facecolor='blue',
linewidth=lw2,
alpha=0.5)
plt.hist(rcvSig[1][0][3, 0:648],
bins=np.linspace(-2.1, 2.1, 22),
density=False,
histtype='stepfilled',
facecolor='red',
linewidth=lw2,
alpha=0.75)
plt.plot([-0.5, -0.5], [0, 500], 'k:')
plt.xlabel("Best fit amplitude [K]")
plt.ylabel("Samples")
plt.legend(["True (-0.5 K)", "63-99 MHz linphys", "50-100 MHz linphys"])
plt.ylim([0, 450])
plt.savefig("{}_hist_recovered_amp_phys.png".format(outputBase))
# nta2 = ta2.shape[0];
print('Integrating...')
ta1Sum = np.sum(ta1, axis=0)
#ta2Sum = np.sum(ta2, axis=0);
taMean = (ta1Sum) / (nta1)
print('Select frequency range...')
indices = np.array(range(nfreqsfull))[(freqsfull >= fmin)
& (freqsfull <= fmax)]
taMean = taMean[indices]
freqs = freqsfull[indices]
nfreqs = len(freqs)
print('Fit polynomial...')
components = models.linearPolynomialComponents(freqs, 75, nterms, beta=beta)
fit, rms = models.fitLinear(taMean, components)
residuals = taMean - models.linearPolynomial(freqs, 75, fit, beta=beta)
kernel = np.ones(60) / 60
smooth = np.convolve(residuals, kernel, 'same')
print('RMS: {}'.format(rms))
# Make the requested plots
print('Plotting...')
fig = 1
cmap = 'jet'
if bPlotResiduals:
ymax = np.max(residuals)
ymin = np.min(residuals)
def main():
# Parse command line arguments
parser = argparse.ArgumentParser(description='Fit simulated data.')
parser.add_argument('infile',
help='Input file containing foreground simulations')
parser.add_argument('-m',
'--model',
nargs=1,
default=['linpoly'],
help='linpoly, linphys, or linlog')
parser.add_argument(
'-s',
'--addsig',
action='store_true',
help='Adds a simulated signal to every foreground realization.')
parser.add_argument(
'-n',
'--addnoise',
action='store_true',
help='Adds simulated thermal noise to every foreground realization.')
parser.add_argument(
'-g',
'--grid',
action='store_true',
help="Perform grid search for a signal in each simulation.")
parser.add_argument('-f',
'--fit',
action='store_true',
help="Fit a linear model to each simulation.")
parser.add_argument(
'-c',
'--cov',
action='store_true',
help=
"Use an estimated covariance matrix in the grid search. ONLY used in grid search. Currently this is placeholder diagonal matrix based on thermal noise."
)
parser.add_argument('-t', '--nterms', nargs=1, type=int, default=[5])
parser.add_argument('-b', '--beta', nargs=1, type=int, default=[-2.5])
parser.add_argument(
'-r',
'--thermal',
nargs=1,
type=float,
default=[0.025],
help='Specify the thermal noise per frequency bin in Kelvin')
parser.add_argument('-o',
'--output',
nargs=1,
default='./',
help='Path to directory for output files.')
args = parser.parse_args()
print(args)
outputDir = args.output[0]
#'/home/loco/analysis/';
inputFile = args.infile
#'/home/loco/analysis/foreground_mini_simulations_freq0.csv';
bIncludeSignal = args.addsig
#False;
bIncludeNoise = args.addnoise
#False;
bDoGridSearch = args.grid
#False;
bDoForegroundOnlyFit = args.fit
#False;
bUseCov = args.cov
model = args.model[0]
#'linpoly';
thermalNoise = args.thermal[0]
#0.025;
beta = args.beta[0]
#-2.5;
nterms = args.nterms[0]
#5;
vc = 75.0
nInputParams = 7
signalFunction = models.flattenedGaussian
signalArgs = [80, 20, 7, -0.5]
# --------------------------------------------------------------------------- #
# Read in the input foreground simulation file
# --------------------------------------------------------------------------- #
nrows = models.numRows(inputFile)
set, freqs, params, paramLabels = models.readFile(inputFile)
nfreqs = len(freqs)
# --------------------------------------------------------------------------- #
# Add any requested additions
# --------------------------------------------------------------------------- #
if bIncludeSignal is True:
set = (set.transpose() +
signalFunction(freqs, signalArgs).transpose()).transpose()
if bIncludeNoise is True:
set = set + thermalNoise * np.random.standard_normal(size=set.shape)
# Make sure things are float32
freqs = freqs
set = set
# --------------------------------------------------------------------------- #
# Generate the foreground components of the models we'll fit
# --------------------------------------------------------------------------- #
if model == 'linpoly':
foregroundComponents = models.linearPolynomialComponents(freqs,
vc,
nterms,
beta=beta)
elif model == 'linphys':
foregroundComponents = models.linearPhysicalComponents(freqs, vc)
elif model == 'linlog':
foregroundComponents = models.linearLogExpansionComponents(freqs,
vc,
nterms,
beta=beta)
else:
print("Failed. Can't find model: {}".format(model))
return
# --------------------------------------------------------------------------- #
# Fit the simulated foregrounds
# --------------------------------------------------------------------------- #
if bDoForegroundOnlyFit is True:
fits = [
models.fitLinear(set[:, x], foregroundComponents)
for x in range(set.shape[1])
]
fo_recovParams = np.array([x[0] for x in fits]).transpose()
fo_recovRMS = np.array([x[1] for x in fits])
fo_recovResiduals = set - foregroundComponents.dot(fo_recovParams)
# Write results to file
filename = "{}_fits_{}_sig{}_noise{}_terms{}_beta{}.csv".format(
inputFile[:-4], model, bIncludeSignal, bIncludeNoise, nterms, beta)
with open(filename, "w") as csvFile:
csvWriter = csv.writer(csvFile)
# Write header
csvWriter.writerow(["f{}".format(n)
for n in range(nterms)] + ["RMS [K]"] +
["{0:8.6f}".format(n) for n in freqs])
# Write data
for j in range(set.shape[1]):
csvWriter.writerow(
np.concatenate((np.array(fo_recovParams[:, j]),
fo_recovRMS[j], fo_recovResiduals[:, j])))
# --------------------------------------------------------------------------- #
# Do grid search for signal
# --------------------------------------------------------------------------- #
if bDoGridSearch is True:
# Generate the signal grid for searching (20500 points)
steps_sig = [
np.linspace(55, 100, 10),
np.linspace(5, 50, 10),
np.linspace(1, 9, 5),
np.linspace(-2, 2, 41)
]
# Generate the signal grid for searching (1004400 points)
steps_sig = [
np.linspace(60, 90, 31),
np.linspace(1, 40, 40),
np.linspace(1, 10, 10),
np.linspace(-2, 2, 81)
]
# Generate all parmeter grid points
params_sig = models.populateGrid(steps_sig)
# Generate realizations of the signal at each parameter grid point
realizations_sig = np.array([
models.flattenedGaussian(freqs, p) for p in params_sig.transpose()
]).transpose()
# Generate noise covariance matrix
if bUseCov:
noiseCov = np.diag(thermalNoise**2 * np.ones([nfreqs]))
else:
noiseCov = None
# Allocate output arrays
gs_recovParams = np.zeros([nterms, set.shape[1]])
gs_recovSig = np.zeros([params_sig.shape[0], set.shape[1]])
gs_recovRMS = np.zeros(set.shape[1])
gs_recovResiduals = np.zeros([nfreqs, set.shape[1]])
# Write results to file on the fly
filename = "{}_gridsearch{}_{}_sig{}_noise{}_cov{}_terms{}_beta{}.csv".format(
inputFile[:-4], params_sig.shape[1], model, bIncludeSignal,
bIncludeNoise, bUseCov, nterms, beta)
with open(filename, "w") as csvFile:
csvWriter = csv.writer(csvFile)
# Write header
csvWriter.writerow(
["f{}".format(n) for n in range(nterms)] +
["s{}".format(n) for n in range(params_sig.shape[0])] +
["RMS [K]"] + ["{0:8.6f}".format(n) for n in freqs])
# Calculate foreground model compontens inversion once
XTX = np.linalg.inv(
np.dot(foregroundComponents.T, foregroundComponents))
XT = foregroundComponents.T
# Loop over all simulated spectra and perform the grid search on each
tic = time.time()
for i in range(set.shape[1]):
#fits, rmss = models.searchSignalTrials(set[:,i], foregroundComponents,
# freqs, signalFunction, params_sig, dataCov=noiseCov);
#fits, rmss = models.searchSignalRealizations(set[:,i], foregroundComponents,
# freqs, realizations_sig, dataCov=noiseCov);
fits, rmss = models.searchSignalRealizationsFast(
set[:, i], XTX, XT, realizations_sig)
index = np.argmin(rmss)
gs_recovParams[:, i] = fits[:, index]
gs_recovSig[:, i] = params_sig[:, index]
gs_recovRMS[i] = rmss[index]
gs_recovResiduals[:, i] = set[:, i] - foregroundComponents.dot(
gs_recovParams[:, i]) - signalFunction(
freqs, gs_recovSig[:, i])
#gs_recovResiduals[:,i] = set[:,i] - foregroundComponents.dot(gs_recovParams[:,i]) - realizations_sig[:,i];
# Write row
csvWriter.writerow(
list(gs_recovParams[:, i]) + list(gs_recovSig[:, i]) +
[gs_recovRMS[i]] + list(gs_recovResiduals[:, i]))
# Print update to stdout
print(
"{0:8.2f} s ({1} of {2}): RMS {3:6.3} K -- {4:6.3f}, {5:6.3f}, {6:6.3f}, {7:6.3f}"
.format(time.time() - tic, i, set.shape[1], gs_recovRMS[i],
gs_recovSig[0, i], gs_recovSig[1, i],
gs_recovSig[2, i], gs_recovSig[3, i]))