nblocks = int(np.ceil(inf.N*inf.dt/winlen))

nbin = inf.N/float(nblocks)

# Compute maximum Fourier power for each block of time series data

maxpows = np.zeros(nblocks)

for iblock in xrange(nblocks):

istart = int(np.round(iblock*nbin))

iend = int(np.round((iblock+1)*nbin))

powers = np.abs(np.fft.rfft(data[istart:iend]))**2

maxpows[iblock] = np.max(powers[1:])

imax = np.argmax(powers[1:])

maxpows[-1] = np.median(maxpows[:-1])

# Scale the maximum powers

maxpows /= np.median(maxpows)

#maxpows /=np.median(powers)

nblocks = int(np.ceil(inf.N*inf.dt/winlen))

nbin = inf.N/float(nblocks)

# Compute maximum Fourier power for each block of time series data

minpows = np.zeros(nblocks)

medpows = np.zeros(nblocks)

variance = np.zeros(nblocks)

for iblock in xrange(nblocks):

istart = int(np.round(iblock*nbin))

iend = int(np.round((iblock+1)*nbin))

#powers = np.abs(np.fft.rfft(data[istart:iend]))**2

powers = np.abs(data[istart:iend])

minpows[iblock] = np.min(powers[1:])

medpows[iblock] = np.median(powers[1:])

variance[iblock] = np.var(powers[1:])

imin = np.argmin(powers[1:])

minpows[-1] = np.median(minpows[:-1])

minpows -= medpows

# Scale the minimum powers

minpows /= np.median(minpows)

variance /= np.median(variance)

print minpows[:25]

print variance[:25]

#return minpows, medpows

nblocks = len(maxpows)

nbin = len(data)/float(nblocks)

# Generate a mask based on block-wise Fourier power

# being larger than a threshold

mask = np.zeros(len(data), dtype=bool)

for iblock in xrange(len(maxpows)):

istart = int(np.round(iblock*nbin))

iend = int(np.round((iblock+1)*nbin))

if maxpows[iblock] > thresh:

mask[istart:iend] = 1

masked = np.ma.masked_array(data, mask=mask, fill_value=np.median(data))

nblocks = len(minpows)

nbin = len(data)/float(nblocks)

# Generate a mask based on block-wise Fourier power

# being larger than a threshold

mask = np.zeros(len(data), dtype=bool)

block_to_mask = []

for iblock in xrange(len(minpows)):

istart = int(np.round(iblock*nbin))

iend = int(np.round((iblock+1)*nbin))

if minpows[iblock] > thresh:

mask[istart:iend] = 1

block_to_mask.append(iblock)

masked = np.ma.masked_array(data, mask=mask, fill_value=np.median(data))

maskedfn = outbasenm+"_noradar.dat"

(masked.filled(fill_value=np.ma.median(masked))).tofile(maskedfn)

# Create soft link for inf file

inflink = maskedfn[:-4]+".inf"

if not os.path.exists(inflink):

# Invert the mask

inverted = invert_mask(masked)

slices = np.ma.flatnotmasked_contiguous(inverted)

if slices:

towrite = np.empty((len(slices), 2))

for ii, badslice in enumerate(slices):

towrite[ii] = (badslice.start-1000, badslice.stop+1000)

#radar_samples = np.flatnonzero(np.ma.getmaskarray(data))

with open(outbasenm+"_radar_samples.txt", 'w') as ff:

ff.write("# Samples containing the radar\n")

ff.write("# Intervals are samples to remove 'start:stop' (inclusive!)\n")

ff.write("# First sample number is 0\n")

ff.write("# Number of data samples to remove: %d of %d (%.2g %%)\n" %

(inverted.count(), len(inverted), 100.0*inverted.count()/len(inverted)))

if slices:

np.savetxt(ff, towrite, "%d", delimiter=':')

if inf.bary:

raise ValueError("Radar removal should only be performed " \

"using topocentric time series as input!")

if inf.DM != 0:

raise ValueError("Radar removal should only be performed " \

fig = plt.figure(figsize=(16,4))

times = np.arange(len(masked))*inf.dt

warnings.warn("Only plotting every 10th point of time series.")

plt.plot(times[::10], masked.data[::10], 'k-', drawstyle='steps-post',

label='Time series', zorder=1)

inverted = invert_mask(masked)

slices = np.ma.flatnotmasked_contiguous(inverted)

if slices:

for ii, badslice in enumerate(slices):

if ii == 0:

label='Radar indentified'

else:

label="_nolabel"

tstart = inf.dt*(badslice.start)

tstop = inf.dt*(badslice.stop-1)

plt.axvspan(tstart, tstop, alpha=0.5,

fc='r', ec='none', zorder=0, label=label)

plt.figtext(0.02, 0.02,

"Frac. of data masked: %.2f %%" % ((len(masked)-masked.count())/float(len(masked))*100),

size='x-small')

plt.figtext(0.02, 0.05, inf.basenm, size='x-small')

plt.xlabel("Time (s)")

plt.ylabel("Intensity")

plt.xlim(0, times.max()+inf.dt)

fig = plt.figure(figsize=(16,4))

# Make diagnostic plots

freqs = np.fft.fftfreq(int(inf.N), inf.dt)

nfft = int(inf.N/2)

freqs = freqs[:nfft]

plt.plot(freqs, np.abs(np.fft.rfft(masked.data)[:nfft])**2, 'r-')

plt.plot(freqs, np.abs(np.fft.rfft(masked.filled(fill_value=np.ma.median(masked)))[:nfft])**2, 'k-')

plt.xlabel("Freq (Hz)")

plt.ylabel("Raw Power")

plt.xscale('log')

plt.yscale('log')

plt.figtext(0.02, 0.02,

"Frac. of data masked: %.2f %%" % ((len(masked)-masked.count())/float(len(masked))*100),

size='x-small')

plt.figtext(0.02, 0.05, inf.basenm, size='x-small')

plt.xlim(0.066, 2000)

# Get input dat/inf file names

if not fn.endswith(".dat"):

raise ValueError("Input file must be a PRESTO '.dat' file!")

inffn = fn[:-4]+'.inf'

# Read inf/dat files

inf = infodata.infodata(inffn)

rawdata = np.fromfile(fn, dtype=DTYPE, count=inf.N)

#validate_timeseries(inf)

fn = args.infn

if args.outbasenm is None:

outbasenm = fn[:-4] # Trim off '.dat'

else:

outbasenm = args.outbasenm

rawdata, inf = read_datfile(fn)

maxpows = compute_maxpows(rawdata, inf, args.winlen)

data = apply_mask_to_maxpows(rawdata, maxpows, args.maxthresh)

minpows, medpows = compute_minpows(data, inf, args.winlen)

data, mask, blocks_to_mask = apply_mask_to_minpows(data, minpows, medpows, args.minthresh)

print "Frac. of data masked: %.2f %%" % ((len(data)-data.count())/float(len(data))*100),

# Write out masked file

#write_masked_dm0_timeseries(data, outbasenm)

# Write out list of masked samples

write_radar_intervals(data, outbasenm)

if args.plot_timeseries:

plot_timeseries_comparison(data, inf)

try:

plt.savefig(outbasenm+"_timeseries_compare.png")

except:

pass

if args.plot_powerspec:

plot_powerspec_comparison(data, inf)

try:

plt.savefig(outbasenm+"_powerspec_compare.png")

except: