def scan_for_chirps(conf, dt=0.1):
"""
go through data files and look for unique soundings
"""
data_dir = conf.output_dir
# detection files have names chirp*.h5
if conf.realtime:
this_day_dname = "%s/%s" % (conf.output_dir,
cd.unix2dirname(time.time()))
# today
fl = glob.glob("%s/chirp*.h5" % (this_day_dname))
fl.sort()
# latest 100 detections
if len(fl) > 500:
fl = fl[(len(fl) - 500):len(fl)]
if len(fl) == 0:
print("no chirp detections yet")
return
else:
# look for all in batch mode
fl = glob.glob("%s/2*/chirp*.h5" % (data_dir))
fl.sort()
chirp_rates = []
f0 = []
chirp_times = []
snrs = []
for f in fl:
try:
h = h5py.File(f, "r")
chirp_times.append(n.copy(h[("chirp_time")]))
chirp_rates.append(n.copy(h[("chirp_rate")]))
f0.append(n.copy(h[("f0")]))
if "snr" in h.keys():
snrs.append(n.copy(h[("snr")]))
else:
snrs.append(-1.0)
h.close()
except:
print("Couldn't open %s" % (f))
chirp_times = n.array(chirp_times)
chirp_rates = n.array(chirp_rates)
f0 = n.array(f0)
snrs = n.array(snrs)
n_ionograms = 0
crs = n.unique(chirp_rates)
for c in crs:
idx = n.where(chirp_rates == c)[0]
t0s, num_dets = cluster_times(chirp_times[idx],
dt,
min_det=conf.min_detections)
for ti, t0 in enumerate(t0s):
if not conf.realtime:
print("Found chirp-rate %1.2f kHz/s t0=%1.4f num_det %d" %
(c / 1e3, t0, num_dets[ti]))
n_ionograms += 1
if conf.plot_timings:
plt.axhline(t0, color="red")
dname = "%s/%s" % (data_dir, cd.unix2dirname(n.floor(t0)))
if not os.path.exists(dname):
os.mkdir(dname)
fname = "%s/par-%1.4f.h5" % (dname, n.floor(t0))
if not os.path.exists(fname):
ho = h5py.File(fname, "w")
tnow = time.time()
t1 = (t0 + conf.maximum_analysis_frequency / c)
print(
"Found chirp-rate %1.2f kHz/s t0=%1.4f num_det %d started %1.2f s ago %1.2f s left"
% (c / 1e3, t0, num_dets[ti], tnow - t0, t1 - tnow))
print("writing file %s" % (fname))
ho["chirp_rate"] = c
ho["t0"] = t0
sweep_idx = n.where((n.abs(chirp_times - t0) < dt)
& (n.abs(chirp_rates - c) < 0.1))[0]
ho["f0"] = f0[sweep_idx]
ho["t0s"] = chirp_times[sweep_idx]
ho["snrs"] = snrs[sweep_idx]
ho.close()
if conf.plot_timings:
plt.plot(f0[idx] / 1e6, chirp_times[idx], ".")
if conf.plot_timings:
plt.xlabel("Frequency (MHz)")
plt.ylabel("Time (unix)")
plt.xlim([0, conf.maximum_analysis_frequency / 1e6])
plt.title("Chirp-rate %1.2f kHz/s" % (c / 1e3))
plt.show()
if not conf.realtime:
print("Found %d ionograms in total" % (n_ionograms))
def summary_plots(conf,t0):
fl=glob.glob("%s/%s/*.h5"%(conf.output_dir,cd.unix2dirname(t0)))
fl.sort()
n_ionograms=len(fl)
h=h5py.File(fl[0],"r")
freqs=n.copy(h["freqs"][()])
ranges=h["ranges"][()]
S=h["S"][()]
ft0=float(n.copy(h[("t0")]))
dt=(ft0-n.floor(ft0))
dr=dt*c.c/1e3
# converted to one-way travel time
range_gates=dr+ranges/1e3
SH = n.zeros([n_ionograms,S.shape[0]])
SV = n.zeros([n_ionograms,S.shape[1]])
dayno=n.floor(t0/(24.0*3600.0))
day_t0=dayno*24*3600.0
cid=int(n.copy(h[("id")])) # ionosonde id
img_fname_r="%s/%s/rstack-%03d-%1.2f.png"%(conf.output_dir,cd.unix2dirname(day_t0),cid,day_t0)
img_fname_f="%s/%s/fstack-%03d-%1.2f.png"%(conf.output_dir,cd.unix2dirname(day_t0),cid,day_t0)
hours=n.zeros(n_ionograms)
h.close()
# tbd time of day
for fi,f in enumerate(fl):
h=h5py.File(f,"r")
S=h["S"][()]
filet0=h["t0"][()]
for i in range(S.shape[0]):
noise=n.nanmedian(S[i,:])
S[i,:]=(S[i,:]-noise)/noise
S[S<=0.0]=1e-3
SH[fi,:]=n.max(S,axis=1)
SV[fi,:]=n.max(S,axis=0)
hours[fi]= (filet0-day_t0)/3600.0
h.close()
dBH=10.0*n.log10(SH.T)
# dBH=dBH-n.median(dBH)
fig=plt.figure(figsize=(1.5*8,1.5*6))
nfloor=n.median(dBH)
plt.pcolormesh(hours,freqs/1e6,dBH,vmin=nfloor-3,vmax=20+nfloor,cmap="inferno")
plt.colorbar()
plt.title("Range stack\n%s %s"%(conf.station_name,cd.unix2datestr(day_t0)))
plt.xlabel("Time (UTC hour of day)")
plt.ylabel("Frequency (MHz)")
plt.xlim([0,24])
plt.tight_layout()
plt.savefig(img_fname_r)
fig.clf()
plt.clf()
plt.close("all")
# if conf.copy_to_server:
# os.system("rsync -av %s %s/latest_rstack_%s.png"%(img_fname_r,conf.copy_destination,conf.station_name))
dBV=10.0*n.log10(SV.T)
nfloor=n.median(dBV)
fig=plt.figure(figsize=(1.5*8,1.5*6))
plt.pcolormesh(hours,range_gates,dBV,vmin=nfloor-3,vmax=20+nfloor,cmap="inferno")
plt.colorbar()
plt.title("Frequency stack\n%s %s"%(conf.station_name,cd.unix2datestr(day_t0)))
plt.xlabel("Time (UTC hour of day)")
plt.ylabel("One-way virtual range (km)")
plt.xlim([0,24])
plt.tight_layout()
plt.savefig(img_fname_f)
plt.clf()
fig.clf()
plt.close("all")
b = d.get_bounds(conf.channel)
n_windows = int(n.floor((b[1] - i0) / dt))
for wi in range(n_windows):
t0 = time.time()
S[:] = 0.0
for fi in range(nfft):
F = fft(wfun * d.read_vector_c81d(wi * dt + fi * fftlen + i0,
fftlen, conf.channel))
S += (F * n.conj(F)).real
t1 = time.time()
print(t1 - t0)
print("done")
try:
t0 = i0 / sr
dname = "%s/%s" % (conf.output_dir, cd.unix2dirname(t0))
if not os.path.exists(dname):
os.mkdir(dname)
ofname = "%s/spec-%s-%1.2f.h5" % (dname, conf.station_name, t0)
print("Writing to %s" % ofname)
ho = h5py.File(ofname, "w")
ho["spec"] = S
ho["nfft"] = nfft
ho["f0"] = conf.center_freq
ho["sr"] = conf.sample_rate
ho["dt"] = dt
ho["station_name"] = conf.station_name
ho["ch"] = conf.channel
ho.close()
except:
traceback.print_exc(file=sys.stdout)
def get_next_chirp_par_file(conf, d):
"""
wait until we encounter a parameter file with remaining time
"""
# find the next sounder that can be measured
while True:
ch = conf.channel
b = d.get_bounds(ch)
buffer_t0 = np.floor(np.float128(b[0]) / np.float128(conf.sample_rate))
while np.isnan(buffer_t0):
b = d.get_bounds(ch)
buffer_t0 = np.floor(
np.float128(b[0]) / np.float128(conf.sample_rate))
# t1=np.floor(np.float128(b[1])/np.float128(conf.sample_rate))
print("nan bounds for ringbuffer. trying again")
time.sleep(1)
# todo: look at today and yesterday. only looking
# at today will result in a few lost ionograms
# when the day is changing
dname = "%s/%s" % (conf.output_dir, cd.unix2dirname(time.time()))
fl = glob.glob("%s/par*.h5" % (dname))
fl.sort()
if len(fl) > 0:
for fi in range(len(fl)):
ftry = fl[len(fl) - fi - 1]
# proceed if this hasn't already been analyzed.
if not os.path.exists("%s.done" % (ftry)):
h = h5py.File(ftry, "r")
t0 = float(np.copy(h[("t0")]))
i0 = np.int64(t0 * conf.sample_rate)
chirp_rate = float(np.copy(h[("chirp_rate")]))
h.close()
t1 = conf.maximum_analysis_frequency / chirp_rate + t0
tnow = time.time()
# if the beginning of the buffer is before the end of the chirp,
# start analyzing as there is at least some of the the ionogram
# still in the buffer. the start of the buffer is
# before the the chirp ends
# t0 ---- t1
# bt0-------bt1
if buffer_t0 < t1:
# if not already analyzed, analyze it
if not os.path.exists("%s.done" % (ftry)):
ho = h5py.File("%s.done" % (ftry), "w")
ho["t_an"] = time.time()
ho.close()
print(
"Rank %d analyzing %s time left in sweep %1.2f s"
% (rank, ftry, t1 - tnow))
return (ftry)
else:
# we haven't analyzed this one, but we no longer
# can, because it is not in the buffer
print(
"Not able to analyze %s (%1.2f kHz/s), because it is no longer in the buffer. Buffer start at %1.2f and chirp ends at %1.2f"
% (ftry, chirp_rate / 1e3, buffer_t0, t1))
ho = h5py.File("%s.done" % (ftry), "w")
ho["t_an"] = time.time()
ho.close()
time.sleep(0.01)
# didn't find anything. let's wait.
time.sleep(1)
def chirp_downconvert(conf,
t0,
d,
i0,
ch,
rate,
dec=2500,
realtime_req=None,
cid=0):
cput0 = time.time()
sleep_time = 0.0
sr = conf.sample_rate
cf = conf.center_freq
dur = conf.maximum_analysis_frequency / rate
if realtime_req == None:
realtime_req = dur
idx = 0
step = 1000
n_windows = int(dur * sr / (step * dec)) + 1
cdc = cl.chirp_downconvert(f0=-cf,
rate=rate,
dec=dec,
dt=1.0 / conf.sample_rate,
n_threads=conf.n_downconversion_threads)
zd_len = n_windows * step
zd = np.zeros(zd_len, dtype=np.complex64)
z_out = np.zeros(step, dtype=np.complex64)
n_out = step
for fi in range(n_windows):
missing = False
try:
if conf.realtime:
b = d.get_bounds(ch)
while ((i0 + idx + step * dec + cdc.filter_len * dec) +
int(conf.sample_rate)) > b[1]:
# wait for more data to be acquired
# as the tail of the buffer doesn't have he data we
# need yet
time.sleep(1.0)
sleep_time += 1.0
b = d.get_bounds(ch)
z = d.read_vector_c81d(i0 + idx, step * dec + cdc.filter_len * dec,
ch)
except:
# z=np.zeros(step*dec+cdc.filter_len*dec,dtype=np.complex64)
missing = True
# we can skip this heavy step if there is missing data
if not missing:
cdc.consume(z, z_out, n_out)
else:
# step chirp time forward
cdc.advance_time(dec * step)
z_out[:] = 0.0
zd[(fi * step):(fi * step + step)] = z_out
idx += dec * step
dr = conf.range_resolution
df = conf.frequency_resolution
sr_dec = sr / dec
ds = get_m_per_Hz(rate)
fftlen = int(sr_dec * ds / dr / 2.0) * 2
fft_step = int((df / rate) * sr_dec)
S = spectrogram(np.conj(zd),
window=fftlen,
step=fft_step,
wf=ss.hann(fftlen))
freqs = rate * np.arange(S.shape[0]) * fft_step / sr_dec
range_gates = ds * np.fft.fftshift(np.fft.fftfreq(fftlen, d=1.0 / sr_dec))
if conf.manual_range_extent:
ridx = np.where((range_gates > conf.min_range)
& (range_gates < conf.max_range))[0]
else:
ridx = np.where(n.abs(range_gates) < conf.max_range_extent)[0]
fidx = n.arange(len(freqs), dtype=n.int)
if conf.manual_freq_extent:
fidx = n.where((freqs > conf.min_freq) & (freqs < conf.max_freq))[0]
try:
dname = "%s/%s" % (conf.output_dir, cd.unix2dirname(t0))
if not os.path.exists(dname):
os.mkdir(dname)
ofname = "%s/lfm_ionogram-%s-%03d-%1.2f.h5" % (
dname, conf.station_name, cid, t0)
print("Writing to %s" % ofname)
ho = h5py.File(ofname, "w")
S0 = n.array(S[:, ridx],
dtype=n.float16) # ionogram frequency-range, save space
ho["S"] = S0[fidx, :]
ho["freqs"] = freqs[fidx] # frequency bins
ho["rate"] = rate # chirp-rate
ho["ranges"] = range_gates[ridx]
ho["t0"] = t0
ho["id"] = cid
ho["station_name"] = conf.station_name
ho["sr"] = float(sr_dec) # ionogram sample-rate
if conf.save_raw_voltage:
ho["z"] = zd
ho["ch"] = ch # channel name
ho.close()
except:
traceback.print_exc(file=sys.stdout)
print("error writing file")
cput1 = time.time()
cpu_time = cput1 - cput0 - sleep_time
print("Done processed %1.2f s in %1.2f s, speed %1.2f * realtime" %
(realtime_req, cpu_time, realtime_req / cpu_time))
sys.stdout.flush()
def plot_ionogram(conf,f,normalize_by_frequency=True):
ho=h5py.File(f,"r")
t0=float(n.copy(ho[("t0")]))
if not "id" in ho.keys():
return
cid=int(n.copy(ho[("id")])) # ionosonde id
img_fname="%s/%s/lfm_ionogram-%03d-%1.2f.png"%(conf.output_dir,cd.unix2dirname(t0),cid,t0)
if os.path.exists(img_fname):
#print("Ionogram plot %s already exists. Skipping"%(img_fname))
ho.close()
return
print("Plotting %s rate %1.2f (kHz/s) t0 %1.5f (unix)"%(f,float(n.copy(ho[("rate")]))/1e3,float(n.copy(ho[("t0")]))))
S=n.copy(n.array(ho[("S")],dtype=n.float64)) # ionogram frequency-range
freqs=n.copy(ho[("freqs")]) # frequency bins
ranges=n.copy(ho[("ranges")]) # range gates
if normalize_by_frequency:
for i in range(S.shape[0]):
noise=n.nanmedian(S[i,:])
S[i,:]=(S[i,:]-noise)/noise
S[S<=0.0]=1e-3
max_range_idx=n.argmax(n.max(S,axis=0))
dB=n.transpose(10.0*n.log10(S))
if normalize_by_frequency == False:
dB=dB-n.nanmedian(dB)
dB[n.isnan(dB)]=0.0
dB[n.isfinite(dB)!=True]=0.0
# assume that t0 is at the start of a standard unix second
# therefore, the propagation time is anything added to a full second
dt=(t0-n.floor(t0))
dr=dt*c.c/1e3
# converted to one-way travel time
range_gates=dr+ranges/1e3
r0=range_gates[max_range_idx]
fig=plt.figure(figsize=(1.5*8,1.5*6))
plt.pcolormesh(freqs/1e6,range_gates,dB,vmin=-3,vmax=30.0,cmap="inferno")
cb=plt.colorbar()
cb.set_label("SNR (dB)")
plt.title("Chirp-rate %1.2f kHz/s t0=%1.5f (unix s)\n%s %s (UTC)"%(float(n.copy(ho[("rate")]))/1e3,float(n.copy(ho[("t0")])),conf.station_name,cd.unix2datestr(float(n.copy(ho[("t0")])))))
plt.xlabel("Frequency (MHz)")
plt.ylabel("One-way range offset (km)")
if conf.manual_range_extent:
plt.ylim([conf.min_range/1e3,conf.max_range/1e3])
else:
plt.ylim([dr-conf.max_range_extent/1e3,dr+conf.max_range_extent/1e3])
# plt.ylim([dr-1000.0,dr+1000.0])
if conf.manual_freq_extent:
plt.xlim([conf.min_freq/1e6,conf.max_freq/1e6])
else:
plt.xlim([0,conf.maximum_analysis_frequency/1e6])
plt.tight_layout()
plt.savefig(img_fname)
fig.clf()
plt.clf()
plt.close("all")
import gc
gc.collect()
ho.close()
sys.stdout.flush()
if conf.copy_to_server:
os.system("rsync -av %s %s/latest_%s.png"%(img_fname,conf.copy_destination,conf.station_name))