def write_care(f,year=2015,month=9,day=5,i0=0,ddir="."):
# dt=60.0*60.0
nt = len(launch_window)#24.0*3600.0/dt
satf = file("%s/sats.txt"%(ddir),"w")
t0 = stuffr.date2unix(year, month, day, 0, 0, 0)
for ti in range(int(nt)):
lt0 = launch_window[ti]*3600.0+t0
lt1 = launch_window[ti]*3600.0+3600.0+t0
sstr = "%03d CARE II %s %s"%(ti+i0,stuffr.unix2datestr(lt0),stuffr.unix2datestr(lt1))
print(sstr)
satf.write("%s\n"%(sstr))
f.write("%1.5f %1.5f 150.012000 400.032000 40.000 40.000 75.272 CAREII\n"%(lt0,lt1))
satf.close()
def check_lock(u, log=None, exit_if_not_locked=False):
locked = u.get_mboard_sensor("gps_locked").to_bool()
f = open("gps.log", "a")
f.write("%s lock=%d\n" % (stuffr.unix2datestr(time.time()), locked))
f.close()
if locked == False:
if log != None:
log.log("Warning, GPS not locked")
if exit_if_not_locked:
print("No GPS lock. Exiting.")
exit(0)
return (locked)
def range_doppler_matched_filter(
d,
out,
code_seeds=[1, 238, 681, 3099, 3263],
code_len=1000,
i0=158699892900000, # first index to analyze
i1=158699892900000, # last sample index to analyze
int_f=4, # range interpolation factor
# this is multiplied by two in the code
n_codes=4, # number of codes to integrate coherently together
sr=100e3,
ignore_freq=250.0, # don't include Doppler shifts
# smaller than this
r_min_an=400, # minimum range to analyze (km)
r_max_an=600, # maximum range to analyze (km)
f_min_an=-4e3, # minimum doppler to analyze (Hz)
f_max_an=1e3, # maximum doppler to analyze
noise_bw=10e3, # receiver noise bandwidth
codes_per_step=1, #
rfi_remove=False,
rx_channels=[0, 1]):
int_f = int_f * 2
codes = get_codes(int_f,
code_len=code_len,
seeds=code_seeds,
n_codes=n_codes)
dr = sc.c / sr / 1e3
c = d.get_channels()
b = d.get_bounds(c[0])
# index of ranges
ridx = n.arange(n_codes * code_len * int_f)
# frequency shifts
fvec = n.fft.fftshift(
n.fft.fftfreq(int_f * n_codes * code_len, d=1 / (int_f * sr)))
fidx = n.where(n.abs(fvec) < 1e3)[0]
# one-way range (distance from transmitter to receiver)
rvec = n.arange(r_max_an * int_f) * (dr / int_f)
# range of Doppler shifts to consider
gfidx = n.where((fvec > f_min_an) & (fvec < f_max_an))[0]
fvec0 = fvec[gfidx]
# do not detect these doppler frequencies
bfidx = n.where(n.abs(fvec0) < ignore_freq)[0]
# noise band
sfidx = n.where(n.abs(fvec0) < noise_bw)[0]
# range gates to analyze
gridx = n.where((rvec > r_min_an) & (rvec < r_max_an))[0]
rvec0 = rvec[gridx]
n_steps = int(n.floor((i1 - i0) / (code_len * codes_per_step)))
# figure out all correlations and cross-correlations
ch_pairs = list(
itertools.combinations(n.arange(len(code_seeds), dtype=n.int), 2))
for i in range(len(code_seeds)):
ch_pairs.append((i, i))
ch_pairs = n.array(ch_pairs)
n_pairs = ch_pairs.shape[0]
# go through all time steps
for si in range(rank, n_steps, size):
S = n.zeros([n_pairs, len(gridx), len(gfidx)], dtype=n.complex64)
read_idx = si * code_len * codes_per_step + i0
read_len = n_codes * code_len + code_len
# go through all rx channels
for chi in rx_channels:
z0 = d.read_vector_c81d(read_idx, read_len, c[chi])
if rfi_remove:
z0 = rfi_rem(z0)
# interpolate using a rectangular filter
# make sure we shift the signal back so that no range offset is created
z0 = n.repeat(z0, int_f)
wf = n.repeat(1.0 / float(int_f), int_f)
z0i = n.fft.ifft(n.fft.fft(z0) * n.fft.fft(wf, len(z0)))
z0i = n.roll(z0i, -int_f / 2)
# go through all transmit code pairs
for ci in range(n_pairs):
c0i = ch_pairs[ci, 0]
c1i = ch_pairs[ci, 1]
# go through all range gates
for rii, ri in enumerate(gridx):
Z0 = n.fft.fft(n.conj(codes[c0i]) * z0i[ri + ridx])
Z1 = n.fft.fft(n.conj(codes[c1i]) * z0i[ri + ridx])
S[ci, rii, :] += n.fft.fftshift(Z0 * n.conj(Z1))[gfidx]
# average over all XCs to obtain power estimate
S0 = n.sqrt(n.mean(n.abs(S)**2.0, axis=0))
noise_floor = n.median(S0)
# create signal_to_noise ratio estimate
snr = S0 / noise_floor
# create a copy
snr0 = n.copy(snr)
# because we are repeating the code, we are introducing some periodic
# features in the range-doppler matched filter output.
# gently try to filter these out using a 2d FFT
F = n.fft.fft2(snr0)
# FA=n.abs(F)
# MFA=ss.medfilt2d(FA,21)
# plt.pcolormesh(10.0*n.log10(FA-MFA))
# plt.colorbar()
# plt.show()
for fri in range(F.shape[0]):
med_amp = n.median(n.abs(F[fri, :]))
bad_f_idx = n.where(n.abs(F[fri, :]) > 10.0 * med_amp)[0]
F[fri, bad_f_idx] = 0.0
snr0 = n.abs(n.fft.ifft2(F))
plt.figure(figsize=(16, 9))
# n_floor=n.median(snr0)
# snr0=snr0-n_floor
# snr0=snr0/n.median(n.abs(snr0))
plt.pcolormesh(fvec0, rvec0, snr0, vmin=-2, vmax=20.0, cmap="inferno")
plt.title("%s\n" % (stuffr.unix2datestr(read_idx / sr)))
plt.xlim([f_min_an, f_max_an])
plt.ylim([r_min_an, r_max_an])
plt.xlabel("Doppler shift (Hz)")
plt.ylabel("Transmitter to receiver range (km)")
plt.colorbar()
plt.tight_layout()
plt.savefig("%s/rd-%06d.png" % (out, read_idx))
plt.clf()
plt.close()
snr0[:, n.min(bfidx):n.max(bfidx)] = 0.0
rp, fp = n.unravel_index(n.argmax(snr0), snr.shape)
f0 = fvec0[fp]
r0 = rvec0[rp]
peak_snr = n.max(snr0)
high_snr = n.max(snr0, axis=1)
low_snr = n.max(snr, axis=1)
h_r = n.argmax(high_snr)
l_r = n.argmax(low_snr)
h = h5py.File("%s/snr_%06d.h5" % (out, read_idx), "w")
h["XC"] = S
h["channel_pairs"] = ch_pairs
h["channel_code_seeds"] = code_seeds
h["high_snr"] = high_snr
h["low_snr"] = low_snr
h["high_r"] = h_r
h["low_r"] = l_r
h["dop_freq"] = fvec0
h["rvec"] = rvec0
h["f0"] = f0
h["r0"] = r0
h["t0"] = read_idx
h["sr"] = sr
h["snr0"] = peak_snr
h.close
print(
"segment %d/%d (%s) max_snr=%1.2f range=%1.2f (km) doppler=%1.2f (Hz)"
% (si, n_steps, stuffr.unix2datestr(
read_idx / sr), peak_snr, r0, f0))
if op.t1 < 0.0:
op.t1 = b0[1] / sample_rate - 2
# get baseline
try:
z0 = n.mean(
d.read_vector_c81d(long(op.baseline_time * sample_rate), op.integrate,
"000"))
z1 = n.mean(
d.read_vector_c81d(long(op.baseline_time * sample_rate), op.integrate,
"001"))
except:
print("Couldn't find data for determining baseline delay at %s" %
(stuffr.unix2datestr(op.baseline_time)))
exit(0)
# phase in 5 MHz to picoseconds ( (1/5e6)/ 1e-12)
reference_delay_ps = 200e3 * n.angle(z0 / z1) / 2.0 / n.pi
#print(""reference_delay_ps)
n_samples = long(n.floor((op.t1 - op.t0) * sample_rate))
# if overview plot, calculate delay sparsely over span of data
if op.latest:
idx0 = long(b1[1] - op.integrate)
idx1 = long(b1[1])
z0 = n.mean(d.read_vector_c81d(b1[1] - op.integrate, op.integrate, "000"))
z1 = n.mean(d.read_vector_c81d(b1[1] - op.integrate, op.integrate, "001"))
delay_ps = 200e3 * n.angle(z0 / z1) / 2.0 / n.pi - reference_delay_ps
print("t0 %1.3f t1 %1.3f delay %1.3f reference time %1.2f" %
def xc2img(fname="r1s/rd-1529980785.h5",
odir="./r1s/",
obs_lat=67.365524,
obs_lon=26.637495,
map_lat=68.0,
map_lon=26.0,
obs_h=500.0,
dB_thresh=5.0,
alias_num=1.0,
plot_map=True,
plot_latlongh=True,
plot_direction_cosines=True,
map_width=300e3,
map_height=300e3,
gc_len=5,
f_smooth=3,
height_color=True,
N=400,
t0=0.0):
h = h5py.File(fname, "r")
if h["t0"].value < t0:
h.close()
return
range_gates = h["range"].value + h["ipp_range"].value * alias_num
# antenna positions: range from array center
a_range = h["antenna_coords"].value[:, 0]
# antenna positions: azimuth relative to array center
a_az = h["antenna_coords"].value[:, 1]
# x and y positions in local horizontal plane with x east-west and y north-south
rx_y = -a_range * n.cos(n.pi * a_az / 180.0)
rx_x = -a_range * n.sin(n.pi * a_az / 180.0)
# antenna index pairs
aidx = n.copy(h["ant_pairs"].value)
# cross-correlation-range-doppler data cube
X = n.copy(h["X"].value)
if f_smooth > 1:
for ai in range(X.shape[0]):
for ri in range(X.shape[1]):
row = n.convolve(n.repeat(1.0 / f_smooth, f_smooth),
X[ai, ri, :],
mode="same")
X[ai, ri, :] = row
# calculate antenna position vectors
u = []
v = []
for i in range(aidx.shape[0]):
u.append(rx_x[aidx[i, 0]] - rx_x[aidx[i, 1]])
v.append(rx_y[aidx[i, 0]] - rx_y[aidx[i, 1]])
u = n.array(u)
v = n.array(v)
# snr
S = h["S"].value
# doppler velocities
vels = h["vels"].value
# snr in dB
dB = 10.0 * n.log10(S)
noise_floor = n.nanmedian(dB)
dB = dB - noise_floor
dB[0:gc_len, :] = 0.0
for ri in range(dB.shape[0]):
dB[ri, :] = dB[ri, :] - n.median(dB[ri, :])
# plt.pcolormesh(dB)
# plt.colorbar()
# plt.show()
# find pixels to image where snr is sufficiently high
gidx = n.where(dB.flatten() > dB_thresh)[0]
gidx = n.unravel_index(gidx, dB.shape)
# number of direction cosines to sample in the east and west direction
# N^2 directions in search space
a = aoa_find(N, u, v)
I = n.zeros([N, N], dtype=n.float32)
V = n.zeros([N, N], dtype=n.float32)
Nm = n.zeros([N, N])
dcx = []
dcy = []
pv = []
plats = []
plons = []
phgts = []
pp = []
for i in range(len(gidx[0])):
xi = gidx[0][i]
yi = gidx[1][i]
print(
"%s %d/%d dB %1.2f" %
(stuffr.unix2datestr(h["t0"].value), i, len(gidx[0]), dB[xi, yi]))
m_phase_diffs = n.exp(1j * n.angle(X[:, xi, yi]))
dcosx, dcosy, az, el, I0 = find_ds(m_phase_diffs, a)
r0 = range_gates[xi] * 1e3
# translate to lat long height
llh = jcoord.az_el_r2geodetic(obs_lat, obs_lon, 0.0, az, el, r0)
plats.append(llh[0])
plons.append(llh[1])
phgts.append(llh[2])
dcx.append(dcosx)
dcy.append(dcosy)
pv.append(h["vels"].value[yi])
pp.append(dB[xi, yi])
I += I0 * S[xi, yi]
V += I0 * h["vels"].value[yi]
Nm += I0
nidx = n.where(Nm > 0)
V[nidx] = V[nidx] / Nm[nidx]
I[nidx] = I[nidx] / Nm[nidx]
dcx = n.array(dcx)
dcy = n.array(dcy)
plons = n.array(plons)
plats = n.array(plats)
phgts = n.array(phgts)
pp = n.array(pp)
pv = n.array(pv)
ecefs = jcoord.geodetic2ecef(plats, plons, phgts)
ho = h5py.File("%s/points-%d.h5" % (odir, h["t0"].value), "w")
ho["ecef_m"] = ecefs
ho["lon_deg"] = plons
ho["lat_deg"] = plats
ho["height_km"] = phgts / 1e3
ho["SNR_dB"] = pp
ho["dop_vel"] = pv
ho["t0"] = h["t0"].value
ho.close()
if plot_map:
# setup Lambert Conformal basemap.False68.29911309985064, 23.3381106574698
m = Basemap(width=map_width,
height=map_height,
projection='lcc',
resolution='i',
lat_0=map_lat,
lon_0=map_lon)
# draw coastlines.
m.drawmapboundary(fill_color="black")
try:
m.drawcoastlines(color="white")
except:
print("no coastlines")
m.drawcountries(color="white")
parallels = n.arange(0., 81, 1.)
# labels = [left,right,top,bottom]
m.drawparallels(parallels,
labels=[True, False, False, False],
color="white")
meridians = n.arange(10., 351., 2.)
m.drawmeridians(meridians,
labels=[False, False, False, True],
color="white")
print("got here")
oidx = n.argsort(pp)
if len(pp) > 0:
xlat, ylon = m(plons, plats)
#
if height_color:
m.scatter(xlat[oidx],
ylon[oidx],
c=phgts[oidx] / 1e3,
s=1.0,
marker="o",
vmin=80,
vmax=105)
cb = plt.colorbar()
cb.set_label("Height (km)")
else:
m.scatter(xlat[oidx],
ylon[oidx],
c=pp[oidx],
s=1.0,
marker="o",
vmin=dB_thresh,
vmax=40.0)
cb = plt.colorbar()
cb.set_label("SNR (dB)")
plt.title(stuffr.unix2datestr(h["t0"].value))
plt.tight_layout()
plt.savefig("%s/map-%d.png" % (odir, h["t0"].value))
plt.close()
plt.clf()
if len(plats) > 0 and plot_latlongh:
ecefs = jcoord.geodetic2ecef(plats, plons, phgts)
plt.subplot(121)
plt.scatter(plats[oidx],
phgts[oidx] / 1e3,
c=pp[oidx],
vmin=0,
vmax=40.0)
plt.colorbar()
plt.xlabel("Latitude (deg)")
plt.ylabel("Height (km)")
plt.xlim([67.5, 68.2])
plt.ylim([80, 110.0])
plt.subplot(122)
plt.scatter(plons[oidx],
phgts[oidx] / 1e3,
c=pp[oidx],
vmin=0,
vmax=40.0)
plt.colorbar()
plt.xlim([24.5, 26.])
plt.ylim([80, 110.0])
plt.xlabel("Longitude (deg)")
plt.ylabel("Height (km)")
plt.tight_layout()
plt.savefig("%s/points-%d.png" % (odir, h["t0"].value))
plt.close()
if plot_direction_cosines:
oidx = n.argsort(pp)
fig = plt.figure()
# plt.style.use('dark_background')
phi = n.linspace(0, 2 * n.pi, num=500)
plt.scatter(-dcx[oidx],
-dcy[oidx],
c=pv[oidx],
vmin=-50,
vmax=50,
edgecolors="none",
cmap="Spectral")
plt.colorbar()
plt.plot(n.cos(phi), n.sin(phi), color="black")
plt.xlim([-1.1, 1.1])
plt.ylim([-1.1, 1.1])
plt.title(stuffr.unix2datestr(h["t0"].value))
plt.savefig("%s/vel-%d.png" % (odir, h["t0"].value))
plt.close()
fig = plt.figure()
# plt.style.use('dark_background')
plt.scatter(-dcx[oidx],
-dcy[oidx],
c=pp[oidx],
vmin=0,
vmax=40,
edgecolors="none")
plt.colorbar()
plt.plot(n.cos(phi), n.sin(phi), color="black")
plt.xlim([-1.1, 1.1])
plt.ylim([-1.1, 1.1])
plt.title(stuffr.unix2datestr(h["t0"].value))
plt.savefig("%s/img-%d.png" % (odir, h["t0"].value))
plt.close()
h.close()
def analyze_file(
fname="/media/j/4f5bab17-2890-4bb0-aaa8-ea42d65fdac8/mr_trails/raw.sodankyla.5fc9d2a8",
odir="/media/j/4f5bab17-2890-4bb0-aaa8-ea42d65fdac8/mr_trails/xc",
rem_ionosonde=True,
plot_coherence=False,
plot_snr=True,
plot_angles=False,
t_mess=0.25):
"""
t_mess = coherent integration length (seconds)
rem_ionosonde = perform outlier remove to reduce ionosonde related RFI
fname = the raw data file from a skymet radar
odir = the output directory.
"""
os.system("mkdir -p %s" % (odir))
r = read_ud3.UD3_Reader(fname, t_mess=t_mess)
n_int = 40
d = r.get_next_chunk()
if rem_ionosonde:
d = remove_ionosonde(d)
n_chan = d.shape[0]
n_range = d.shape[1]
n_time = d.shape[2]
wf = ss.hann(n_time)
freqs = n.fft.fftshift(
n.fft.fftfreq(d.shape[2], d=r.IPP * r.n_integrations))
vels = freqs * c.c / 2.0 / 36.9e6
ranges = r.deltaR * n.arange(d.shape[1]) + r.range_offset
ipp_range = r.IPP * c.c / 2.0 / 1e3
t0 = r.time
eff_ipp = r.PRF / r.n_integrations
dtau = (1.0 / eff_ipp) * d.shape[2]
n_pairs = int(d.shape[0] * (d.shape[0] - 1) / 2)
ant_pairs = list(itertools.combinations(n.arange(d.shape[0]), 2))
ant_pairs_m = n.array(ant_pairs)
antenna_coords = r.antenna_coords
r = read_ud3.UD3_Reader(fname, t_mess=t_mess)
n_files = int(n.floor(r.n_chunks / n_int))
for k in range(n_files):
print("%d/%d" % (k, n_files))
X = n.zeros([n_pairs, d.shape[1], d.shape[2]], dtype=n.complex64)
S = n.zeros([d.shape[1], d.shape[2]])
for ti in range(n_int):
d = r.get_next_chunk()
if rem_ionosonde:
d = remove_ionosonde(d)
for j in range(n_range):
for pi in range(n_pairs):
i0 = ant_pairs[pi][0]
i1 = ant_pairs[pi][1]
A = n.fft.fft(wf * d[i0, j, :])
B = n.fft.fft(wf * d[i1, j, :])
XS = n.fft.fftshift(A * n.conj(B))
X[pi, j, :] += XS
S[j, :] += n.abs(n.fft.fftshift(A * n.conj(B)))
X = X / float(n_int)
dB = 10.0 * n.log10(S)
vmed = n.nanmedian(dB)
dB = dB - vmed
tdata = dtau * k * n_int + t0
if plot_snr:
plt.pcolormesh(vels, ranges, dB, vmin=0, vmax=17, cmap="viridis")
plt.colorbar()
plt.xlabel("Doppler (m/s)")
plt.ylabel("Range (km)")
plt.title(stuffr.unix2datestr(tdata))
plt.xlim([n.min(vels), n.max(vels)])
plt.ylim([n.min(ranges), n.max(ranges)])
plt.title(stuffr.unix2datestr(tdata))
plt.savefig("%s/rd-%d.png" % (odir, tdata))
plt.close()
plt.clf()
for pi in range(n_pairs):
if plot_angles:
plt.pcolormesh(vels,
ranges,
n.angle(X[pi, :, :]),
cmap="jet",
vmin=-n.pi,
vmax=n.pi)
plt.title(ant_pairs[pi])
plt.colorbar()
plt.xlabel("Doppler (m/s)")
plt.ylabel("Range (km)")
plt.xlim(-200, 200)
plt.title("%s (%d,%d)" % (stuffr.unix2datestr(tdata),
ant_pairs[pi][0], ant_pairs[pi][1]))
plt.ylim([n.min(ranges), n.max(ranges)])
plt.savefig("%s/an-%d-%03d.png" % (odir, tdata, pi))
plt.close()
plt.clf()
if plot_coherence:
coh = n.abs(X[pi, :, :])
plt.pcolormesh(vels,
ranges,
coh,
cmap="viridis",
vmin=0,
vmax=1.0)
plt.colorbar()
plt.xlabel("Doppler (m/s)")
plt.ylabel("Range (km)")
plt.xlim(-250, 250)
plt.ylim([n.min(ranges), n.max(ranges)])
plt.title(ant_pairs[pi])
plt.savefig("%s/xc-%d-%03d.png" % (odir, tdata, pi))
plt.close()
plt.clf()
ho = h5py.File("%s/rd-%d.h5" % (odir, tdata), "w")
ho["t0"] = tdata
ho["X"] = X
ho["S"] = S
ho["ant_pairs"] = ant_pairs_m
ho["range"] = ranges
ho["ipp_range"] = ipp_range
ho["vels"] = vels
ho["antenna_coords"] = antenna_coords
ho.close()
def analyze_latest_sweep(s, data_path="/dev/shm"):
"""
Analyze an ionogram, make some plots, save some data
"""
# TODO: save raw voltage to file,
# then analyze raw voltage file with common program
# figure out what cycle is ready
n_rg = iono_config.n_range_gates #g#2000
t0 = n.uint64(
n.floor(time.time() / (s.sweep_len_s)) * s.sweep_len_s - s.sweep_len_s)
sfreqs = n.array(s.freqs)
iono_freqs = 0.5 * (sfreqs[:, 0] + sfreqs[:, 1])
fmax = n.max(iono_freqs)
n_plot_freqs = int((fmax + 0.5) / 0.1) + 1
iono_p_freq = n.linspace(0, fmax + 0.5, num=n_plot_freqs)
I = n.zeros([n_plot_freqs, n_rg], dtype=n.float32)
IS = n.zeros([sfreqs.shape[0], n_rg], dtype=n.float32)
n_t = int(s.freq_dur / 0.1)
all_spec = n.zeros([s.n_freqs, n_t, n_rg], dtype=n.float32)
dt = 10000.0 / 100e3
rvec = n.arange(float(n_rg)) * 1.5
fvec = n.fft.fftshift(n.fft.fftfreq(n_t, d=dt))
hdname = stuffr.unix2iso8601_dirname(t0)
dname = "%s/%s" % (iono_config.ionogram_path, hdname)
os.system("mkdir -p %s" % (dname))
z_all = n.zeros([s.n_freqs, int(s.freq_dur * 100000)], dtype=n.complex64)
noise_floors = []
for i in range(s.n_freqs):
fname = "%s/raw-%d-%03d.bin" % (data_path, t0, i)
if os.path.exists(fname):
z = n.fromfile(fname, dtype=n.complex64)
z_all[i, :] = z
N = len(z)
# print(len(z))
res = p.analyze_prc(z,
rfi_rem=False,
spec_rfi_rem=True,
dec=1,
code_type=iono_config.code_type,
Nranges=n_rg)
# print(res["res"].shape)
plt.subplot(121)
# print(dt)
tvec = n.arange(N / 10000.0) * dt
dBr = 10.0 * n.log10(n.transpose(n.abs(res["res"])**2.0))
noise_floor = n.nanmedian(dBr)
noise_floor_0 = noise_floor
noise_floors.append(noise_floor_0)
dBr = dBr - noise_floor
plt.pcolormesh(tvec, rvec, dBr, vmin=-3, vmax=30.0)
plt.xlabel("Time (s)")
plt.title("Range-Time Power f=%d (dB)\nnoise_floor=%1.2f (dB)" %
(i, noise_floor))
plt.ylabel("Range (km)")
plt.ylim([0, 500])
plt.colorbar()
plt.subplot(122)
S = n.abs(res["spec"])**2.0
#sw=n.fft.fft(n.repeat(1.0/4,4),S.shape[0])
#for rg_id in range(S.shape[1]):
# S[:,rg_id]=n.roll(n.real(n.fft.ifft(n.fft.fft(S[:,rg_id])*sw)),-2)
all_spec[i, :, :] = S
pif = int(iono_freqs[i] / 0.1)
I[pif, :] += n.max(S, axis=0)
IS[i, :] = n.max(S, axis=0)
# normalize by median std estimate
# I[i,:]=I[i,:]/n.median(n.abs(S-n.median(S)))
dBs = 10.0 * n.log10(n.transpose(S))
noise_floor = n.nanmedian(dBs)
dBs = dBs - noise_floor
plt.pcolormesh(fvec,
rvec - iono_config.range_shift * 1.5,
dBs,
vmin=-3,
vmax=30.0)
plt.ylim([0, 500])
plt.title("Range-Doppler Power (dB)\nnoise_floor=%1.2f (dB)" %
(noise_floor))
plt.xlabel("Frequency (Hz)")
plt.ylabel("Virtual range (km)")
plt.colorbar()
plt.tight_layout()
plt.savefig("%s/iono-%03d.png" % (dname, i))
plt.close()
plt.clf()
else:
return (0)
print("file %s not found" % (fname))
i_fvec = n.zeros(s.n_freqs)
for fi in range(s.n_freqs):
i_fvec[fi] = s.freq(fi)
dB = 10.0 * n.log10(n.transpose(I))
dB[n.isinf(dB)] = n.nan
noise_floor = n.nanmedian(dB)
for i in range(dB.shape[1]):
dB[:, i] = dB[:, i] - n.nanmedian(dB[:, i])
dB[n.isnan(dB)] = -3
noise_floor_0 = n.mean(n.array(noise_floors))
plt.figure(figsize=(1.5 * 8, 1.5 * 6))
plt.pcolormesh(n.concatenate((iono_p_freq, [fmax + 0.1])),
rvec - iono_config.range_shift * 1.5,
dB,
vmin=-3,
vmax=20.0)
plt.title(
"%s %s\nnoise_floor=%1.2f (dB)" %
(iono_config.instrument_name, stuffr.unix2datestr(t0), noise_floor_0))
plt.xlabel("Frequency (MHz)")
plt.ylabel("Virtual range (km)")
plt.colorbar()
plt.ylim([0, 800.0])
plt.xlim([n.min(iono_freqs) - 0.5, n.max(iono_freqs) + 0.5])
plt.tight_layout()
datestr = stuffr.unix2iso8601(t0)
ofname = "%s/%s.png" % (dname, datestr)
print("Saving ionogram %s" % (ofname))
plt.savefig(ofname)
plt.clf()
plt.close()
# make link to latest plot
os.system("ln -sf %s latest.png" % (ofname))
ofname = "%s/raw-%s.h5" % (dname, datestr)
if iono_config.save_raw_voltage:
save_raw_data(ofname,
t0,
z_all,
s.freqs,
iono_config.station_id,
sr=100000,
freq_dur=s.freq_dur,
codes=s.codes,
lat=iono_config.lat,
lon=iono_config.lon,
code_type=iono_config.code_type)
iono_ofname = "%s/ionogram-%s.h5" % (dname, datestr)
print("Saving ionogram %s" % (iono_ofname))
ho = h5py.File(iono_ofname, "w")
ho["I"] = IS
# ho["spec"]=all_spec
ho["I_rvec"] = rvec
ho["t0"] = t0
ho["lat"] = iono_config.lat
ho["lon"] = iono_config.lon
ho["I_fvec"] = sfreqs
ho["ionogram_version"] = 1
ho.close()
delete_old_files(t0)
def phase_channels(conf,d,i0,carrier_width=10.0,cphases=None,camps=None,use_cphases=False):
"""
simple calculation of phase differences between ch0 and other channels
as well as channel amplitude.
can be used to beamform all channels together.
"""
if use_cphases:
print("Using calibrated phases")
else:
cphases=n.zeros(len(conf.ch))
camps=n.ones(len(conf.ch))
fvec=n.fft.fftshift(n.fft.fftfreq(conf.nfft,d=1.0/conf.sample_rate))
fidx=n.where( (fvec>conf.fmin)&(fvec<conf.fmax))[0]
carrier_fidx=n.where( (fvec>-10.0)&(fvec<10.0))[0]
n_freq=len(fidx)
nfft=conf.nfft
step_len=conf.step_len*conf.sample_rate
t=n.arange(nfft,dtype=n.float32)/conf.sample_rate
wfun=s.chebwin(nfft,150)
n_chan=len(conf.ch)
ch_pairs=[]
n_pair=0
for i in range(1,n_chan):
ch_pairs.append((0,i))
n_pair+=1
S=n.zeros([n_pair,conf.nsteps,n_freq],dtype=n.complex64)
overlap=int(nfft/conf.overlap_fraction)
nmax_avg=int(n.floor((conf.step_len*conf.sample_rate-nfft-conf.offset)/overlap))
n_avg=1
tvec=n.zeros(conf.nsteps)
pwrs=n.zeros(len(conf.ch))
npwrs=n.zeros(len(conf.ch))
for step_idx in range(conf.nsteps):
fnow = conf.f0 + step_idx*conf.fstep
fshift = conf.center_freq-fnow
dshift=n.exp(1j*2.0*n.pi*fshift*t)
for pi,ci in enumerate(ch_pairs):
for avg_i in range(n_avg):
inow=i0+step_idx*step_len + avg_i*overlap
print("%s n_avg %d/%d f0 %1.2f"%(stuffr.unix2datestr(inow/conf.sample_rate),n_avg,nmax_avg,fnow/1e6))
tvec[step_idx]=step_idx*step_len/conf.sample_rate
z0=dshift*d.read_vector_c81d(inow,nfft,conf.ch[ci[0]])*camps[ci[0]]*n.exp(1j*cphases[ci[0]])
pwrs[ci[0]]+=n.mean(n.abs(z0)**2.0)
npwrs[ci[0]]+=1.0
z1=dshift*d.read_vector_c81d(inow,nfft,conf.ch[ci[1]])*camps[ci[1]]*n.exp(1j*cphases[ci[1]])
pwrs[ci[1]]+=n.mean(n.abs(z1)**2.0)
npwrs[ci[1]]+=1.0
X0=n.fft.fftshift(n.fft.fft(wfun*z0))
X1=n.fft.fftshift(n.fft.fft(wfun*z1))
S[pi,step_idx,:]+=X0[fidx]*n.conj(X1[fidx])
gfidx=n.where( n.abs((fvec[fidx])>carrier_width))[0]
phase_diffs=[]
for i in range(n_pair):
xc=n.copy(S[i,:,gfidx])
xc=xc.flatten()
xc_idx=n.argsort(xc)
phase_diff=n.angle(n.sum(xc[xc_idx[int(len(xc_idx)*0.3):int(len(xc_idx)*0.9)]]))
phase_diffs.append(phase_diff)
if use_cphases:
plt.pcolormesh(n.angle(S[i,:,:]))
plt.colorbar()
plt.show()
plt.plot(n.angle(n.sum(S[i,:,:],axis=0)))
plt.axhline(phase_diff)
plt.show()
ch_pwrs=pwrs/npwrs
phases=n.zeros(n_chan)
# simple phaseup with reference to channel 0
# ph0-ph1
# ph0-ph2
# ph0-ph3
# ph0-ph4
for i in range(n_pair):
phases[i+1]=phase_diffs[i]
# plt.plot(phases)
# plt.show()
# pha = 0
# pha - phb = mab
# pha - phc =
return(1.0/n.sqrt(ch_pwrs),phases)
def calculate_sweep(conf,d,i0,use_cphases=False,cphases=None,camps=None):
ofname="img/%s_sweep_%1.2f.h5"%(conf.prefix,i0/conf.sample_rate)
if os.path.exists(ofname) and conf.overwrite == False:
print("File %s already exists. Skipping."%(ofname))
if use_cphases:
print("Using calcibrated phases")
else:
cphases=n.zeros(len(conf.ch))
camps=n.ones(len(conf.ch))
fvec=n.fft.fftshift(n.fft.fftfreq(conf.nfft,d=1.0/conf.sample_rate))
fidx=n.where( (fvec>conf.fmin)&(fvec<conf.fmax))[0]
carrier_fidx=n.where( (fvec>-10.0)&(fvec<10.0))[0]
n_freq=len(fidx)
nfft=conf.nfft
step_len=conf.step_len*conf.sample_rate
t=n.arange(nfft,dtype=n.float32)/conf.sample_rate
wfun=s.chebwin(nfft,150)
S=n.zeros([conf.nsteps*conf.nsubsteps,n_freq])
N_avgd=n.zeros([conf.nsteps*conf.nsubsteps,n_freq])
overlap=int(nfft/conf.overlap_fraction)
nmax_avg=int(n.floor((conf.step_len*conf.sample_rate-conf.trim_end)/overlap/conf.nsubsteps))+1
sub_len=int(n.floor((step_len-conf.trim_end)/conf.nsubsteps))
if conf.fast:
n_avg=n.min([conf.n_avg,nmax_avg])
else:
n_avg=nmax_avg
tvec=n.zeros(conf.nsteps*conf.nsubsteps)
carrier = n.zeros(conf.nsteps*conf.nsubsteps,dtype=n.complex64)
f0s=n.zeros(conf.nsteps*conf.nsubsteps)
for step_idx in range(conf.nsteps):
fnow = conf.f0 + step_idx*conf.fstep
fshift = conf.center_freq-fnow
dshift=n.exp(1j*2.0*n.pi*fshift*t)
step_i0=i0+step_idx*step_len
for sub_idx in range(conf.nsubsteps):
f0s[step_idx*conf.nsubsteps+sub_idx]=fnow
tvec[step_idx*conf.nsubsteps+sub_idx]=(step_idx*step_len+sub_idx*sub_len)/conf.sample_rate
for avg_i in range(n_avg):
inow=i0 + step_idx*step_len + sub_idx*sub_len + avg_i*overlap
# make sure this fits the step
if (inow+nfft-step_i0+conf.trim_end) < step_len:
print("%s n_avg %d/%d f0 %1.2f"%(stuffr.unix2datestr(inow/conf.sample_rate),avg_i,nmax_avg,fnow/1e6))
z=n.zeros(nfft,dtype=n.complex128)
# beamform signals
for ch_i in range(len(conf.ch)):
try:
z+=dshift*d.read_vector_c81d(inow,nfft,conf.ch[ch_i])*camps[ch_i]*n.exp(1j*cphases[ch_i])
except:
print("missing data")
X=n.fft.fftshift(n.fft.fft(wfun*z))
if conf.debug:
plt.plot(fvec,10.0*n.log10(X))
plt.show()
S[step_idx*conf.nsubsteps+sub_idx,:]+=n.abs(X[fidx])**2.0
N_avgd[step_idx*conf.nsubsteps+sub_idx,:]+=1.0
carrier[step_idx*conf.nsubsteps+sub_idx]+=n.sum(n.abs(X[carrier_fidx])**2.0)
S=S/N_avgd
# ofname="img/%s_sweep_%1.2f.h5"%(conf.prefix,i0/conf.sample_rate)
print("Saving %s"%(ofname))
ho=h5py.File(ofname,"w")
ho["S"]=S
ho["phases"]=cphases
ho["amps"]=camps
ho["time_vec"]=tvec
ho["freq_vec"]=fvec[fidx]
ho["f0s"]=f0s
ho["carrier_pwr"]=carrier
ho["center_freq"]=conf.center_freq
ho["fscale"]=str(conf.fscale)
ho["t0"]=i0/conf.sample_rate
ho["date"]=stuffr.unix2datestr(i0/conf.sample_rate)
ho.close()
clean_image.plot_file(ofname,show_plot=conf.show_plot,vmin=conf.vmin,vmax=conf.vmax)
def rti_coh_incoh(
fname="/data3/SLICE_RAW/MRRAW_20171220/raw.sodankyla.5a39ae21",
t_mess=0.06,
alias_num=0.0,
at0=None,
at1=None,
ofname="out.h5"):
print("read")
r = read_ud3.UD3_Reader(fname, t_mess=t_mess)
print(r.n_chan)
print(r.antenna_coords)
d = r.get_next_chunk()
print(r.n_chunks)
n_chan = d.shape[0]
n_range = d.shape[1]
n_time = d.shape[2]
print(n_time)
freqs = n.fft.fftshift(
n.fft.fftfreq(d.shape[2], d=r.IPP * r.n_integrations))
vels = freqs * c.c / 2.0 / (r.frequency * 1e6)
ipp_range = alias_num * r.IPP * c.c / 2.0 / 1e3
ranges = ipp_range + r.deltaR * n.arange(d.shape[1]) + r.range_offset
t0 = r.time
eff_prf = r.PRF / r.n_integrations
print(eff_prf)
dtau = (1.0 / eff_prf) * d.shape[2]
r = read_ud3.UD3_Reader(fname, t_mess=t_mess)
print(d.shape)
n_c = r.n_chunks - 1
tvec_all = n.arange(n_c) * dtau + r.time
if t0 != None:
gidx = n.where((tvec_all > at0) & (tvec_all < at1))[0]
else:
gidx = n.arange(n_c)
n_tv = len(gidx)
RTI = n.zeros([n_tv, n_range], dtype=n.float32)
DOP = n.zeros([n_tv, n_range], dtype=n.float32)
tvec = []
oidx = 0
for k in range(n_c):
print("%d/%d" % (k, n_c))
S = n.zeros([d.shape[1], d.shape[2]])
d = r.get_next_chunk()
print(stuffr.unix2datestr(r.time + dtau * k))
# print(d)
# if d == None:
# break
if k in gidx:
print(d.shape)
tvec.append(r.time + dtau * k)
for i in range(n_chan):
if i == 0 and False:
plt.pcolormesh(n.abs(d[i, :, :]), vmin=0, vmax=512)
plt.title(stuffr.unix2datestr(r.time + dtau * k))
plt.colorbar()
plt.show()
for j in range(n_range):
z = d[i, j, :]
z = z - n.median(z)
S[j, :] += n.fft.fftshift(n.abs(n.fft.fft(z))**2.0)
for j in range(n_range):
RTI[oidx, j] = n.max(S[j, :])
DOP[oidx, j] = vels[n.argmax(S[j, :])]
oidx += 1
tvec = n.array(tvec)
ho = h5py.File(ofname, "w")
ho["tvec"] = tvec
ho["ranges"] = ranges
ho["RTI"] = RTI
ho["DOP"] = DOP
ho.close()
return (tvec, ranges, RTI, DOP)
def analyze_latest_sweep(s, data_path="/dev/shm"):
"""
Analyze an ionogram, make some plots, save some data
"""
# figure out what cycle is ready
t0 = n.uint64(
n.floor(time.time() / (s.sweep_len_s)) * s.sweep_len_s - s.sweep_len_s)
I = n.zeros([s.n_freqs, 1000], dtype=n.float32)
all_spec = n.zeros([s.n_freqs, 20, 1000], dtype=n.float32)
dt = 10000.0 / 100e3
rvec = n.arange(1000.0) * 1.5
fvec = n.fft.fftshift(n.fft.fftfreq(20, d=dt))
dname = "%s/%s" % (iono_config.ionogram_path, stuffr.sec2dirname(t0))
os.system("mkdir -p %s" % (dname))
for i in range(s.n_freqs):
fname = "%s/raw-%d-%03d.bin" % (data_path, t0, i)
if os.path.exists(fname):
z = n.fromfile(fname, dtype=n.complex64)
N = len(z)
# print(len(z))
res = p.analyze_prc(z,
rfi_rem=False,
spec_rfi_rem=False,
dec=1,
code_type=iono_config.code_type)
# print(res["res"].shape)
plt.subplot(121)
# print(dt)
tvec = n.arange(N / 10000.0) * dt
dBr = 10.0 * n.log10(n.transpose(n.abs(res["res"])**2.0))
noise_floor = n.nanmedian(dBr)
dBr = dBr - noise_floor
plt.pcolormesh(tvec, rvec, dBr, vmin=-3, vmax=n.max(dBr))
plt.xlabel("Time (s)")
plt.title("Range-Time Power f=%d (dB)\nnoise_floor=%1.2f (dB)" %
(i, noise_floor))
plt.ylabel("Range (km)")
plt.ylim([0, 500])
plt.colorbar()
plt.subplot(122)
S = n.abs(res["spec"])**2.0
all_spec[i, :, :] = S
I[i, :] = n.max(S, axis=0)
# normalize by median std estimate
# I[i,:]=I[i,:]/n.median(n.abs(S-n.median(S)))
dBs = 10.0 * n.log10(n.transpose(S))
noise_floor = n.nanmedian(dBs)
dBs = dBs - noise_floor
plt.pcolormesh(fvec, rvec, dBs, vmin=-3, vmax=n.max(dBs))
plt.ylim([0, 500])
plt.title("Range-Doppler Power (dB)\nnoise_floor=%1.2f (dB)" %
(noise_floor))
plt.xlabel("Frequency (Hz)")
plt.ylabel("Range (km)")
plt.colorbar()
plt.tight_layout()
plt.savefig("%s/iono-%d.png" % (dname, i))
plt.close()
plt.clf()
else:
return (0)
print("file %s not found" % (fname))
i_fvec = n.zeros(s.n_freqs)
for fi in range(s.n_freqs):
i_fvec[fi] = s.freq(fi)
dB = 10.0 * n.log10(n.transpose(I))
noise_floor = n.nanmedian(dB)
for i in range(dB.shape[1]):
dB[:, i] = dB[:, i] - n.nanmedian(dB[:, i])
plt.figure(figsize=(1.5 * 8, 1.5 * 6))
plt.pcolormesh(i_fvec / 1e6, rvec, dB, vmin=-3, vmax=100.0)
plt.title("Ionogram %s\nnoise_floor=%1.2f (dB)" %
(stuffr.unix2datestr(t0), noise_floor))
plt.xlabel("Frequency (MHz)")
plt.ylabel("Range (km)")
plt.colorbar()
plt.ylim([0, 800.0])
plt.tight_layout()
ofname = "%s/ionogram-%d.png" % (dname, t0)
print("Saving ionogram %s" % (ofname))
plt.savefig(ofname)
plt.clf()
plt.close()
os.system("ln -sf %s latest.png" % (ofname))
ho = h5py.File("%s/ionogram-%d.h5" % (dname, t0), "w")
ho["I"] = I
ho["i_freq_Hz"] = i_fvec
ho["freq_Hz"] = fvec
ho["range_km"] = rvec
ho["spectra"] = all_spec
ho["t0"] = t0
ho.close()
delete_old_files(t0)
if op.baseline_time < 0.0:
op.baseline_time = t_now - 5 * 60.0
if op.t0 < 0.0:
op.t0 = t_now - 5 * 60.0
if op.t1 < 0.0:
op.t1 = b0[1] / sample_rate - 2
# get baseline
try:
z0 = n.mean(d.read_vector_c81d(long(op.baseline_time * sample_rate), op.integrate, "000"))
z1 = n.mean(d.read_vector_c81d(long(op.baseline_time * sample_rate), op.integrate, "001"))
except:
print("Couldn't find data for determining baseline delay at %s" % (stuffr.unix2datestr(op.baseline_time)))
exit(0)
# phase in 5 MHz to picoseconds ( (1/5e6)/ 1e-12)
reference_delay_ps = 200e3 * n.angle(z0 / z1) / 2.0 / n.pi
# print(""reference_delay_ps)
n_samples = long(n.floor((op.t1 - op.t0) * sample_rate))
# if overview plot, calculate delay sparsely over span of data
if op.latest:
idx0 = long(b1[1] - op.integrate)
idx1 = long(b1[1])
z0 = n.mean(d.read_vector_c81d(b1[1] - op.integrate, op.integrate, "000"))
z1 = n.mean(d.read_vector_c81d(b1[1] - op.integrate, op.integrate, "001"))
delay_ps = 200e3 * n.angle(z0 / z1) / 2.0 / n.pi - reference_delay_ps
def decode_channel(
dir_name=[
"/data0/2019.10.24/test200e3_7.953e6",
"/data1/2019.10.24/test200e3_7.953e6"
],
out_dir_name="/data0/magrad_24.10",
idx0=None,
ch="cha",
n_ipp=500,
ipp=20000,
n_fft=100,
offset=160, # tx on offset
plot=True,
sr=200e3):
sr_factor = 1e6 / sr
d = drf.DigitalRFReader(dir_name)
b = d.get_bounds(ch)
md = d.read_metadata(b[0], b[0] + 1, ch)
for mk in md:
sr = md[mk]["sample_rate_numerator"]
if idx0 == None:
i0 = b[0]
else:
i0 = idx0
print(i0)
n_rg = ipp / n_fft
S = n.zeros([n_rg, n_fft])
WS = n.zeros([n_rg, n_fft])
z = d.read_vector_c81d(i0, n_ipp * ipp, ch)
wf = ss.hann(n_fft)
fvec = n.fft.fftshift(n.fft.fftfreq(n_fft, d=1 / float(sr))) / 1e3
rvec = n.arange(n_rg, dtype=n.float64) * n_fft * 3e8 / sr / 1e3 / 2.0
ipp_idx = 0
for i in range(n_ipp / 10):
S0 = n.zeros([n_rg, n_fft])
W0 = n.zeros([n_rg, n_fft])
for j in range(10):
for ri in range(n_rg):
zin = z[ipp_idx * ipp + ri * n_fft + n.arange(n_fft)]
S00 = n.abs(n.fft.fftshift(n.fft.fft(wf * zin)))**2.0
W0[ri, :] += S00**2.0
S0[ri, :] += S00
ipp_idx += 1
W0 = W0 - S0**2.0
S += S0 / W0
WS += 1.0 / W0
S = S / WS
for si in range(S.shape[1]):
S[:, si] = S[:, si] / n.median(S[:, si])
plt.figure(figsize=(15, 10))
dB = 10.0 * n.log10(S)
nfloor = n.nanmedian(dB)
plt.pcolormesh(fvec, rvec, dB, vmin=nfloor, vmax=nfloor + 20)
plt.colorbar()
plt.title("HF ISR spectrum\n%s" % (stuffr.unix2datestr(i0 / float(sr))))
plt.xlabel("Doppler (kHz)")
plt.ylabel("Range (km)")
plt.tight_layout()
plt.ylim([n.min(rvec), n.max(rvec)])
plt.xlim([n.min(fvec), n.max(fvec)])
plt.savefig("%s/isr-%d.png" % (out_dir_name, i0))
plt.clf()
plt.close()
ho = h5py.File("%s/isr-%d.h5" % (out_dir_name, i0), "w")
ho["i0"] = i0
ho["S"] = S
ho["rvec"] = rvec
ho["fvec"] = fvec
ho.close()
def plot_file(fname, show_plot=False, clean_plot=False, vmin=-6, vmax=50.0):
h = h5py.File(fname, "r+")
I = h["S"].value
fscale = h["fscale"].value
dB = 10.0 * n.log10(I)
dB = dB - n.nanmedian(10.0 * n.log10(I[I > 0]))
fig = plt.figure(figsize=(8 * 1.5, 6 * 1.5))
ax1 = fig.add_subplot(111)
#plt.title("%s"%(stuffr.unix2datestr(h["t0"].value)),y=1.06)
freq_p = n.copy(h["freq_vec"].value)
if fscale == "kHz":
freq_p = freq_p / 1e3
tvec = h["time_vec"].value
dBT = n.transpose(dB)
plt.title(
"%s $f_0=%1.2f$ (MHz)" %
(stuffr.unix2datestr(h["t0"].value), n.min(h["f0s"].value) / 1e6),
y=1.06)
cm = ax1.pcolormesh(tvec, freq_p, dBT, vmin=vmin, vmax=vmax, cmap="plasma")
ax1.set_xlabel("Time (s)")
ax1.set_ylabel("Frequency offset (%s)" % (fscale))
cb = fig.colorbar(cm, ax=ax1)
cb.set_label("dB")
ax1.set_ylim([n.min(freq_p), n.max(freq_p)])
ax1.set_xlim([n.min(tvec), n.max(tvec)])
if n.max(h["f0s"].value) - n.min(h["f0s"].value) > 10.0:
ax2 = ax1.twiny()
ax2.plot(h["f0s"].value / 1e6, n.ones(len(h["f0s"].value)), alpha=0)
ax2.set_xlabel("Heating frequency (MHz)")
ax2.set_ylim([n.min(freq_p), n.max(freq_p)])
ax2.set_xlim(
[n.min(h["f0s"].value / 1e6),
n.max(h["f0s"].value / 1e6)])
plt.tight_layout()
imgfname = "%s.png" % (fname)
plt.savefig(imgfname)
os.system("cp %s /tmp/0latest.png" % (imgfname))
if show_plot:
plt.show()
plt.clf()
plt.close()
if clean_plot:
dBc = n.copy(dB)
s0 = dBc.shape[0]
std_est = n.median(n.abs(dBc[0:(s0 - 1), :] - dBc[1:(s0), :]))
diff = n.abs(dBc[0:(s0 - 1), :] - dBc[1:(s0), :])
# plt.pcolormesh(diff)
# plt.show()
idx = n.where(diff > 20.0 * std_est)
# print(idx)
dBc[idx[0] + 1, idx[1]] = n.nan
dBc[idx[0], idx[1]] = n.nan
# plt.pcolormesh(n.transpose(dBc),vmin=-3,vmax=40,cmap="plasma")
# plt.show()
for i in range(dB.shape[1]):
print("%d/%d" % (i, dB.shape[1]))
m = dBc[:, i]
sigma = n.ones(len(m))
xhat = t2(m, sigma=sigma, alpha=0.1)
shit_idx = n.where(n.isnan(dBc[:, i]))[0]
dBc[shit_idx, i] = xhat[shit_idx]
dBc = dBc - n.nanmedian(dBc)
if "dB_clean" in h.keys():
del h["dB_clean"]
h["dB_clean"] = dBc
fig = plt.figure(figsize=(8 * 1.5, 6 * 1.5))
ax1 = fig.add_subplot(111)
plt.title(
"%s $f_0=%1.2f$ (MHz)" %
(stuffr.unix2datestr(h["t0"].value), n.min(h["f0s"].value) / 1e6),
y=1.06)
cm = ax1.pcolormesh(h["time_vec"].value,
freq_p,
n.transpose(dBc),
vmin=vmin,
vmax=vmax,
cmap="plasma")
ax1.set_xlabel("Time (s)")
ax1.set_ylabel("Frequency offset (%s)" % (fscale))
ax1.set_xlim([n.min(tvec), n.max(tvec)])
cb = fig.colorbar(cm, ax=ax1)
cb.set_label("dB")
ax1.set_ylim([n.min(freq_p), n.max(freq_p)])
print(h["f0s"].value)
if n.max(h["f0s"].value) - n.min(h["f0s"].value) > 10.0:
ax2 = ax1.twiny()
print(h["f0s"].value)
ax2.plot(h["f0s"].value / 1e6,
n.ones(len(h["f0s"].value)),
alpha=0)
ax2.set_xlabel("Heating frequency (MHz)")
ax2.set_ylim([n.min(freq_p), n.max(freq_p)])
ax2.set_xlim(
[n.min(h["f0s"].value / 1e6),
n.max(h["f0s"].value / 1e6)])
plt.tight_layout()
plt.savefig("%s.c.png" % (sys.argv[1]))
if show_plot:
plt.show()
plt.clf()
plt.close()
h.close()
def plot_slices(data_dir="/data1/noire/noire/oul/2022-05-02",
r0=850e3,
name="Skitbotn"):
fl = glob.glob("%s/lfm*.h5" % (data_dir))
fl.sort()
fof2 = []
hmf = []
tv = []
# plot this frequency
f0 = 5.5
# plot this range
S0 = n.zeros([len(fl), 650])
S1 = n.zeros([len(fl), 310])
h = h5py.File(fl[0], "r")
print(h.keys())
fr = n.copy(h["freqs"][()] / 1e6)
ra = n.copy(h["ranges"][()])
fidx = n.argmin(n.abs(f0 - fr))
ridx = n.argmin(n.abs(r0 - ra))
h.close()
fii = 0
for fi, f in enumerate(fl):
try:
h = h5py.File(f, "r")
print(h.keys())
S = normalize(h["S"][()])
S0[fii, :] = S[fidx, :] #/n.median(n.abs(S[fidx,:]))
S1[fii, :] = S[:, ridx] #/n.median(n.abs(S[:,ridx]))
tv.append(h["t0"][()])
fii += 1
h.close()
# print(f)
except:
h.close()
pass
S0 = S0[0:fii, :]
S1 = S1[0:fii, :]
S0[S0 < 0] = 1e-3
S1[S1 < 0] = 1e-3
tv = n.array(tv)
thour = (tv - tv[0]) / 3600.0
plt.pcolormesh(thour,
ra / 1e3,
10.0 * n.log10(n.transpose(S0)),
vmin=0,
vmax=20)
plt.title("%s %s\nFrequency=%1.2f MHz" %
(name, stuffr.unix2datestr(tv[0]), f0))
plt.xlabel("Time (hour)")
plt.ylabel("Virtual path length (km)")
plt.colorbar()
plt.tight_layout()
plt.savefig("range_%s.png" % (name))
plt.clf()
plt.close()
plt.pcolormesh(thour, fr, 10.0 * n.log10(n.transpose(S1)), vmin=0, vmax=20)
plt.title("%s %s\nPath length=%1.2f km" %
(name, stuffr.unix2datestr(tv[0]), r0 / 1e3))
plt.xlabel("Time (hour)")
plt.ylabel("Frequency (MHz)")
plt.colorbar()
plt.tight_layout()
plt.savefig("freq_%s.png" % (name))
plt.clf()
plt.close()
ho = h5py.File("%s.h5" % (name), "w")
ho["S0"] = S0
ho["S1"] = S1
ho["thour"] = thour
ho["t_unix"] = tv
ho["freq"] = fr
ho["ranges"] = ra / 1e3
ho.close()
def plot_cc(ch0="cha",
ch1="chc",
n_avg=1,
fft_len=1024 * 2,
dt=10,
rfi_rem=False,
plot_phase=True,
realtime=False,
flim=[0, 0],
now=False):
# fft_len=512*2
n_fft = 500 * 2
sr = 1e6
# cf=4.2e6
step_size = int(n.floor((dt * sr - fft_len) / n_fft))
#print(step_size)
d = drf.DigitalRFReader(dnames)
print(d.get_properties(ch0))
print(d.get_channels())
b = d.get_bounds(ch0)
print(b)
print((b[1] - b[0]) / 1e6)
S = n.zeros([fft_len, n_fft], dtype=n.complex64)
b = d.get_bounds(ch0)
cycle_num = int(n.floor(b[1] / sr / dt)) - 1
print("%s %s %d cycles since 1970" % (ch0, ch1, cycle_num))
if now:
i0 = (long(b[1] / (dt * sr)) - 1) * long(dt) * long(sr)
else:
i0 = int(cycle_num * dt * sr)
# wf=ss.hann(fft_len)
wf = ss.chebwin(fft_len, 100.0)
for si in range(n_fft):
print("%d/%d" % (si, n_fft))
for ai in range(n_avg):
z0 = d.read_vector_c81d(i0 + step_size * si + fft_len * ai,
fft_len, ch0)
z1 = d.read_vector_c81d(i0 + step_size * si + fft_len * ai,
fft_len, ch1)
if rfi_rem:
if ai == 0:
S[:, si] = n.fft.fftshift(
n.fft.fft(wf * z0) * n.conj(n.fft.fft(wf * z1)))
else:
p0 = n.abs(S[:, si])
p1 = n.fft.fftshift(
n.fft.fft(wf * z0) * n.conj(n.fft.fft(wf * z1)))
gidx = n.where(n.abs(p1) < p0)[0]
S[gidx, si] = p1[gidx]
else:
S[:, si] += n.fft.fftshift(
n.fft.fft(wf * z0) * n.conj(n.fft.fft(wf * z1)))
tv = n.arange(n_fft) * step_size / sr
fv = n.fft.fftshift(n.fft.fftfreq(fft_len, d=1 / sr)) + cf
# S2=n.copy(n.abs(S))
# for si in range(S.shape[1]):
# S2[:,si]=(S2[:,si]-n.median(S2[:,si]))/n.median(n.abs(S2[:,si]-n.median(S2[:,si])))
dB = 10.0 * n.log10(S)
for si in range(S.shape[1]):
dB[:, si] = dB[:, si] - n.nanmedian(dB[:, si])
plt.close()
plt.figure(figsize=(15, 10))
plt.clf()
plt.pcolormesh(fv / 1e6, tv, n.transpose(dB), vmin=-3, cmap="inferno")
plt.title("%s - %s\n%s-%s" % (stuffr.unix2datestr(
i0 / sr), stuffr.unix2datestr(i0 / sr + dt), ch0, ch1))
plt.xlabel("Frequency (MHz)")
plt.ylabel("Time (s)")
plt.colorbar()
if flim[0] != 0:
plt.xlim(flim)
else:
plt.xlim([(cf - sr / 2) / 1e6, (cf + sr / 2) / 1e6])
plt.tight_layout()
if now:
ofname = "img/see-%1.2f-%s-%s.png" % (time.time(), ch0, ch1)
plt.savefig(ofname)
os.system("cp %s img/0latest.png" % (ofname))
plt.pause(10)
else:
plt.savefig("img/xcp-%d-%s-%s.png" % (i0, ch0, ch1))
plt.clf()
plt.close()
if plot_phase:
plt.figure(figsize=(15, 10))
plt.pcolormesh(fv / 1e6, tv, n.transpose(n.angle(S)))
plt.title("%s - %s\n%s-%s" % (stuffr.unix2datestr(
i0 / sr), stuffr.unix2datestr(i0 / sr + dt), ch0, ch1))
plt.colorbar()
plt.xlabel("Frequency (MHz)")
plt.ylabel("Time (s)")
plt.xlim([(cf - sr / 2) / 1e6, (cf + sr / 2) / 1e6])
plt.tight_layout()
if now:
# plt.savefig("img/see-%1.2f.png"%(time.time()))
plt.pause(10.0)
# plt.savefig("img/see-%1.2f.png"%(time.time()))
else:
plt.savefig("img/xca-%d-%s-%s.png" % (i0, ch0, ch1))
plt.clf()
plt.close()
if __name__ == "__main__":
if len(sys.argv) == 2:
conf=sc.see_config(sys.argv[1])
else:
conf=sc.see_config()
print(conf)
d=drf.DigitalRFReader(conf.data_dirs)
print("Channels")
chs=d.get_channels()
print(chs)
print("Data extent:")
b=d.get_bounds(chs[0])
print("%s-%s"%(stuffr.unix2datestr(b[0]/conf.sample_rate),stuffr.unix2datestr(b[1]/conf.sample_rate)))
if conf.debug_timing:
debug_start(conf,d)
i0=conf.t0*conf.sample_rate + conf.offset
if conf.realtime:
tnow = b[1]/conf.sample_rate
cycle_num = int(n.floor((tnow-conf.t0)/conf.cycle_len)-1)
n_cycles=cycle_num+1
else:
cycle_num = conf.cycle_num
n_cycles = conf.n_cycles
b = d.get_bounds(op.channel)
if b[0] > idx:
idx = numpy.array(b[0])
while idx+op.anlen > b[1]:
d = drf.read_hdf5(op.datadir)
b = d.get_bounds(op.channel)
print("not enough data yet, sleeping.")
time.sleep(op.anlen/sr)
try:
res = None
res = analyze_prc(d,idx0=idx,an_len=op.anlen, clen=op.codelen, cache=True)
plt.clf()
M = 10.0*numpy.log10((numpy.abs(res["spec"])))
plt.pcolormesh(numpy.transpose(M),vmin=(numpy.median(M)-1.0))
plt.colorbar()
plt.title(stuffr.unix2datestr(idx/sr))
plt.savefig("%s/hfradar/spec-%06d.png"%(op.datadir,idx/sr))
print("%d"%(idx))
except Exception:
print("no data, skipping.")
idx = idx + op.anlen
idx.tofile("%s/hfradar/last.dat"%(op.datadir))
if idx > b[1]:
print "Done processing. Sleeping 60 seconds"
time.sleep(op.anlen/sr)
def calculate_sweep_xc(conf,d,i0,use_cphases=False,cphases=None,camps=None):
if use_cphases:
print("Using calcibrated phases")
else:
cphases=n.zeros(len(conf.ch))
camps=n.ones(len(conf.ch))
fvec=n.fft.fftshift(n.fft.fftfreq(conf.nfft,d=1.0/conf.sample_rate))
fidx=n.where( (fvec>conf.fmin)&(fvec<conf.fmax))[0]
carrier_fidx=n.where( (fvec>-10.0)&(fvec<10.0))[0]
n_freq=len(fidx)
nfft=conf.nfft
step_len=conf.step_len*conf.sample_rate
t=n.arange(nfft,dtype=n.float32)/conf.sample_rate
wfun=s.chebwin(nfft,150)
n_chan=len(conf.ch)
cpix=ito.combinations(n.arange(n_chan,dtype=n.int),2)
ch_pairs=[]
for i in range(n_chan):
ch_pairs.append((i,i))
for cpi in cpix:
ch_pairs.append((cpi[0],cpi[1]))
n_pairs=len(ch_pairs)
S=n.zeros([n_pairs,conf.nsteps,n_freq],dtype=n.complex64)
overlap=int(nfft/conf.overlap_fraction)
nmax_avg=int(n.floor((conf.step_len*conf.sample_rate-nfft-conf.offset)/overlap))+1
if conf.fast:
n_avg=n.min([conf.n_avg,nmax_avg])
else:
n_avg=nmax_avg
tvec=n.zeros(conf.nsteps)
carrier = n.zeros(conf.nsteps,dtype=n.complex64)
f0s=n.zeros(conf.nsteps)
for step_idx in range(conf.nsteps):
fnow = conf.f0 + step_idx*conf.fstep
f0s[step_idx]=fnow
fshift = conf.center_freq-fnow
dshift=n.exp(1j*2.0*n.pi*fshift*t)
tvec[step_idx]=step_idx*step_len/conf.sample_rate
for avg_i in range(n_avg):
inow=i0+step_idx*step_len + avg_i*overlap
print("%s n_avg %d/%d f0 %1.2f"%(stuffr.unix2datestr(inow/conf.sample_rate),n_avg,nmax_avg,fnow/1e6))
for pi,chp_i in enumerate(ch_pairs):
z0=dshift*d.read_vector_c81d(inow,nfft,conf.ch[chp_i[0]])*camps[chp_i[0]]*n.exp(1j*cphases[chp_i[0]])
z1=dshift*d.read_vector_c81d(inow,nfft,conf.ch[chp_i[1]])*camps[chp_i[1]]*n.exp(1j*cphases[chp_i[1]])
X0=n.fft.fftshift(n.fft.fft(wfun*z0))
X1=n.fft.fftshift(n.fft.fft(wfun*z1))
S[pi,step_idx,:]+=X0[fidx]*n.conj(X1[fidx])
if conf.fscale == "kHz":
fvec=fvec/1e3
ho=h5py.File("img/%s_sweep_xc_%1.2f.h5"%(conf.prefix,i0/conf.sample_rate),"w")
ho["S"]=S
ho["ch_pairs"]=ch_pairs
ho["phases"]=cphases
ho["amps"]=camps
ho["time_vec"]=tvec
ho["freq_vec"]=fvec[fidx]
ho["f0s"]=f0s
ho["center_freq"]=conf.center_freq
ho["t0"]=i0/conf.sample_rate
ho["date"]=stuffr.unix2datestr(i0/conf.sample_rate)
ho.close()
import digital_rf as drf
import matplotlib.pyplot as plt
import numpy as n
import stuffr
# real-time plot, just plot one integration period at last data in directory.
import sys
ch = "ch1"
if len(sys.argv) > 1:
print("using channel %s" % (sys.argv[1]))
ch = sys.argv[1]
d = drf.DigitalRFReader("/data0/test200e3_7.953e6")
print("Recorded channels")
print(d.get_channels())
prop = d.get_properties(ch)
b = d.get_bounds(ch)
sr = prop["samples_per_second"]
i0 = long(n.floor(b[1] / sr)) * sr
ipp = 100000 / 5
z = d.read_vector_c81d(i0, ipp, ch)
plt.plot(z.real)
plt.plot(z.imag)
plt.title("t=%s (UTC)" % (stuffr.unix2datestr(i0 / sr)))
plt.show()
# plt.pcolormesh(n.abs(C))
# plt.show()
for seed_idx in seeds:
r = prc_lib.analyze_prc(n.copy(z),
Nranges=600,
code=bpsk(seed_idx, code_len),
gc_rem=False,
rfi_rem=False,
dec=1,
station=seed_idx)
S += n.abs(r["spec"])**2.0
RTI += n.abs(r["res"])**2.0
plt.figure(figsize=(16, 10))
plt.subplot(211)
peak_snr = n.max(n.abs(r["spec"])**2.0)
plt.title("%s-%s" %
(stuffr.unix2datestr((si * code_len * N_per_seg + b[0]) / 100e3),
stuffr.unix2datestr(
((si + 1) * code_len * N_per_seg + b[0]) / 100e3)))
n_floor_r = n.median(RTI)
n_floor_s = n.median(S)
S = (S - n_floor_s) / n_floor_s
RTI = (RTI - n_floor_r) / n_floor_r
plt.pcolormesh(10.0 * n.log10(n.transpose(n.abs(RTI))), vmin=-3, vmax=20)
plt.colorbar()
plt.subplot(212)
plt.pcolormesh(n.transpose(S), vmin=-3, vmax=20)
plt.colorbar()
plt.show()
def analyze_latest_sweep(ic, data_path="/dev/shm"):
"""
Analyze an ionogram, make some plots, save some data
"""
# TODO: save raw voltage to file,
# then analyze raw voltage file with common program
# figure out what cycle is ready
s = ic.s
n_rg = ic.n_range_gates
t0 = n.uint64(
n.floor(time.time() / (s.sweep_len_s)) * ic.s.sweep_len_s -
ic.s.sweep_len_s)
sfreqs = n.array(ic.s.freqs)
iono_freqs = sfreqs[:, 0]
fmax = n.max(iono_freqs)
#
plot_df = 0.1
n_plot_freqs = int((fmax + 0.5) / plot_df)
iono_p_freq = n.arange(
n_plot_freqs) * plot_df # n.linspace(0,fmax+0.5,num=n_plot_freqs)
I = n.zeros([n_plot_freqs, n_rg], dtype=n.float32)
IS = n.zeros([sfreqs.shape[0], n_rg], dtype=n.float32)
# number of transmit "pulses"
n_t = int(ic.s.freq_dur * 1000000 / (ic.code_len * ic.dec))
all_spec = n.zeros([ic.s.n_freqs, n_t, n_rg], dtype=n.float32)
# IPP length
dt = ic.dec * ic.code_len / 1e6
# range step
dr = ic.dec * c.c / ic.sample_rate / 2.0 / 1e3
rvec = n.arange(float(n_rg)) * dr
p_rvec = n.arange(float(n_rg) + 1) * dr
fvec = n.fft.fftshift(n.fft.fftfreq(n_t, d=dt))
hdname = stuffr.unix2iso8601_dirname(t0, ic)
dname = "%s/%s" % (ic.ionogram_path, hdname)
os.system("mkdir -p %s" % (dname))
print("Duration of each frequency: {}".format(ic.s.freq_dur))
z_all = n.zeros([ic.s.n_freqs, int(ic.s.freq_dur * 100000)],
dtype=n.complex64)
noise_floors = []
for i in range(ic.s.n_freqs):
fname = "%s/raw-%d-%03d.bin" % (data_path, t0, i)
if os.path.exists(fname):
z = n.fromfile(fname, dtype=n.complex64)
z_all[i, :] = z
N = len(z)
code_idx = ic.s.code_idx(i)
res = p.analyze_prc2(z,
code=ic.orig_codes[code_idx],
cache_idx=code_idx,
rfi_rem=False,
spec_rfi_rem=True,
n_ranges=n_rg)
plt.figure(figsize=(1.5 * 8, 1.5 * 6))
plt.subplot(121)
tvec = n.arange(int(N / ic.code_len), dtype=n.float64) * dt
p_tvec = n.arange(int(N / ic.code_len) + 1, dtype=n.float64) * dt
with n.errstate(divide='ignore'):
dBr = 10.0 * n.log10(n.transpose(n.abs(res["res"])**2.0))
noise_floor = n.nanmedian(dBr)
noise_floor_0 = noise_floor
noise_floors.append(noise_floor_0)
dBr = dBr - noise_floor
dB_max = n.nanmax(dBr)
plt.pcolormesh(p_tvec,
p_rvec - ic.range_shift * dr,
dBr,
vmin=0,
vmax=ic.max_plot_dB)
plt.xlabel("Time (s)")
plt.title(
"Range-Time Power f=%d (dB)\nnoise_floor=%1.2f (dB) peak SNR=%1.2f"
% (i, noise_floor, dB_max))
plt.ylabel("Range (km)")
plt.ylim([-10, ic.max_plot_range])
plt.colorbar()
plt.subplot(122)
# S=n.abs(res["spec"])**2.0
S = res["spec_snr"]
#sw=n.fft.fft(n.repeat(1.0/4,4),S.shape[0])
#for rg_id in range(S.shape[1]):
# S[:,rg_id]=n.roll(n.real(n.fft.ifft(n.fft.fft(S[:,rg_id])*sw)),-2)
all_spec[i, :, :] = S
# 100 kHz steps for ionogram freqs
pif = n.argmin(n.abs(iono_freqs[i] - iono_p_freq))
# pif=int(iono_freqs[i]/0.1)
# collect peak SNR across all doppler frequencies
I[pif, :] += n.max(S, axis=0)
IS[i, :] = n.max(S, axis=0)
# SNR in dB scale
with n.errstate(divide='ignore'):
dBs = 10.0 * n.log10(n.transpose(S))
noise_floor = n.nanmedian(dBs)
max_dB = n.nanmax(dBs)
plt.pcolormesh(fvec,
rvec - ic.range_shift * dr,
dBs,
vmin=0,
vmax=ic.max_plot_dB)
plt.ylim([-10, ic.max_plot_range])
plt.title(
"Range-Doppler Power (dB)\nnoise_floor=%1.2f (dB) peak SNR=%1.2f (dB)"
% (noise_floor, max_dB))
plt.xlabel("Frequency (Hz)")
plt.ylabel("Virtual range (km)")
cb = plt.colorbar()
cb.set_label("SNR (dB)")
plt.tight_layout()
plt.savefig("%s/iono-%03d.png" % (dname, i))
plt.close()
plt.clf()
else:
return (0)
print("file %s not found" % (fname))
i_fvec = n.zeros(ic.s.n_freqs)
for fi in range(ic.s.n_freqs):
i_fvec[fi] = s.freq(fi)
with n.errstate(divide='ignore'):
dB = 10.0 * n.log10(n.transpose(I))
dB[n.isinf(dB)] = n.nan
noise_floor = n.nanmedian(dB)
for i in range(dB.shape[1]):
dB[:, i] = dB[:, i] - n.nanmedian(dB[:, i])
dB[n.isnan(dB)] = -3
noise_floor_0 = n.mean(n.array(noise_floors))
plt.figure(figsize=(1.5 * 8, 1.5 * 6))
max_dB = n.nanmax(dB)
plt.pcolormesh(n.concatenate((iono_p_freq, [fmax + 0.1])),
rvec - ic.range_shift * 1.5,
dB,
vmin=0,
vmax=ic.max_plot_dB)
plt.title(
"%s %s\nnoise_floor=%1.2f (dB) peak SNR=%1.2f" %
(ic.instrument_name, stuffr.unix2datestr(t0), noise_floor_0, max_dB))
plt.xlabel("Frequency (MHz)")
plt.ylabel("Virtual range (km)")
#plt.colorbar()
cb = plt.colorbar()
cb.set_label("SNR (dB)")
plt.ylim([-10, ic.max_plot_range])
plt.xlim([n.min(iono_freqs) - 0.5, n.max(iono_freqs) + 0.5])
plt.tight_layout()
datestr = stuffr.unix2iso8601(t0)
ofname = "%s/%s.png" % (dname, datestr)
print("Saving ionogram %s" % (ofname))
plt.savefig(ofname)
plt.clf()
plt.close()
# make link to latest plot
os.system("ln -sf %s latest.png" % (ofname))
ofname = "%s/raw-%s.h5" % (dname, datestr)
if ic.save_raw_voltage:
save_raw_data(ofname,
t0,
z_all,
ic.s.freqs,
ic.station_id,
sr=ic.sample_rate / ic.dec,
freq_dur=ic.s.freq_dur)
iono_ofname = "%s/ionogram-%s.h5" % (dname, datestr)
print("Saving ionogram %s" % (iono_ofname))
with h5py.File(iono_ofname, "w") as ho:
ho["I"] = IS
ho["I_rvec"] = rvec
ho["t0"] = t0
ho["lat"] = ic.lat
ho["lon"] = ic.lon
ho["I_fvec"] = sfreqs
ho["ionogram_version"] = 1
delete_old_files(t0)
def analyze_ionogram(
fname="/home/markus/j/ionosonde/results/2020-05-22T09:00:00Z/raw-2020-05-22T09:30:00.h5",
avg_spec=False,
plot_ionogram=False,
plot_spectra=False,
use_old=False,
max_range=1000,
min_range=0,
version=1):
h = h5py.File(fname, "r")
t0 = h["t0"].value
hdname = stuffr.unix2iso8601_dirname(h["t0"].value)
dname = "%s/%s" % (iono_config.ionogram_path, hdname)
os.system("mkdir -p %s" % (dname))
datestr = stuffr.unix2iso8601(t0)
iono_ofname = "%s/ionogram-%s.h5" % (dname, datestr)
print("looking for %s" % (iono_ofname))
if use_old:
if os.path.exists(iono_ofname):
hi = h5py.File(iono_ofname, "r")
I = n.copy(hi["I"].value)
r = n.copy(hi["I_rvec"].value)
f = n.copy(hi["I_fvec"].value)
hi.close()
return (I, r, f)
if not "version" in h.keys():
print("Not correction file version")
h.close()
return
if h["version"].value != version:
print("Not correct file version")
h.close()
return
# if use_old:
# if "I" in h.keys():
# I=n.copy(h["I"].value)
# I_fvec=n.copy(h["I_fvec"].value)
# I_rvec=n.copy(h["I_rvec"].value)
# h.close()
# return(I,I_rvec,I_fvec)
# float16 re and im to complex64
z_all = n.array(h["z_re"].value + h["z_im"].value * 1j, dtype=n.complex64)
freqs = h["freqs"].value
codes = h["codes"].value
code_type = h["code_type"].value
if "code_len" in h.keys():
code_len = h["code_len"].value
else:
code_len = 10000
# print(codes)
sample_rate = h["sample_rate"].value
dr = c.c / h["sample_rate"].value / 2.0 / 1e3
t0 = h["t0"].value
n_freqs = freqs.shape[0]
if "station_id" in h.keys():
station_id = h["station_id"].value
else:
station_id = 0
iono_freqs = 0.5 * (freqs[:, 0] + freqs[:, 1])
fmax = n.max(iono_freqs)
n_plot_freqs = int((fmax + 0.5) / 0.1) + 1
iono_p_freq = n.linspace(0, fmax + 0.5, num=n_plot_freqs)
I = n.zeros([n_plot_freqs, code_len], dtype=n.float32)
wf = create_waveform.create_prn_dft_code(clen=code_len, seed=station_id)
WF = n.fft.fft(wf)
rvec = n.arange(code_len) * dr
IS = n.zeros([n_freqs, code_len])
for i in range(n_freqs):
z = n.copy(z_all[i, :])
z = z - n.mean(z)
N_codes = len(z) / code_len
z.shape = (N_codes, code_len)
echoes = n.zeros([N_codes, code_len], dtype=n.complex64)
spec = n.zeros([N_codes, code_len], dtype=n.float)
for ci in range(N_codes):
echoes[ci, :] = n.fft.ifft(n.fft.fft(z[ci, :]) / WF)
# remove edge effect when hopping in frequency
echoes[N_codes - 1, :] = echoes[N_codes - 2, :]
for ri in range(code_len):
spec[:, ri] = n.fft.fftshift(n.abs(n.fft.fft(echoes[:, ri]))**2.0)
for fi in range(N_codes):
spec[fi, :] = spec[fi, :] / n.median(n.abs(spec[fi, :]))
if avg_spec:
sw = n.fft.fft(n.repeat(1.0 / 4, 4), N_codes)
for ri in range(code_len):
spec[:, ri] = n.roll(
n.real(n.fft.ifft(n.fft.fft(spec[:, ri]) * sw)), -2)
pif = int(iono_freqs[i] / 0.1)
I[pif, :] += n.max(spec, axis=0)
IS[i, :] = n.max(spec, axis=0)
if plot_spectra:
tv = n.arange(N_codes)
dBP = n.transpose(10.0 * n.log10(n.abs(echoes)**2.0))
nf = n.nanmedian(dBP)
plt.pcolormesh(tv, rvec, dBP, vmin=nf, vmax=nf + 20)
plt.ylim([0, 800])
plt.colorbar()
plt.show()
dBS = n.transpose(10.0 * n.log10(spec))
nf = n.nanmedian(dBS)
dop = 3e8 * n.fft.fftshift(
n.fft.fftfreq(N_codes, d=code_len /
float(sample_rate))) / 2.0 / (freqs[i, 0] * 1e6)
plt.pcolormesh(dop, rvec, dBS, vmin=nf, vmax=nf + 20)
plt.xlabel("Doppler shift (m/s)")
plt.ylabel("Range (km)")
plt.ylim([0, 800])
plt.colorbar()
plt.show()
if plot_ionogram:
dBI = n.transpose(10.0 * n.log10(I))
dBI[n.isinf(dBI)] = n.nan
noise_floor = n.nanmedian(dBI)
dBI = dBI - noise_floor
dBI[n.isnan(dBI)] = -3
plt.pcolormesh(n.concatenate((iono_p_freq, [fmax + 0.1])),
rvec,
dBI,
vmin=-3,
vmax=20.0)
plt.title("%s %s\nNoise floor=%1.2f (dB)" %
(iono_config.instrument_name,
stuffr.unix2datestr(h["t0"].value), noise_floor))
plt.xlim([n.min(iono_freqs) - 0.5, n.max(iono_freqs) + 0.5])
# plt.pcolormesh(freqs[:,0],rvec,dBI,vmin=0,vmax=20)
plt.ylim([0, 800])
plt.colorbar()
plt.xlabel("Frequency (MHz)")
plt.ylabel("Virtual range (km)")
plt.tight_layout()
ofname = "%s/%s.png" % (dname, datestr)
print("Saving ionogram %s" % (ofname))
plt.savefig(ofname)
plt.clf()
plt.close()
print("Saving ionogram %s" % (iono_ofname))
ho = h5py.File(iono_ofname, "w")
ho["I"] = IS
ho["I_rvec"] = rvec
ho["t0"] = h["t0"].value
ho["lat"] = h["lat"].value
ho["lon"] = h["lon"].value
ho["I_fvec"] = freqs
ho["ionogram_version"] = 1
ho.close()
h.close()
return (IS, rvec, freqs)
def test_envisat():
import dpt_tools as dpt
import space_object as so
import radar_library as rl
import stuffr
mass = 0.8111E+04
diam = 0.8960E+01
m_to_A = 128.651
a = 7159.5
e = 0.0001
i = 98.55
raan = 248.99
aop = 90.72
M = 47.37
A = mass / m_to_A
# epoch in unix seconds
ut0 = 1241136000.0
# modified julian day epoch
mjd0 = dpt.unix_to_jd(ut0) - 2400000.5
print(mjd0)
o = so.SpaceObject(a=a,
e=e,
i=i,
raan=raan,
aop=aop,
mu0=M,
C_D=2.3,
A=A,
m=mass,
diam=diam,
mjd0=mjd0)
e3d = rl.eiscat_3d()
print("EISCAT Skibotn location x,y,z ECEF (meters)")
print(e3d._tx[0].ecef)
ski_ecef = e3d._tx[0].ecef
print("EISCAT Skibotn location %1.3f %1.3f %1.3f (lat,lon,alt)" %
(e3d._tx[0].lat, e3d._tx[0].lon, 0.0))
t_obs = n.linspace(4440, 5280, num=100) + 31.974890
t_obs2 = n.linspace(4440, 5280, num=100) + 31.974890 + 1.0
# t_obs=n.linspace(0,5280,num=100)
# t_obs2=n.linspace(0,5280,num=100)+1
print(
"MJD %1.10f %sZ" %
(mjd0 + t_obs[0] / 3600.0 / 24.0, stuffr.unix2datestr(ut0 + t_obs[0])))
ecef = o.get_state(t_obs)
ecef2 = o.get_state(t_obs2)
print("ECEF state x,y,z,vx,vy,vz (km and km/s)")
print(ecef[:, 0] / 1e3)
print(
"Time (UTC) Range (km) Vel (km/s) ECEF X (km) ECEF Y (km) ECEF Z (km)"
)
for i in range(len(t_obs)):
dist = n.linalg.norm(ecef[0:3, i] - ski_ecef)
dist2 = n.linalg.norm(ecef2[0:3, i] - ski_ecef)
vel = (dist2 - dist) / 1.0
print("%s %1.3f %1.3f %1.3f %1.3f %1.3f" %
(stuffr.unix2datestr(ut0 + t_obs[i]), dist / 1e3, vel / 1e3,
ecef[0, i] / 1e3, ecef[1, i] / 1e3, ecef[2, i] / 1e3))
