def _alignTimes(tlist, teclist, polylist, residuallist, fs):
tmin = []
tmax = []
for i in range(len(tlist)):
tmin.append(tlist[i].min())
tmax.append(tlist[i].max())
tstart = max(pyGps.datetime2posix(tmin))
tend = min(pyGps.datetime2posix(tmax))
t = []
tec2 = []
poly2 = []
res2 = []
for i in range(len(teclist)):
tt, tec1 = correctSampling(tlist[i], teclist[i], fs=fs)
tt, poly1 = correctSampling(tlist[i], polylist[i], fs=fs)
tt, res1 = correctSampling(tlist[i], residuallist[i], fs=fs)
tt = np.array(tt)
idt = np.where((tt>=tstart) & (tt<=tend))[0]
t.append(tt[idt])
tec2.append(tec1[idt])
poly2.append(poly1[idt])
res2.append(res1[idt])
return t, tec2, poly2, res2
def correctSampling(t, y, fs=1):
ts = pyGps.datetime2posix(t)
td = np.diff(ts)
idt = np.where(td != fs)[0]
if idt.shape[0] > 0:
while True:
td = np.diff(ts)
idt = np.where(td != fs)[0]
if idt.shape[0] == 0:
break
ts = np.insert(ts, idt[0] + 1, ts[idt[0]] + fs)
y = np.insert(y, idt[0] + 1, np.NaN)
return ts, y
def createTimeArray(timelim):
ts = pyGps.datetime2posix(timelim)
t = range(int(ts[0]), int(ts[1])+1)
return np.array(t)
def convertCORS2HDF(decimate='_30', days=[232,233], polynom_order=10, sv=[2,5], hdffilename='test'):#,6,12,19,24,25]):
folder = '/media/smrak/Eclipse2017/Eclipse/cors/all/'
el_mask = 25
fs = int(decimate[1:])
corr_hours = 24
# Get time arry in posix time - 1s resolution
observatiom_time_limit = [datetime.datetime.strptime('2017 '+str(233)+' 14 30 0', '%Y %j %H %M %S'),
datetime.datetime.strptime('2017 '+str(233)+' 22 30 0', '%Y %j %H %M %S')]
time_array = ec.createTimeArray(observatiom_time_limit)
c = 1
# Get rxlist for the deay of eclipse
rxlist = ec.getRxList(folder+str(233)+'/', sufix='*_30.17o')
# Open HDF File to write
h5file = h5py.File('/media/smrak/Eclipse2017/Eclipse/hdf/'+hdffilename+'.h5', 'a')
try:
h5file.create_dataset('obstimes', data=time_array)
except:
pass
# Iterate through stations in th elist
rxindex = np.arange(0, len(rxlist))
for rxi in rxindex:
rx = rxlist[rxi]
h5file = h5py.File('/media/smrak/Eclipse2017/Eclipse/hdf/'+hdffilename+'.h5', 'a')
print ('------------ '+ rx + " " + str(rxi) + ' out of ' + str(len(rxlist)) + ' ------------------')
diff_tec = np.nan*np.zeros((len(time_array), len(sv)))
lat = np.nan*np.zeros((len(time_array), len(sv)))
lon = np.nan*np.zeros((len(time_array), len(sv)))
residuals = np.nan*np.zeros((len(time_array), len(sv)))
gr = h5file.create_group(rx)
# if c >=100:
# h5file.close()
# break
# Iterate through all satellites
for i in range(len(sv)):
teclist = []
tlist = []
polylist = []
residuallist = []
# Do the processing for 2 successive days
for day in days:
# Set day and time correction if receiver is from Minnesota
if day == 232 and rx[:2] == 'mn':
day = 231
corr_hours = 48
# Set file names and time limits
timelim = [datetime.datetime.strptime('2017 '+str(day)+' 15 0 0', '%Y %j %H %M %S'),
datetime.datetime.strptime('2017 '+str(day)+' 21 0 0', '%Y %j %H %M %S')]
hdffile = folder+str(day)+'/'+rx+str(day)+'0'+decimate+'.h5'
yamlfile = folder+str(day)+'/'+rx+str(day)+'0.yaml'
navfile = '/media/smrak/Eclipse2017/Eclipse/nav/jplm'+str(day)+'0.17n'
# Open the OBS file
try:
data = read_hdf(hdffile)
# Get time, TEC
try:
t, tec, lla = ec.returnTEC(data, sv=sv[i], navfile=navfile, yamlfile=yamlfile,
timelim=timelim, el_mask=el_mask, lla=True,
svbias=True, vertical=True)
ix, intervals = ec.getIntervals(tec, maxgap=16, maxjump=1)
p = np.nan*np.ones(tec.shape[0])
for lst in intervals:
p[lst[0]:lst[1]] = ec.polynom(tec[lst[0]:lst[1]], order=polynom_order)
p[0:10] = np.nan
p[-10:] = np.nan
z = tec - p
teclist.append(tec)
tlist.append(t)
polylist .append(p)
residuallist.append(z)
#Save parameters for the eclipse day
if day == 233:
ts = pyGps.datetime2posix(t)
idt = np.where(np.isin(time_array, ts))[0]
lat[idt,i] = lla[0]
lon[idt,i] = lla[1]
residuals[idt,i] = z
except Exception as e:
print ('Line 140: ',e)
except Exception as e:
print ('Line 142: ', e)
# Reset time to th same day, round on 24 hours
if len(tlist) == 2:
try:
tlist[0] = tlist[0] + datetime.timedelta(hours=corr_hours)
t, tec2, poly2, res2 = ec._alignTimes(tlist, teclist, polylist, residuallist, fs)
idt = np.where(np.isin(time_array, t[1]))[0]
diff_tec[idt,i] = tec2[1] - tec2[0]
except Exception as e:
# break
print ('Line 154: ', e)
else:
print ('One day is missing for the receiver: ', rx)
gr.create_dataset('lat', data=lat)
gr.create_dataset('lon', data=lon)
gr.create_dataset('res', data=residuals)
gr.create_dataset('dtec', data=diff_tec)
c+=1
h5file.close()
def convertCORS2HDF(decimate='_30',
days=[232, 233],
polynom_order=10,
sv=[2, 5],
hdffilename='test'): #,6,12,19,24,25]):
el_mask = 40
fs = int(decimate[1:])
corr_hours = 24
# Get time arry in posix time - 1s resolution
observatiom_time_limit = [
datetime.datetime.strptime('2017 ' + str(233) + ' 15 0 0',
'%Y %j %H %M %S'),
datetime.datetime.strptime('2017 ' + str(233) + ' 21 0 0',
'%Y %j %H %M %S')
]
time_array = ec.createTimeArray(observatiom_time_limit)
c = 1
# Get rxlist for the deay of eclipse
rxlist = ec.getRxList(ALLDIR / f'{233}/', sufix='*_30.17o')
# Open HDF File to write
with h5py.File(OBSOUT / f'{hdffilename}.h5', 'a') as f:
try:
f['/obstimes'] = time_array
except Exception:
pass
# Iterate through stations in th elist
rxindex = np.arange(0, len(rxlist))
for rxi in rxindex:
rx = rxlist[rxi]
print(
f'------------ {rx} {rxi} out of {len(rxlist)} ------------------')
diff_tec = np.nan * np.zeros((len(time_array), len(sv)))
lat = np.nan * np.zeros((len(time_array), len(sv)))
lon = np.nan * np.zeros((len(time_array), len(sv)))
residuals = np.nan * np.zeros((len(time_array), len(sv)))
# if c >=100:
# h5file.close()
# break
# Iterate through all satellites
for i in range(len(sv)):
teclist = []
tlist = []
polylist = []
residuallist = []
# Do the processing for 2 successive days
for day in days:
# Set day and time correction if receiver is from Minnesota
if day == 232 and rx[:2] == 'mn':
day = 231
corr_hours = 48
# Set file names and time limits
timelim = [
datetime.datetime.strptime('2017 ' + str(day) + ' 15 0 0',
'%Y %j %H %M %S'),
datetime.datetime.strptime('2017 ' + str(day) + ' 21 0 0',
'%Y %j %H %M %S')
]
hdffile = ALLDIR / f'{day}/{rx}{day}0{decimate}.h5'
yamlfile = ALLDIR / f'{day}/{rx}{day}0.yaml'
navfile = NAVDIR / f'{day}0.17n'
# Open the OBS file
try:
data = read_hdf(hdffile)
# Get time, TEC
try:
t, tec, lla = ec.returnTEC(data,
sv=sv[i],
navfile=navfile,
yamlfile=yamlfile,
timelim=timelim,
el_mask=el_mask,
lla=True,
svbias=True,
vertical=True)
ix, intervals = ec.getIntervals(tec,
maxgap=16,
maxjump=1)
p = np.nan * np.ones(tec.shape[0])
for lst in intervals:
p[lst[0]:lst[1]] = ec.polynom(tec[lst[0]:lst[1]],
order=polynom_order)
p[0:10] = np.nan
p[-10:] = np.nan
z = tec - p
teclist.append(tec)
tlist.append(t)
polylist.append(p)
residuallist.append(z)
#Save parameters for the eclipse day
if day == 233:
ts = pyGps.datetime2posix(t)
idt = np.where(np.isin(time_array, ts))[0]
lat[idt, i] = lla[0]
lon[idt, i] = lla[1]
residuals[idt, i] = z
except Exception as e:
print('Line 140: ', e)
except Exception as e:
print('Line 142: ', e)
# Reset time to th same day, round on 24 hours
if len(tlist) == 2:
try:
tlist[0] = tlist[0] + datetime.timedelta(hours=corr_hours)
t, tec2, poly2, res2 = ec._alignTimes(
tlist, teclist, polylist, residuallist, fs)
idt = np.where(np.isin(time_array, t[1]))[0]
diff_tec[idt, i] = tec2[1] - tec2[0]
except Exception as e:
# break
print('Line 154: ', e, file=stderr)
else:
print('One day is missing for the receiver: ', rx)
with h5py.File(OBSOUT / hdffilename + '.h5', 'a') as f:
f[f'{rx}/lat'] = lat
f[f'{rx}/lon'] = lon
f[f'{rx}/res'] = residuals
f[f'{rx}/dtec'] = diff_tec
c += 1
def saveTrajectories2HDF(day='184',
state='tn',
el_mask=30,
altitude=100,
hdffilename='trajectories_tn.h5',
loop=True):
hdffolder = '/home/smrak/sharedrive/cors/'
navfolder = '/home/smrak/sharedrive/cors/nav/'
ymlfolder = '/home/smrak/sharedrive/cors/'
#Find HDF5 filename
wlstr = '*' + day + '*.h5'
filestr = os.path.join(hdffolder + state + '/' + day + '/', wlstr)
flistHDF = glob.glob(filestr)
#Find NAV filename
wlstr = '*' + str(day) + '*.17n'
filestr = os.path.join(navfolder, wlstr)
flistNAV = glob.glob(filestr)
#Find YAML filenamE
wlstr = '*' + str(day) + '*.yaml'
filestr = os.path.join(ymlfolder + state + '/' + day + '/', wlstr)
flistYML = glob.glob(filestr)
h5file = h5py.File(hdffilename, 'w')
gr = h5file.create_group(day)
h5state = gr.create_group(state)
if loop:
for i in range(len(flistHDF)):
hdffn = flistHDF[i]
navfn = flistNAV[0]
ymlfn = flistYML[i]
# YAML import header
stream = yaml.load(open(ymlfn, 'r'))
rx_xyz = stream.get('APPROX POSITION XYZ')
rx_llt = ecef2geodetic(rx_xyz[0], rx_xyz[1], rx_xyz[2])
#Timelim
timelim = [
datetime.datetime(2017, 7, 3, 15, 30, 0),
datetime.datetime(2017, 7, 3, 19, 30, 0)
]
h5rx = h5state.create_group(hdffn[-11:-7])
h5rx.create_dataset('rxpos', data=rx_llt)
# HDF Observations
try:
data = read_hdf(hdffn)
obstimes = np.array((data.major_axis))
obstimes = pandas.to_datetime(
obstimes[::10]) - datetime.timedelta(seconds=18)
idt = np.where((obstimes > timelim[0])
& (obstimes < timelim[1]))
obstimes = obstimes[idt]
#
h5rx.create_dataset('time',
data=pyGps.datetime2posix(obstimes))
#
dumb = data['L1', :, 1, 'data']
svlist = dumb.axes[0]
for sv in svlist:
# GPS only svnum <=32
az = np.nan * np.zeros(obstimes.shape[0])
el = np.nan * np.zeros(obstimes.shape[0])
lat = np.nan * np.zeros(obstimes.shape[0])
lon = np.nan * np.zeros(obstimes.shape[0])
if sv <= 32:
h5sv = h5rx.create_group('sv' + str(sv))
aer = pyGps.getIonosphericPiercingPoints(rx_xyz,
sv,
obstimes,
altitude,
navfn,
cs='aer')
llt = pyGps.getIonosphericPiercingPoints(rx_xyz,
sv,
obstimes,
altitude,
navfn,
cs='wsg84')
idel = np.where(aer[1] > el_mask)[0]
az[idel] = aer[0][idel]
el[idel] = aer[1][idel]
lat[idel] = llt[0][idel]
lon[idel] = llt[1][idel]
h5sv.create_dataset('az', data=az)
h5sv.create_dataset('el', data=el)
h5sv.create_dataset('lat', data=lat)
h5sv.create_dataset('lon', data=lon)
print('Saving data for sat: ' + str(sv))
else:
break
except:
raise ValueError
h5file.close()
def plotTrajectories(day='184', state='tn', timelim=None):
# Read in Totality path
data = h5py.File('/home/smrak/Documents/eclipse/totality.h5', 'r')
center_lat = np.array(data['path/center_lat'])
center_lon = np.array(data['path/center_lon'])
north_lat = np.array(data['path/north_lat'])
north_lon = np.array(data['path/north_lon'])
south_lat = np.array(data['path/south_lat'])
south_lon = np.array(data['path/south_lon'])
if timelim is not None:
timelim = pyGps.datetime2posix(timelim)
if isinstance(state, str):
file = state + 'list.h5'
data = h5py.File(file, 'r')
path = data[day + '/' + state + '/']
c = 0
for k in path.keys():
time = np.array(data[day + '/' + state + '/' + '/' + k + '/time'])
svlist = np.array(data[day + '/' + state + '/' + '/' + k])
rxpos = np.array(data[day + '/' + state + '/' + '/' + k +
'/rxpos'])
lat = []
lon = []
for sv in svlist:
if (sv != 'time') and (sv != 'rxpos') and (int(sv[2:]) < 33):
if timelim is None:
lat.append(
np.array(data[day + '/' + state + '/' + k + '/' +
sv + '/lat']))
lon.append(
np.array(data[day + '/' + state + '/' + k + '/' +
sv + '/lon']))
elif (timelim is not None) and isinstance(timelim, list):
idt = np.where((time >= timelim[0])
& (time <= timelim[1]))[0]
tmp = np.array(data[day + '/' + state + '/' + k + '/' +
sv + '/lat'])
lat.append(tmp[idt])
tmp = np.array(data[day + '/' + state + '/' + k + '/' +
sv + '/lon'])
lon.append(tmp[idt])
if c == 0:
ax, m = pyGpsUtils.plotGpsMapTrajectory(
lat=lat,
lon=lon,
rx=np.array([rxpos[0], rxpos[1]
]), #totalityc = [center_lat,center_lon],
totalityu=[north_lat, north_lon],
totalityd=[south_lat, south_lon],
labels=None,
ms=30,
timelim=None,
latlim=[30, 42],
parallels=[30, 35, 40],
meridians=[-100, -85, -70],
lonlim=[-100, -70],
center=[40, -80])
c += 1
else:
ax, m = pyGpsUtils.plotGpsMapTrajectory(
lat=lat,
lon=lon,
rx=np.array([rxpos[0], rxpos[1]
]), #totalityc = [center_lat,center_lon],
#totalityu = [north_lat,north_lon], totalityd = [south_lat,south_lon],
labels=None,
ms=30,
timelim=None,
latlim=[30, 42],
parallels=[30, 35, 40],
meridians=[-100, -85, -70],
lonlim=[-100, -70],
center=[40, -80],
ax=ax,
m=m)
elif isinstance(state, list):
c = 0
for st in state:
file = st + 'list.h5'
data = h5py.File(file, 'r')
path = data[day + '/' + st + '/']
for k in path.keys():
time = np.array(data[day + '/' + st + '/' + '/' + k + '/time'])
svlist = np.array(data[day + '/' + st + '/' + '/' + k])
rxpos = np.array(data[day + '/' + st + '/' + '/' + k +
'/rxpos'])
lat = []
lon = []
for sv in svlist:
if (sv != 'time') and (sv != 'rxpos') and (int(sv[2:]) <
33):
if timelim is None:
lat.append(
np.array(data[day + '/' + st + '/' + k + '/' +
sv + '/lat']))
lon.append(
np.array(data[day + '/' + st + '/' + k + '/' +
sv + '/lon']))
elif (timelim is not None) and isinstance(
timelim, list):
idt = np.where((time >= timelim[0])
& (time <= timelim[1]))[0]
tmp = np.array(data[day + '/' + st + '/' + k +
'/' + sv + '/lat'])
lat.append(tmp[idt])
tmp = np.array(data[day + '/' + st + '/' + k +
'/' + sv + '/lon'])
lon.append(tmp[idt])
if c == 0:
ax, m = pyGpsUtils.plotGpsMapTrajectory(
lat=lat,
lon=
lon, #rx=np.array([rxpos[0],rxpos[1]]), #totalityc = [center_lat,center_lon],
totalityu=[north_lat, north_lon],
totalityd=[south_lat, south_lon],
labels=None,
ms=10,
timelim=None,
latlim=[30, 42],
parallels=[30, 35, 40],
meridians=[-100, -85, -70],
lonlim=[-100, -70],
center=[40, -80])
c += 1
else:
ax, m = pyGpsUtils.plotGpsMapTrajectory(
lat=lat,
lon=
lon, #rx=np.array([rxpos[0],rxpos[1]]), #totalityc = [center_lat,center_lon],
#totalityu = [north_lat,north_lon], totalityd = [south_lat,south_lon],
labels=None,
ms=10,
timelim=None,
latlim=[30, 42],
parallels=[30, 35, 40],
meridians=[-100, -85, -70],
lonlim=[-100, -70],
center=[40, -80],
ax=ax,
m=m)
else:
print(
'Something went wrong. Chack that state and hdffilenamelist are of the same type and length'
)
data.close()