def mergeSignals(csvFiles: list, logger: logging.Logger) -> pd.DataFrame:
"""
merge dataframes form bith signals into 1 dataframe
"""
# read both files into separate dataframes
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
dfObs = []
for i, csvFile in enumerate(csvFiles):
dfObs.append(pd.read_csv(csvFile, sep=',', parse_dates=True))
# get list of SVs for each signaltype
dCSV[i]['SVs'] = dfObs[i].columns.values[1:]
dCSV[i]['#SVs'] = len(dCSV[i]['SVs'])
# rename the columns in each dataframe to reflect PRN-ST as column name
colNames = dfObs[i].columns.values[1:] + '-{st:s}'.format(
st=dCSV[i]['signal'])
colNames = np.concatenate([['time'], colNames])
dfObs[i].columns = colNames
# show the dataframe
amutils.logHeadTailDataFrame(df=dfObs[i],
dfName=dCSV[i]['signal'],
callerName=cFuncName,
logger=logger)
dfObs[i]['time'] = pd.to_datetime(dfObs[i]['time'],
format='%Y-%m-%d %H:%M:%S')
# print('type of time = {type!s}'.format(type=type(dfObs[i]['time'][0])))
# sys.exit(0)
# get list of unique SVs
print(type(dCSV[1]['SVs']))
dCSV['SVs'] = np.intersect1d(dCSV[0]['SVs'], dCSV[1]['SVs'])
dCSV['#SVs'] = len(dCSV['SVs'])
logger.info('{func:s}: information:\n{dict!s}'.format(dict=dCSV,
func=cFuncName))
# merge both dataframes on 'time'
return pd.merge(dfObs[0], dfObs[1], on=['time'], how='outer')
def store_to_cvs(df: pd.DataFrame,
ext: str,
dInfo: dict,
logger: logging.Logger,
index: bool = True):
"""
store the dataframe to a CSV file
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
csv_name = amc.dRTK['info']['rtkPosFile'] + '.' + ext
dInfo[ext] = csv_name
df.to_csv(csv_name, index=index, header=True)
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=df,
dfName=csv_name)
logger.info('{func:s}: stored dataframe as csv file {csv:s}'.format(
csv=colored(csv_name, 'green'), func=cFuncName))
def enu_statistics(dRtk: dict, dfENU: pd.DataFrame, logger: logging.Logger) -> pd.DataFrame:
"""
enu_statistics calculates the statistics of the ENU coordinates passed
"""
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
dfENUStats = dfENU[['dUTM.E', 'dUTM.N', 'dEllH']].describe()
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfENUStats, dfName='dfENUStats')
# add statistics for UTM coordinate differences
dENUStats = {}
for col in ('dUTM.E', 'dUTM.N', 'dEllH'):
dCol = {}
for index, row in dfENUStats.iterrows():
dCol[index] = row[col]
dENUStats[col] = dCol
logger.debug('{func:s}: statistics for {col:s}\n{stat!s}'.format(col=col, stat=dENUStats[col], func=cFuncName))
# add to global dRTK dict
dRtk['stats'] = dENUStats
return dfENUStats
def parseSatelliteStatistics(statsSat: tempfile._TemporaryFileWrapper, logger: logging.Logger) -> pd.DataFrame:
"""
parseSatelliteStatistics reads the SAT statitics file into a dataframe
"""
# set current function name
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: Parsing RTKLib satellites file {file:s} ({info:s})'.format(func=cFuncName, file=statsSat.name, info=colored('be patient', 'red')))
# read in the satellite status file
dfSat = pd.read_csv(statsSat.name, header=None, sep=',', names=rtkc.dRTKPosStat['Res']['colNames'], usecols=rtkc.dRTKPosStat['Res']['useCols'])
# add DT column
dfSat['DT'] = dfSat.apply(lambda x: gpstime.UTCFromWT(x['WNC'], x['TOW']), axis=1)
# if PRres == 0.0 => than I suppose only 4 SVs used, so no residuals can be calculated, so change to NaN
dfSat.PRres.replace(0.0, np.nan, inplace=True)
amc.logDataframeInfo(df=dfSat, dfName='dfSat', callerName=cFuncName, logger=logger)
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfSat, dfName='dfSat')
return dfSat
def parseRTKLibPositionFile(logger: logging.Logger) -> pd.DataFrame:
"""
parse the position file from RTKLIB processing into a dataframe
"""
# set current function name
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: parsing RTKLib position file {posf:s}'.format(func=cFuncName, posf=amc.dRTK['info']['rtkPosFile']))
# check whether the datafile is readable
endHeaderLine = amutils.line_num_for_phrase_in_file('% GPST', amc.dRTK['info']['rtkPosFile'])
dfPos = pd.read_csv(amc.dRTK['info']['rtkPosFile'], header=endHeaderLine, delim_whitespace=True)
dfPos = dfPos.rename(columns={'%': 'WNC', 'GPST': 'TOW', 'latitude(deg)': 'lat', 'longitude(deg)': 'lon', 'height(m)': 'ellH', 'sdn(m)': 'sdn', 'sde(m)': 'sde', 'sdu(m)': 'sdu', 'sdne(m)': 'sdne', 'sdeu(m)': 'sdeu', 'sdun(m)': 'sdun', 'age(s)': 'age'})
# convert the GPS time to UTC
dfPos['DT'] = dfPos.apply(lambda x: gpstime.UTCFromWT(x['WNC'], x['TOW']), axis=1)
dTime = {}
dTime['epochs'] = dfPos.shape[0]
dTime['date'] = dfPos.DT.iloc[0].strftime('%d %b %Y')
dTime['start'] = dfPos.DT.iloc[0].strftime('%H:%M:%S')
dTime['end'] = dfPos.DT.iloc[-1].strftime('%H:%M:%S')
amc.dRTK['Time'] = dTime
# add UTM coordinates
dfPos['UTM.E'], dfPos['UTM.N'], dfPos['UTM.Z'], dfPos['UTM.L'] = utm.from_latlon(dfPos['lat'].to_numpy(), dfPos['lon'].to_numpy())
logger.info('{func:s}: added UTM coordiantes'.format(func=cFuncName))
# inform user
amc.logDataframeInfo(df=dfPos, dfName='dfPos', callerName=cFuncName, logger=logger)
logger.info('{func:s}: dTime = {time!s}'.format(func=cFuncName, time=dTime))
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfPos, dfName='{posf:s}'.format(posf=amc.dRTK['info']['rtkPosFile']))
# put the info of the dfPosn into debug logging
logger.debug('{func:s}: dfPos info\n{info!s}'.format(info=dfPos.info(), func=cFuncName))
return dfPos
def crdDiff(dMarker: dict, dfUTMh: pd.DataFrame, plotCrds: list,
logger: logging.Logger) -> Tuple[pd.DataFrame, dict]:
"""
calculates the differences of UTM,ellH using reference position or mean position
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
# determine the difference to weighted average or marker position of UTM (N,E), ellH to plot
dfCrd = pd.DataFrame(columns=plotCrds)
# determine the coordinates of used reference (either mean or user determined)
if [dMarker['UTM.E'], dMarker['UTM.N'],
dMarker['ellH']] == [np.NaN, np.NaN, np.NaN]:
# so no reference position given use mean position
originCrds = [float(amc.dRTK['WAvg'][crd]) for crd in plotCrds]
else:
# make difference to reference position
originCrds = [float(amc.dRTK['marker'][crd]) for crd in plotCrds]
# subtract origin coordinates from UTMh positions
dfCrd = dfUTMh.sub(originCrds, axis='columns')
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=dfCrd,
dfName='dfCrd')
crdMax = max(dfCrd.max())
crdMin = min(dfCrd.min())
crdMax = int(crdMax + (1 if crdMax > 0 else -1))
crdMin = int(crdMin + (1 if crdMin > 0 else -1))
dCrdLim = {'max': crdMax, 'min': crdMin}
return dfCrd, dCrdLim
def rise_set_times(prn: str, df_obstab: pd.DataFrame, nomint_multi: int,
logger: logging.Logger) -> Tuple[int, list, list, list]:
"""
rise_set_times determines observed rise and set times for PRN
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
# index the rows for this PRN
prn_loc = df_obstab.loc[df_obstab['PRN'] == prn].index
# logger.info('{func:s}: Data for {prn:s} are at indices\n{idx!s}'.format(prn=colored(prn, 'green'), idx=prn_loc, func=cFuncName))
# create a df_prn only using these rows
df_prn = df_obstab.iloc[prn_loc]
# amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=df_prn, dfName='df_prn[{:s}]'.format(prn), head=30)
# determine the datetime gaps for this PRN
df_prn['gap'] = (df_prn['DATE_TIME'] -
df_prn['DATE_TIME'].shift(1)).astype('timedelta64[s]')
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=df_prn,
dfName='df_prn[{:s}]'.format(prn))
# find the nominal time interval of observations
nominal_interval = df_prn['gap'].median()
# find the indices where the interval is bigger than nominal_interval
# idx_arc_start = df_prn[(df_prn['gap'] > nomint_multi * nominal_interval) | (df_prn['gap'].isna())].index.tolist()
idx_arc_start = [
df_prn.index[0]
] + df_prn[df_prn['gap'] > nomint_multi * nominal_interval].index.tolist()
idx_arc_end = df_prn[df_prn['gap'].shift(-1) > nomint_multi *
nominal_interval].index.tolist() + [df_prn.index[-1]]
# find the number of observations for each arc
obs_arc_count = []
for arc_start, arc_end in zip(idx_arc_start, idx_arc_end):
obs_arc_count.append(
prn_loc.get_loc(arc_end) - prn_loc.get_loc(arc_start) + 1)
# get the corresponding data time info
df_tmp = df_prn.loc[idx_arc_start][['DATE_TIME']]
# lst_time = [datetime.strptime(dt.strftime('%H:%M:%S'), '%H:%M:%S').time() for dt in df_tmp['DATE_TIME']]
# df_tmp.loc[:, 'time'] = lst_time
# dt_arc_start = pd.to_datetime(df_tmp['DATE_TIME']).tolist()
dt_arc_start = [
datetime.strptime(dt.strftime('%H:%M:%S'), '%H:%M:%S').time()
for dt in df_tmp['DATE_TIME']
]
print('dt_arc_start = {!s}'.format(dt_arc_start))
df_tmp = df_prn.loc[idx_arc_end][['DATE_TIME']]
# dt_arc_end = pd.to_datetime(df_tmp['DATE_TIME']).tolist()
dt_arc_end = [
datetime.strptime(dt.strftime('%H:%M:%S'), '%H:%M:%S').time()
for dt in df_tmp['DATE_TIME']
]
print('dt_arc_end = {!s}'.format(dt_arc_end))
logger.info(
'{func:s}: nominal observation interval for {prn:s} = {tint:f}'.
format(prn=colored(prn, 'green'),
tint=nominal_interval,
func=cFuncName))
# logger.info('{func:s}: {prn:s} rises at:\n{arcst!s}'.format(prn=colored(prn, 'green'), arcst=df_prn.loc[idx_arc_end][['DATE_TIME', 'PRN', 'gap']], func=cFuncName))
# logger.info('{func:s}: {prn:s} sets at:\n{arcend!s}'.format(prn=colored(prn, 'green'), arcend=df_prn.loc[idx_arc_end][['DATE_TIME', 'PRN', 'gap']], func=cFuncName))
for i, (stdt, enddt) in enumerate(zip(dt_arc_start, dt_arc_end)):
logger.info(
'{func:s}: arc[{nr:d}]: {stdt:s} -> {enddt:s}'.format(
nr=i,
stdt=colored(stdt.strftime('%H:%M:%S'), 'yellow'),
enddt=colored(enddt.strftime('%H:%M:%S'), 'yellow'),
func=cFuncName))
# copy the gap column for this PRN into original df
df_obstab.loc[df_prn.index, 'gap'] = df_prn['gap']
return nominal_interval, dt_arc_start, dt_arc_end, obs_arc_count
def plot_utm_ellh(dRtk: dict,
dfUTM: pd.DataFrame,
logger: logging.Logger,
showplot: bool = False):
"""
plots the UTM coordinates
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
# select colors for position mode
# colors = []
# colors.append([51 / 256., 204 / 256., 51 / 256.])
# colors.append([51 / 256., 51 / 256., 255 / 256.])
# colors.append([255 / 256., 51 / 256., 51 / 256.])
colors = [
'tab:white', 'tab:green', 'tab:olive', 'tab:orange', 'tab:cyan',
'tab:blue', 'tab:red', 'tab:pink', 'tab:purple', 'tab:brown'
]
# what to plot
crds2Plot = ['UTM.E', 'UTM.N', 'ellH', 'age']
stdDev2Plot = ['sde', 'sdn', 'sdu']
stdDevWAvg = ['sdUTM.E', 'sdUTM.N', 'sdellH']
# set up the plot
plt.style.use('ggplot')
# subplots
fig, ax = plt.subplots(nrows=len(crds2Plot),
ncols=1,
sharex=True,
figsize=(20.0, 16.0))
# make title for plot
ax[0].annotate(
'{camp:s} - {date:s} - {marker:s} ({pos:s}, quality {mode:s})'.format(
camp=dRtk['campaign'],
date=dRtk['obsStart'].strftime('%d %b %Y'),
marker=dRtk['marker'],
pos=dRtk['posFile'],
mode=dRtk['rtkqual'].upper()),
xy=(0.5, 1),
xycoords='axes fraction',
xytext=(0, 0),
textcoords='offset pixels',
horizontalalignment='center',
verticalalignment='bottom',
weight='strong',
fontsize='large')
# copyright this
ax[-1].annotate(r'$\copyright$ Alain Muls ([email protected])',
xy=(1, 1),
xycoords='axes fraction',
xytext=(0, 0),
textcoords='offset pixels',
horizontalalignment='right',
verticalalignment='bottom',
weight='strong',
fontsize='large')
# determine the difference to weighted average or marker position of UTM (N,E), ellH to plot
dfCrd = pd.DataFrame(columns=crds2Plot[:3])
originCrds = [float(amc.dRTK['WAVG'][crd]) for crd in crds2Plot[:3]]
dfCrd = dfUTM[crds2Plot[:3]].sub(originCrds, axis='columns')
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=dfCrd,
dfName='dfCrd')
crdMax = max(dfCrd.max())
crdMin = min(dfCrd.min())
crdMax = int(crdMax + (1 if crdMax > 0 else -1))
crdMin = int(crdMin + (1 if crdMin > 0 else -1))
# plot the coordinates dN, dE, dU and ns
for i, crd in enumerate(crds2Plot):
if i < 3: # subplots for coordinates display dN, dE, dU
# plot over the different coordinate offsets using different colors
# we plot offset to the weighted averga value
# ax[i].plot(dfUTM['DT'], dCrd[crd], linestyle='', marker='.', markersize=2, color=colors[i], label=crd)
# ax[i].fill_between(dfUTM['DT'], dCrd[crd]-dfUTM[stdDev2Plot[i]], dCrd[crd]+dfUTM[stdDev2Plot[i]], color=colors[i], alpha=0.15, interpolate=False)
for key, value in rtkc.dRTKQual.items():
# get the indices according to the position mode
idx = dfUTM.index[dfUTM['Q'] == key]
rgb = mpcolors.colorConverter.to_rgb(colors[key])
rgb_new = amutils.make_rgb_transparent(rgb, (1, 1, 1), 0.3)
# plot according to the color list if the length of the index is not 0
if len(idx) > 0:
ax[i].errorbar(x=dfUTM.loc[idx]['DT'],
y=dfCrd.loc[idx][crd],
yerr=dfUTM.loc[idx][stdDev2Plot[i]],
linestyle='None',
fmt='o',
ecolor=rgb_new,
capthick=2,
markersize=2,
color=colors[key],
label=value)
# set dimensions of y-axis
ax[i].set_ylim([crdMin, crdMax])
ax[i].set_ylabel('{crd:s} [m]'.format(crd=crd, fontsize='large'))
# lcoation of legend
ax[i].legend(loc='best', markerscale=4)
# annotate plot
annotatetxt = r'WAvg: {crd:.3f}m $\pm$ {sdcrd:.3f}m'.format(
crd=amc.dRTK['WAVG'][crds2Plot[i]],
sdcrd=amc.dRTK['WAVG'][stdDevWAvg[i]])
ax[i].annotate(annotatetxt,
xy=(1, 1),
xycoords='axes fraction',
xytext=(0, 0),
textcoords='offset pixels',
horizontalalignment='right',
verticalalignment='bottom',
fontweight='bold',
fontsize='large')
else: # last subplot: age of corrections & #SVs
# plot #SVs on left axis
# ax[i].set_ylim([0, 24])
# plot AGE value
ax[i].set_ylabel('Age [s]', fontsize='large', color='darkorchid')
ax[i].set_xlabel('Time [sec]', fontsize='large')
ax[i].plot(dfUTM['DT'],
dfUTM['age'],
linestyle='',
marker='x',
markersize=2,
color='darkorchid',
label='age')
ax[i].set_title('#SVs & Age of correction',
fontsize='large',
fontweight='bold')
# plot number of SV on second y-axis
axRight = ax[i].twinx()
axRight.set_ylim([0, 25])
axRight.set_ylabel('#SVs [-]', fontsize='large', color='grey')
ax[i].fill_between(dfUTM['DT'],
0,
dfUTM['ns'],
alpha=0.5,
linestyle='-',
linewidth=3,
color='grey',
label='#SVs',
interpolate=False)
# create the ticks for the time axis
dtFormat = plot_utils.determine_datetime_ticks(
startDT=dfUTM['DT'].iloc[0], endDT=dfUTM['DT'].iloc[-1])
if dtFormat['minutes']:
if dfUTM.shape[0] > 300:
ax[i].xaxis.set_major_locator(
dates.MinuteLocator(byminute=range(1, 60, 10),
interval=1))
else:
ax[i].xaxis.set_major_locator(
dates.MinuteLocator(byminute=range(1, 60), interval=1))
else:
ax[i].xaxis.set_major_locator(
dates.HourLocator(
interval=dtFormat['hourInterval'])) # every 4 hours
ax[i].xaxis.set_major_formatter(
dates.DateFormatter('%H:%M')) # hours and minutes
ax[i].xaxis.set_minor_locator(
dates.DayLocator(interval=1)) # every day
ax[i].xaxis.set_minor_formatter(dates.DateFormatter('\n%d-%m-%Y'))
ax[i].xaxis.set_tick_params(rotation=0)
for tick in ax[i].xaxis.get_major_ticks():
# tick.tick1line.set_markersize(0)
# tick.tick2line.set_markersize(0)
tick.label1.set_horizontalalignment('center')
# save the plot in subdir png of GNSSSystem
pngName = os.path.join(
dRtk['posDir'],
'{name:s}-ENU.png'.format(name=os.path.splitext(dRtk['posFile'])[0]))
fig.savefig(pngName, dpi=fig.dpi)
logger.info('{func:s}: created plot {plot:s}'.format(func=cFuncName,
plot=colored(
pngName,
'green')))
if showplot:
plt.show(block=True)
else:
plt.close(fig)
return
def plotUTMCoords(dStf: dict, dfCrd: pd.DataFrame, logger=logging.Logger):
"""
plots the UTM coordinates and #SVs on 4 different plots as a function of time
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
crds = ['UTM.E', 'UTM.N', 'Height[m]', 'dist', 'NrSV']
# crds = ['dist', 'NrSV']
logger.info(
'{func:s}: start plotting UTM coordinates'.format(func=cFuncName))
amutils.logHeadTailDataFrame(df=dfCrd,
dfName='dfCrd',
callerName=cFuncName,
logger=logger)
# specify the style
mpl.style.use('seaborn')
fig, axes = plt.subplots(nrows=len(crds), ncols=1, sharex=True)
fig.set_size_inches(18.5, 15)
# get the index for 2D/3D
idx3D = dfCrd.index[dfCrd['2D/3D'] == 0]
idx2D = dfCrd.index[dfCrd['2D/3D'] == 1]
# get the index for signals used for PNT AND for 3D/2D
dIdx = {} # dict with indices corresponding to signals & 3D/2D usage
for st, lstSTNames in dStf['signals'].items():
stNames = ",".join(lstSTNames)
logger.info('{func:s}: st = {st:d} name = {name!s}'.format(
st=st, name=stNames, func=cFuncName))
dIdx[st] = {}
dIdx[st]['3D'] = dfCrd.index[(dfCrd['SignalInfo'] == st)
& (dfCrd['2D/3D'] == 0)]
dIdx[st]['2D'] = dfCrd.index[(dfCrd['SignalInfo'] == st)
& (dfCrd['2D/3D'] == 1)]
logger.info(
'{func:s}: list of indices dIdx[{st:d}][3D] = {idx!s}'.format(
st=st, idx=dIdx[st]['3D'], func=cFuncName))
logger.info(
'{func:s}: list of indices dIdx[{st:d}][2D] = {idx!s}'.format(
st=st, idx=dIdx[st]['2D'], func=cFuncName))
# for setting the time on time-scale
dtFormat = plot_utils.determine_datetime_ticks(
startDT=dfCrd['time'].iloc[0], endDT=dfCrd['time'].iloc[-1])
for crd, ax in zip(crds, axes):
# print in order UTM.E, UTM.N, height, and NrSV and indicate 2D/3D by alpha
logger.info('{func:s}: plotting {crd:s}'.format(crd=crd,
func=cFuncName))
# x-axis properties
ax.set_xlim([dfCrd['time'].iloc[0], dfCrd['time'].iloc[-1]])
if dtFormat['minutes']:
ax.xaxis.set_major_locator(
dates.MinuteLocator(byminute=[0, 15, 30, 45], interval=1))
else:
ax.xaxis.set_major_locator(
dates.HourLocator(
interval=dtFormat['hourInterval'])) # every 4 hours
ax.xaxis.set_major_formatter(
dates.DateFormatter('%H:%M')) # hours and minutes
ax.xaxis.set_minor_locator(dates.DayLocator(interval=1)) # every day
ax.xaxis.set_minor_formatter(dates.DateFormatter('\n%d-%m-%Y'))
ax.xaxis.set_tick_params(rotation=0)
for tick in ax.xaxis.get_major_ticks():
# tick.tick1line.set_markersize(0)
# tick.tick2line.set_markersize(0)
tick.label1.set_horizontalalignment('center')
# (re)set the color iterator
colorsIter = iter(list(mcolors.TABLEAU_COLORS))
if crd is not 'NrSV':
# plot according to signals used and 2D/3D
for st, lstSTNames in dStf['signals'].items():
stNames = ",".join(lstSTNames)
for mode in '3D', '2D':
lblTxt = '{st:s} ({mode:s})'.format(st=stNames, mode=mode)
logger.debug('{func:s}: plotting {stm:s}'.format(
stm=lblTxt, func=cFuncName))
# get the index for this sigType & mode
idx = dIdx[st][mode]
ax.plot(dfCrd['time'].iloc[idx],
dfCrd[crd].iloc[idx],
color=next(colorsIter),
linestyle='',
marker='.',
label=lblTxt,
markersize=2)
else:
# plot when 3D posn
ax.fill_between(dfCrd['time'], dfCrd[crd], color='grey', alpha=.2)
# plot when 3D posn
ax.plot(dfCrd['time'].iloc[idx3D],
dfCrd[crd].iloc[idx3D],
color='green',
linestyle='',
marker='.',
markersize=2,
label='3D')
# plot when 2D posn
ax.plot(dfCrd['time'].iloc[idx2D],
dfCrd[crd].iloc[idx2D],
color='red',
linestyle='',
marker='.',
markersize=2,
label='2D')
# name y-axis
ax.set_ylabel(crd, fontsize=14)
# add a legend the plot showing 2D/3D positioning displayed
ax.legend(loc='best', ncol=16, markerscale=5)
# title of plot
title = '{syst:s}: UTM Coordinates'.format(syst=dStf['gnss'])
fig.suptitle(title, fontsize=16)
# copyright this
ax.annotate(r'$\copyright$ Alain Muls ([email protected])',
xy=(1, 0),
xycoords='axes fraction',
xytext=(0, -45),
textcoords='offset pixels',
horizontalalignment='right',
verticalalignment='bottom',
weight='strong',
fontsize='medium')
# Save the file in dir png
pltDir = os.path.join(dStf['dir'], 'png')
os.makedirs(pltDir, exist_ok=True)
pltName = '{syst:s}-UTM.png'.format(syst=dStf['gnss'].replace(' ', '-'))
pltName = os.path.join(pltDir, pltName)
fig.savefig(pltName, dpi=100)
logger.info('{func:s}: plot saved as {name:s}'.format(name=pltName,
func=cFuncName))
plt.show(block=False)
def plotUTMSuppressed(dStf: dict, dfCrd: pd.DataFrame, logger=logging.Logger):
"""
plots the UTM E-N scatter
PVT error code. The following values are defined:
0: no error.
1: not enough measurements
2: not enough ephemerides available
3: DOP too large (larger than 15)
4: sum of squared residuals too large
5: no convergence
6: not enough measurements after outlier rejection
7: position output prohibited due to export laws
8: not enough differential corrections available
9: base station coordinates unavailable
127: valid position output actively suppressed (e.g. PRS denial)
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: start plotting (un)suppressed trajectories'.format(
func=cFuncName))
# specify the style
mpl.style.use('seaborn')
# get the index for PVT suppression
dIdx = {} # dict with indices corresponding to PNT suppression
for errCode in dStf['errCodes']:
dIdx[errCode] = dfCrd.index[dfCrd['Error'] == errCode]
logger.info(
'{func:s}: list of indices dIdx[{errc:d}] = {idx!s}'.format(
errc=errCode, idx=dIdx[errCode], func=cFuncName))
# convert time column to seconds
dsTime = dfCrd['time'] - dfCrd['time'].iloc[0]
dfCrd['sec'] = dsTime.dt.total_seconds()
amutils.logHeadTailDataFrame(df=dfCrd,
dfName='dfCrd plot scatter',
callerName=cFuncName,
logger=logger)
# get index when sec is multiple of 10 minutes
idxTime = dfCrd.index[dfCrd['sec'] % 300 == 0]
logger.debug('{func:s}: indices multiple of 300s = {idx!s}'.format(
idx=idxTime, func=cFuncName))
fig, ax = plt.subplots(nrows=1, ncols=1)
fig.set_size_inches(14, 14)
ax.axis('equal')
# copyright this
ax.annotate(r'$\copyright$ Alain Muls ([email protected])',
xy=(1, 0),
xycoords='axes fraction',
xytext=(0, -45),
textcoords='offset pixels',
horizontalalignment='right',
verticalalignment='bottom',
weight='strong',
fontsize='medium')
# (re)set the color iterator
colorsIter = iter(list(mcolors.TABLEAU_COLORS))
# plot the E-N coordinates according to PVT Error mode
for errCode, errCodeName in dStf['errCodes'].items():
logger.debug('{func:s}: plotting {errc:d}: {errtxt:s}'.format(
errc=errCode, errtxt=errCodeName, func=cFuncName))
# get the index for this error code
idx = dIdx[errCode]
ax.plot(dfCrd['UTM.E'].iloc[idx],
dfCrd['UTM.N'].iloc[idx],
color=next(colorsIter),
linestyle='',
marker='.',
label=errCodeName,
markersize=4)
# ax.plot(dfCrd['UTM.E'].iloc[idx3D], dfCrd['UTM.N'].iloc[idx3D], color='blue', label='3D mode', markersize=2, linestyle='', marker='.')
# ax.plot(dfCrd['UTM.E'].iloc[idx2D], dfCrd['UTM.N'].iloc[idx2D], color='red', label='2D mode', markersize=2, linestyle='', marker='.')
# annotate plot with time
annText = [dfCrd['time'][idx].strftime('%H:%M:%S') for idx in idxTime]
logger.info('{func:s}: annotate text\n{ann!s}'.format(ann=annText,
func=cFuncName))
for idx, text in zip(idxTime, annText):
ax.annotate(text, (dfCrd['UTM.E'].iloc[idx], dfCrd['UTM.N'].iloc[idx]),
textcoords='offset points',
xytext=(0, 10),
ha='center')
# # draw circles for zones on plot
# for zone, zone_crd in dStf['zones'].items():
# E, N, R = zone_crd['UTM.E'], zone_crd['UTM.N'], zone_crd['radius']
# # draw marker & cricle
# ax.scatter(E, N, color='black', marker='^', alpha=0.4)
# newCircle = plt.Circle((E, N), R, color='black', fill=False, clip_on=True)
# ax.add_artist(newCircle)
# # annotate the markers
# ax.annotate('{zone:s}'.format(zone=zone), xy=(E + 2, N), textcoords='data', xycoords='data', clip_on=True, color='black', alpha=0.4)
# # break
# name y-axis
ax.set_xlabel('UTM.E', fontsize=14)
ax.set_ylabel('UTM.N', fontsize=14)
# add a legend the plot showing 2D/3D positioning displayed
ax.legend(loc='best', ncol=16, markerscale=5)
# title of plot
title = '{syst:s}: UTM Trajectory'.format(syst=dStf['gnss'])
fig.suptitle(title, fontsize=16)
# Save the file in dir png
pltDir = os.path.join(dStf['dir'], 'png')
os.makedirs(pltDir, exist_ok=True)
pltName = '{syst:s}-UTMsuppressed.png'.format(
syst=dStf['gnss'].replace(' ', '-'))
pltName = os.path.join(pltDir, pltName)
fig.savefig(pltName, dpi=100)
logger.info('{func:s}: plot saved as {name:s}'.format(name=pltName,
func=cFuncName))
plt.show(block=True)
def main(argv) -> bool:
"""
glabplotposn plots data from gLAB (v6) OUTPUT messages
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
# store cli parameters
amc.dRTK = {}
cli_opt = {}
cli_opt['glab_db'], cli_opt['gnsss'], cli_opt['prcodes'], cli_opt[
'markers'], cli_opt['yyyy'], cli_opt['doy_begin'], cli_opt[
'doy_last'], show_plot, log_levels = treatCmdOpts(argv)
amc.dRTK['options'] = cli_opt
# create logging for better debugging
logger, log_name = amc.createLoggers(os.path.basename(__file__),
dir=os.getcwd(),
logLevels=log_levels)
# check arguments
ret_val = check_arguments(logger=logger)
if ret_val != amc.E_SUCCESS:
sys.exit(ret_val)
# for crds in ['ENU', 'dENU']:
for crds in ['ENU']:
# parse the database file to get the GNSSs and prcodes we need
tmp_name = glabdb_parse.db_parse_gnss_codes(
db_name=amc.dRTK['options']['glab_db'],
crd_types=glc.dgLab['OUTPUT'][crds],
logger=logger)
# read into dataframe
logger.info(
'{func:s}: reading selected information into dataframe'.format(
func=cFuncName))
colnames = ['yyyy', 'doy', 'gnss', 'marker', 'prcodes', 'crds']
if crds == 'ENU':
colnames += ['mean', 'std', 'max', 'min']
print('colnames = {!s}'.format(colnames))
try:
df_crds = pd.read_csv(tmp_name, names=colnames, header=None)
# test time
d = datetime.date(2020, 1, 1) + datetime.timedelta(39 - 1)
print(d)
# convert YYYY/DOY to a datetime.date field
df_crds['DT'] = df_crds.apply(lambda x: datetime.date(
x['yyyy'], 1, 1) + datetime.timedelta(x['doy'] - 1),
axis=1)
except FileNotFoundError as e:
logger.critical('{func:s}: Error = {err!s}'.format(err=e,
func=cFuncName))
sys.exit(amc.E_FILE_NOT_EXIST)
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=df_crds,
dfName='df[{crds:s}]'.format(crds=crds))
# determine statistics
if crds == 'ENU':
# statistics over the coordinates ENU per prcode selected
amc.dRTK['stats_{crd:s}'.format(
crd=crds)] = glabdb_statistics.crd_statistics(
crds=crds,
prcodes=amc.dRTK['options']['prcodes'],
df_crds=df_crds,
logger=logger)
# plot the mean / std values for all prcodes per ENU coordinates
glabdb_plot_crds.plot_glabdb_position(
crds=crds,
prcodes=amc.dRTK['options']['prcodes'],
df_crds=df_crds,
logger=logger,
showplot=show_plot)
# report to the user
logger.info('{func:s}: Project information =\n{json!s}'.format(
func=cFuncName,
json=json.dumps(amc.dRTK,
sort_keys=False,
indent=4,
default=amutils.DT_convertor)))
return amc.E_SUCCESS
def plotUTMScatter(dStf: dict, dfCrd: pd.DataFrame, logger=logging.Logger):
"""
plots the UTM E-N scatter
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: start plotting trajectories'.format(func=cFuncName))
# specify the style
mpl.style.use('seaborn')
# get the index for signals used for PNT AND for 3D/2D
dIdx = {} # dict with indices corresponding to signals & 3D/2D usage
for st, lstSTNames in dStf['signals'].items():
stNames = ",".join(lstSTNames)
logger.info('{func:s}: st = {st:d} name = {name:s}'.format(
st=st, name=stNames, func=cFuncName))
dIdx[st] = {}
dIdx[st]['3D'] = dfCrd.index[(dfCrd['SignalInfo'] == st)
& (dfCrd['2D/3D'] == 0)]
dIdx[st]['2D'] = dfCrd.index[(dfCrd['SignalInfo'] == st)
& (dfCrd['2D/3D'] == 1)]
logger.info(
'{func:s}: list of indices dIdx[{st:d}][3D] = {idx!s}'.format(
st=st, idx=dIdx[st]['3D'], func=cFuncName))
logger.info(
'{func:s}: list of indices dIdx[{st:d}][2D] = {idx!s}'.format(
st=st, idx=dIdx[st]['2D'], func=cFuncName))
# convert time column to seconds
dsTime = dfCrd['time'] - dfCrd['time'].iloc[0]
dfCrd['sec'] = dsTime.dt.total_seconds()
amutils.logHeadTailDataFrame(df=dfCrd,
dfName='dfCrd plot scatter',
callerName=cFuncName,
logger=logger)
# get index when sec is multiple of 10 minutes
idxTime = dfCrd.index[dfCrd['sec'] % 600 == 0]
logger.debug('{func:s}: indices multiple of 300s = {idx!s}'.format(
idx=idxTime, func=cFuncName))
fig, ax = plt.subplots(nrows=1, ncols=1)
fig.set_size_inches(14, 14)
ax.axis('equal')
# copyright this
ax.annotate(r'$\copyright$ Alain Muls ([email protected])',
xy=(1, 0),
xycoords='axes fraction',
xytext=(0, -45),
textcoords='offset pixels',
horizontalalignment='right',
verticalalignment='bottom',
weight='strong',
fontsize='medium')
# get the index for 2D/3D
idx3D = dfCrd.index[dfCrd['2D/3D'] == 0]
idx2D = dfCrd.index[dfCrd['2D/3D'] == 1]
# (re)set the color iterator
colorsIter = iter(list(mcolors.TABLEAU_COLORS))
# plot the E-N coordinates according to signals used and 2D/3D mode
for st, lstSTNames in dStf['signals'].items():
stNames = ",".join(lstSTNames)
for mode in '3D', '2D':
lblTxt = '{st:s} ({mode:s})'.format(st=stNames, mode=mode)
logger.debug('{func:s}: plotting {stm:s}'.format(stm=lblTxt,
func=cFuncName))
# get the index for this sigType & mode
idx = dIdx[st][mode]
ax.plot(dfCrd['UTM.E'].iloc[idx],
dfCrd['UTM.N'].iloc[idx],
color=next(colorsIter),
linestyle='',
marker='.',
label=lblTxt,
markersize=2)
# ax.plot(dfCrd['UTM.E'].iloc[idx3D], dfCrd['UTM.N'].iloc[idx3D], color='blue', label='3D mode', markersize=2, linestyle='', marker='.')
# ax.plot(dfCrd['UTM.E'].iloc[idx2D], dfCrd['UTM.N'].iloc[idx2D], color='red', label='2D mode', markersize=2, linestyle='', marker='.')
# annotate plot with time
annText = [dfCrd['time'][idx].strftime('%H:%M:%S') for idx in idxTime]
logger.info('{func:s}: annotate text\n{ann!s}'.format(ann=annText,
func=cFuncName))
for idx, text in zip(idxTime, annText):
ax.annotate(text, (dfCrd['UTM.E'].iloc[idx], dfCrd['UTM.N'].iloc[idx]),
textcoords='offset points',
xytext=(0, 10),
ha='center')
# annotate with position of marker
logger.info('{func:s}: marker location = {E!s} {N!s}'.format(
E=dStf['marker']['UTM.E'], N=dStf['marker']['UTM.N'], func=cFuncName))
E, N = dStf['marker']['UTM.E'], dStf['marker']['UTM.N']
ax.annotate('marker',
xy=(E, N),
xytext=(E - 200, N),
xycoords='data',
horizontalalignment='right',
verticalalignment='center',
color='magenta')
ax.scatter(E, N, color='magenta', marker='^')
# draw circles for distancd evaluation on plot
for radius in range(5, 50, 5):
newCircle = plt.Circle((E, N),
radius * 1000,
color='red',
fill=False,
clip_on=True,
alpha=0.4)
ax.add_artist(newCircle)
# annotate the radius
ax.annotate('{radius:d} km'.format(radius=radius),
xy=(E + radius * 1000 * np.cos(np.pi * 5 / 4),
N + radius * 1000 * np.sin(np.pi * 5 / 4)),
textcoords='data',
xycoords='data',
clip_on=True,
color='blue',
alpha=0.4)
for radius in 2.5, 7.5:
newCircle = plt.Circle((E, N),
radius * 1000,
color='red',
fill=False,
clip_on=True,
alpha=0.4)
ax.add_artist(newCircle)
# annotate the radius
ax.annotate('{radius:.1f} km'.format(radius=radius),
xy=(E + radius * 1000 * np.cos(np.pi * 5 / 4),
N + radius * 1000 * np.sin(np.pi * 5 / 4)),
textcoords='data',
xycoords='data',
clip_on=True,
color='blue',
alpha=0.4)
# name y-axis
ax.set_xlabel('UTM.E', fontsize=14)
ax.set_ylabel('UTM.N', fontsize=14)
# add a legend the plot showing 2D/3D positioning displayed
ax.legend(loc='best', ncol=16, markerscale=5)
# title of plot
title = '{syst:s}: UTM Trajectory'.format(syst=dStf['gnss'])
fig.suptitle(title, fontsize=16)
# Save the file in dir png
pltDir = os.path.join(dStf['dir'], 'png')
os.makedirs(pltDir, exist_ok=True)
pltName = '{syst:s}-UTMscatter.png'.format(
syst=dStf['gnss'].replace(' ', '-'))
pltName = os.path.join(pltDir, pltName)
fig.savefig(pltName, dpi=100)
logger.info('{func:s}: plot saved as {name:s}'.format(name=pltName,
func=cFuncName))
plt.show(block=True)
def rnxobs_dataframe(rnx_file: str, prn: str, dPRNSysObs: dict, dProgs: dict,
logger: logging.Logger) -> dict:
"""
rnxobs_dataframe selects the observations for a PRN and returns observation dataframe
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
logger.info(
'{func:s}: creating dataframe for PRN {prn:s} with observations {obs:s}'
.format(prn=prn, obs=', '.join(dPRNSysObs), func=cFuncName))
# create tabular output for this PRN
tab_name = os.path.join(tempfile.gettempdir(),
'{tmpname:s}.tab'.format(tmpname=uuid.uuid4().hex))
cmdGFZRNX = '{prog:s} -finp {rnx:s} -fout {out:s} -tab_obs -tab_sep "," -prn {prn:s} -obs_types={obs:s}'.format(
prog=dProgs['gfzrnx'],
rnx=rnx_file,
out=tab_name,
prn=prn,
obs=','.join(dPRNSysObs))
logger.info('{func:s}: Running:\n{prog:s}'.format(prog=colored(
cmdGFZRNX, 'blue'),
func=cFuncName))
# run the program
# gfzrnx -finp P1710171.20O -tab_obs -fout P1710171_20O.tab -prn E09 -obs_types C1C,C5Q -tab_sep ','
exeprogram.subProcessDisplayStdErr(cmd=cmdGFZRNX, verbose=False)
# find the header lines that don't belong to this GNSS system and remove that line
hdlookup = '#HD,'
gnsslookup = '#HD,{gnss:s}'.format(gnss=prn[0])
hd_lines = []
gnss_lines = []
with open(tab_name) as f:
for num, line in enumerate(f, 1):
if hdlookup in line:
# print('found at line: ', num)
hd_lines.append(num)
if gnsslookup in line:
# print('found at line: ', num)
gnss_lines.append(num)
# print('hd_lines = {!s}'.format(hd_lines))
# print('gnss_lines = {!s}'.format(gnss_lines))
remove_lines = [linenr for linenr in hd_lines if linenr not in gnss_lines]
# print('remove_lines = {!s}'.format(remove_lines))
# remove the lines that are not related to current GNSS
amutils.delete_lines(original_file=tab_name, lst_line_number=remove_lines)
# read the CSV created file into a panda dataframe
dfPrn = pd.read_csv(tab_name, delimiter=',')
# add datetime columns
sDT = dfPrn.apply(lambda x: DT_from_DTstr(x['DATE'], x['TIME']), axis=1)
dfPrn.insert(loc=4, column='DT', value=sDT)
amutils.logHeadTailDataFrame(
logger=logger,
callerName=cFuncName,
df=dfPrn,
dfName='{tab:s}'.format(tab='{prn:s} with observables = {obs!s}'.
format(prn=prn, obs=', '.join(dPRNSysObs))))
# remove the temporary tabular file
os.remove(tab_name)
return dfPrn
def parse_elevation_distribution(dRtk: dict, dfSat: pd.DataFrame, logger: logging.Logger) -> Tuple[pd.DataFrame, pd.Series, pd.DataFrame, pd.Series]:
"""
parse_elevation_distribution parses the observed resiudals per constellation and per elevation bin of 10 degrees
"""
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: parses observed resiudals as funcion of elevation²'.format(func=cFuncName))
# define what we use for binning
gnssLetters = ('E', 'G')
gnssNames = ('GAL', 'GPS')
cols = ['Elev', 'PRres', 'CN0']
# define the bins used
elev_bins, step = np.linspace(start=0, stop=90, num=7, endpoint=True, retstep=True, dtype=int, axis=0)
logger.info('{func:s}: elevation bins = {bins!s}'.format(bins=elev_bins, func=cFuncName))
CN0_bins, step = np.linspace(start=10, stop=70, num=13, endpoint=True, retstep=True, dtype=int, axis=0)
logger.info('{func:s}: CN0 bins = {bins!s}'.format(bins=CN0_bins, func=cFuncName))
PRres_bins, step = np.linspace(start=-5, stop=5, num=21, endpoint=True, retstep=True, dtype=float, axis=0)
tmpArr = np.append(PRres_bins, np.inf)
PRres_bins = np.append(-np.inf, tmpArr)
logger.info('{func:s}: PRres bins = {bins!s}'.format(bins=PRres_bins, func=cFuncName))
# add to dRtk the bins used
# create dataframe for CN0 / PRres distribution
dfCN0dist = pd.DataFrame()
dfPRresdist = pd.DataFrame()
# calculate the number of PRres within [-2,+2] and CN0 statistics over the elevation bins
for gnssLetter, gnssName in zip(gnssLetters, gnssNames):
# create a dataframe to work on
dfGNSS = dfSat[dfSat['SV'].str.startswith(gnssLetter)][cols]
if len(dfGNSS.index):
# display frame for this system
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfGNSS, dfName=gnssName)
# go over the elevation bins
for elev_min, elev_max in zip(elev_bins[:-1], elev_bins[1:]):
dfGNSS['elevbin'] = dfGNSS.Elev.between(elev_min, elev_max, inclusive=True)
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfGNSS, dfName='{syst:s} in [{min:d}..{max:d}] deg'.format(syst=gnssName, min=elev_min, max=elev_max))
# create a distribution for PRres
dfPRresdist['{syst:s}[{min:d}..{max:d}]'.format(syst=gnssName, min=elev_min, max=elev_max)] = pd.cut(dfGNSS.loc[dfGNSS['elevbin']]['PRres'], bins=PRres_bins).value_counts(sort=False)
# create a distribution for CN0
dfCN0dist['{syst:s}[{min:d}..{max:d}]'.format(syst=gnssName, min=elev_min, max=elev_max)] = pd.cut(dfGNSS.loc[dfGNSS['elevbin']]['CN0'], bins=CN0_bins).value_counts(sort=False)
# reduce the values to percentage based on observations taken over all bins
dsPRres_per_bin = dfPRresdist.loc[:, dfPRresdist.columns].sum()
PRres_total = dsPRres_per_bin.sum() / 100
dsCN0_per_bin = dfCN0dist.loc[:, dfCN0dist.columns].sum()
CN0_total = dsCN0_per_bin.sum() / 100
# report the distibutions of CN0 and PRres to the user
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfPRresdist, dfName='dfPRresdist')
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfPRresdist / PRres_total, dfName='dfPRresdist in percentage')
logger.info('{func:s}: PRres totals per elev bin = \n{bins!s}'.format(bins=dsPRres_per_bin, func=cFuncName))
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfCN0dist, dfName='dfCN0dist')
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfCN0dist / CN0_total, dfName='dfCN0dist in percentage')
logger.info('{func:s}: CN0 totals per elev bin = \n{bins!s}'.format(bins=dsCN0_per_bin, func=cFuncName))
return dfCN0dist, dsCN0_per_bin, dfPRresdist, dsPRres_per_bin
def readSTFGeodetic(stfFile: str, logger: logging.Logger) -> pd.DataFrame:
"""
read in the STF Geodetic_v2 file using included header information
"""
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
# read in the file with in
logger.info('{func:s}: reading file {file:s}'.format(file=stfFile, func=cFuncName))
dfSTF = pd.read_csv(stfFile, sep=',', skiprows=range(1, 2))
amutils.logHeadTailDataFrame(df=dfSTF, dfName=dSTF['stf'], callerName=cFuncName, logger=logger)
dfSTF.dropna(subset=['Latitude[rad]', 'Longitude[rad]'], inplace=True)
amutils.logHeadTailDataFrame(df=dfSTF, dfName=dSTF['stf'], callerName=cFuncName, logger=logger)
dfSTF.reset_index(inplace=True)
# zone definition
dZone = {}
dZone['allow'] = {'lat': 50.934519, 'lon': 4.466130, 'radius': 300}
dZone['deny'] = {'lat': 50.934877, 'lon': 4.466280, 'radius': 200}
# convert lat/lon to UTM
for zone, zone_crd in dZone.items():
dZone[zone]['UTM.E'], dZone[zone]['UTM.N'], dZone[zone]['UTM.Z'], dZone[zone]['UTM.L'] = UTM.from_latlon(dZone[zone]['lat'], dZone[zone]['lon'])
# add to dict dStf
dSTF['zones'] = dZone
dfSTF['lat'] = np.degrees(dfSTF['Latitude[rad]'])
dfSTF['lon'] = np.degrees(dfSTF['Longitude[rad]'])
# convert the GPS time to UTC
dfSTF['time'] = dfSTF.apply(lambda x: gpstime.UTCFromWT(x['WNc[week]'], x['TOW[s]']), axis=1)
# add UTM coordinates
dfSTF['UTM.E'], dfSTF['UTM.N'], dfSTF['UTM.Z'], dfSTF['UTM.L'] = UTM.from_latlon(dfSTF['lat'].to_numpy(), dfSTF['lon'].to_numpy())
# calculate distance to st-Niklass 51.1577189 4.1915975
dfSTF['dist'] = np.linalg.norm(dfSTF[['UTM.E', 'UTM.N']].sub(np.array([dSTF['marker']['UTM.E'], dSTF['marker']['UTM.N']])), axis=1)
# dfSTF['dist2'] = np.linalg.norm([dfSTF['UTM.E'].iloc[0], dfSTF['UTM.N'].iloc[0]] - [dSTF['marker']['UTM.E'], dSTF['marker']['UTM.N']])
# add info to dSTF about time
dTime = {}
dTime['epochs'] = dfSTF.shape[0]
dTime['date'] = dfSTF.time.iloc[0].strftime('%d %b %Y')
dTime['start'] = dfSTF.time.iloc[0].strftime('%H:%M:%S')
dTime['end'] = dfSTF.time.iloc[-1].strftime('%H:%M:%S')
dSTF['Time'] = dTime
# add info to dSTF about #epochs
dSTF['#epochs'] = dfSTF.shape[0]
# add info to dSTF about used signal types used
dST = {}
sigTypes = dfSTF.SignalInfo.unique()
logger.info('{func:s}: found nav-signals {sigt!s}'.format(sigt=sigTypes, func=cFuncName))
for i, sigType in enumerate(sigTypes):
logger.debug('{func:s}: searching name for sig-type {st!s}'.format(st=sigType, func=cFuncName))
sigTypeNames = []
for k, v in ssnst.dSigType.items():
# logger.debug('{func:s}: checking presence of signal {sig!s}'.format(sig=v, func=cFuncName))
# logger.debug('{func:s}: bin(sigType) = {st!s}'.format(st=bin(sigType), func=cFuncName))
# logger.debug('{func:s}: bin(0b1 << k) = {ssnst!s}'.format(ssnst=bin(0b1 << k), func=cFuncName))
# logger.debug('{func:s}: bin(bin(sigType) & bin(0b1 << k)) = {binops!s})'.format(binops=bin(sigType & (0b1 << k)), func=cFuncName))
# logger.debug('{func:s}: binary check sigtype = {st!s} - ssn = {ssnst!s} operator and = {opsbin!s}'.format(st=bin(sigType), ssnst=bin(0b1 << k), opsbin=bin(sigType & (0b1 << k)), func=cFuncName))
# logger.debug('-' * 10)
if (sigType & (0b1 << k)) != 0:
logger.info('{func:s}: found signal {ssnst:s}'.format(ssnst=v, func=cFuncName))
# add name to the used signal types
sigTypeNames.append(v)
# add signal to the dST dict
dST[sigType] = sigTypeNames
# nrBitsSet = ssnst.countSetBits(sigType)
# lst1Bits = ssnst.findAllSetBits(sigType, nrBitsSet)
# # get the name of the signals
# stName = ssnst.dSigType[lst1Bits[0]]
# if nrBitsSet > 1:
# for j in lst1Bits[1:]:
# stName += '+' + ssnst.dSigType[j]
# dST[sigType] = stName
dSTF['signals'] = dST
logger.info('{func:s}: found signals {signals!s}'.format(signals=dSTF['signals'], func=cFuncName))
# find out what PVT error codess we have
dErrCodes = {}
errCodes = list(set(dfSTF.Error.unique()))
for errCode in errCodes:
logger.debug('{func:s}: searching name for error codes {errc:d}'.format(errc=errCode, func=cFuncName))
for k, v in ssnst.dPVTErrorCode.items():
if (errCode == k):
logger.info('{func:s}: found error code {errc:s}'.format(errc=colored(v, 'green'), func=cFuncName))
# add error code to errCodeNames
dErrCodes[errCode] = v
dSTF['errCodes'] = dErrCodes
logger.info('{func:s}: found error codes {errc!s}'.format(errc=errCodes, func=cFuncName))
# inform user
logger.info('{func:s}: read STF file {file:s}, added UTM coordiantes and GNSS time'.format(file=stfFile, func=cFuncName))
return dfSTF
def main(argv):
"""
pyRTKPlot adds UTM coordinates to output of rnx2rtkp.
If 'stat' file is available, calculates xDOP values, and make plots of statictics
"""
amc.cBaseName = colored(os.path.basename(__file__), 'yellow')
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
# some options for diasplay of dataframes
pd.set_option('display.max_columns', None) # or 1000
pd.set_option('display.max_rows', None) # or 1000
pd.set_option('display.max_colwidth', -1) # or 199
# limit float precision
json.encoder.FLOAT_REPR = lambda o: format(o, '.3f')
# treat command line options
posFile, rootDir, subDir, rtkqual, marker, campaign, excel, logLevels = treatCmdOpts(
argv)
# store cli parameters
amc.dRTK = {}
amc.dRTK['rootDir'] = rootDir
amc.dRTK['subDir'] = subDir
amc.dRTK['posDir'] = os.path.join(rootDir, subDir)
amc.dRTK['posFile'] = posFile
amc.dRTK['rtkqual'] = rtkqual
amc.dRTK['marker'] = marker
amc.dRTK['campaign'] = campaign
amc.dRTK['excel'] = excel
# get th enumeric vale for this quality
amc.dRTK['iQual'] = [
key for key, value in rtkc.dRTKQual.items()
if value == amc.dRTK['rtkqual']
][0]
if amc.dRTK['excel']:
amc.dRTK['xlsName'] = os.path.join(
amc.dRTK['rootDir'],
'{pos:s}.xlsx'.format(pos=amc.dRTK['campaign']))
# create logging for better debugging
logger = amc.createLoggers(baseName=os.path.basename(__file__),
dir=amc.dRTK['posDir'],
logLevels=logLevels)
# change to selected directory if exists
if not os.path.exists(amc.dRTK['posDir']):
logger.error('{func:s}: directory {dir:s} does not exists'.format(
func=cFuncName, dir=colored(amc.dRTK['posDir'], 'red')))
sys.exit(amc.E_DIR_NOT_EXIST)
else:
os.chdir(amc.dRTK['posDir'])
logger.info('{func:s}: changed to dir {dir:s}'.format(
func=cFuncName, dir=colored(amc.dRTK['posDir'], 'green')))
# check wether pos and stat file are present, else exit
if not os.access(os.path.join(amc.dRTK['posDir'], amc.dRTK['posFile']),
os.R_OK):
logger.error('{func:s}: file {pos:s} is not accessible'.format(
func=cFuncName,
pos=os.path.join(amc.dRTK['posDir'], amc.dRTK['posFile'])))
sys.exit(amc.E_FILE_NOT_EXIST)
# read the position file into a dataframe and add dUTM coordinates
dfPos = parse_rtkpos_file.parsePosFile(logger=logger)
# get the indices according to the position mode
idx = dfPos.index[dfPos['Q'] == amc.dRTK['iQual']]
# amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfPos.loc[idx], dfName='{posf:s}'.format(posf=amc.dRTK['posFile']))
# determine statistics for the requested quality mode
logger.info('{func:s}: stats are\n{stat!s}'.format(
func=cFuncName,
stat=dfPos.loc[idx][['lat', 'lon', 'ellH', 'UTM.E',
'UTM.N']].describe()))
# add weighted average for th erequested quality of position
llh = ['lat', 'lon', 'ellH', 'UTM.E', 'UTM.N']
dSDenu = ['sdn', 'sde', 'sdu', 'sdn', 'sde']
dWAVG = {}
for values in zip(llh, dSDenu):
dWAVG[values[0]] = parse_rtkpos_file.wavg(group=dfPos.loc[idx],
avg_name=values[0],
weight_name=values[1])
# calculate the stddev of the weighted average
for crd in llh[2:]:
# print('crd = {!s}'.format(crd))
dWAVG['sd{crd:s}'.format(crd=crd)] = parse_rtkpos_file.stddev(
dfPos.loc[idx][crd], dWAVG[(crd)])
# print(dWAVG)
# print('{crd:s} = {sd:.3f}'.format(crd=crd, sd=dWAVG['sd{crd:s}'.format(crd=crd)]))
# get UTM coordiantes/zone for weigted average
dWAVG['UTM.E'], dWAVG['UTM.N'], dWAVG['UTM.Z'], dWAVG[
'UTM.L'] = utm.from_latlon(dWAVG['lat'], dWAVG['lon'])
amc.dRTK['WAVG'] = dWAVG
logger.info('{func:s}: weighted averages: {wavg!s}'.format(func=cFuncName,
wavg=dWAVG))
amutils.logHeadTailDataFrame(
logger=logger,
callerName=cFuncName,
df=dfPos,
dfName='{posf:s}'.format(posf=amc.dRTK['posFile']))
# create UTM plot
plot_utm.plot_utm_ellh(dRtk=amc.dRTK,
dfUTM=dfPos,
logger=logger,
showplot=True)
# add results to campaign file
addRTKResult(logger)
# write to csv file
csvName = os.path.join(amc.dRTK['posDir'],
'{pos:s}.csv'.format(pos=amc.dRTK['posFile']))
dfPos.to_csv(csvName, index=None, header=True)
# add sheet write to excel workbook
if amc.dRTK['excel']:
sheetName = '{pos:s}-{qual:s}'.format(pos=os.path.splitext(
os.path.basename(amc.dRTK['posFile']))[0],
qual=amc.dRTK['rtkqual'])
df2excel.append_df_to_excel(filename=amc.dRTK['xlsName'],
df=dfPos,
sheet_name=sheetName,
truncate_sheet=True,
startrow=0,
index=False,
float_format="%.9f")
logger.info(
'{func:s}: added sheet {sheet:s} to workbook {wb:s}'.format(
func=cFuncName, sheet=sheetName, wb=amc.dRTK['xlsName']))
logger.info('{func:s}: amc.dRTK =\n{settings!s}'.format(func=cFuncName,
settings=amc.dRTK))
def main(argv):
"""
creates a combined SBF file from hourly or six-hourly SBF files
"""
amc.cBaseName = colored(os.path.basename(__file__), 'yellow')
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
# treat command line options
dirSTF, fileSTF, GNSSsyst, crdMarker, logLevels = treatCmdOpts(argv)
# create logging for better debugging
logger = amc.createLoggers(os.path.basename(__file__), dir=dirSTF, logLevels=logLevels)
# check if arguments are accepted
workDir = checkExistenceArgs(stfDir=dirSTF, stfFile=fileSTF, logger=logger)
# create dictionary with the current info
global dSTF
dSTF = {}
dSTF['dir'] = workDir
dSTF['gnss'] = GNSSsyst
dSTF['stf'] = fileSTF
# set the reference point
dMarker = {}
dMarker['lat'], dMarker['lon'], dMarker['ellH'] = map(float, crdMarker)
if [dMarker['lat'], dMarker['lon'], dMarker['ellH']] == [0, 0, 0]:
dMarker['lat'] = dMarker['lon'] = dMarker['ellH'] = np.NaN
dMarker['UTM.E'] = dMarker['UTM.N'] = np.NaN
dMarker['UTM.Z'] = dMarker['UTM.L'] = ''
else:
dMarker['UTM.E'], dMarker['UTM.N'], dMarker['UTM.Z'], dMarker['UTM.L'] = UTM.from_latlon(dMarker['lat'], dMarker['lon'])
logger.info('{func:s}: marker coordinates = {crd!s}'.format(func=cFuncName, crd=dMarker))
dSTF['marker'] = dMarker
# # add jammer location coordinates
# dMarker = {}
# dMarker['geod'] = {}
# dMarker['geod']['lat'] = 51.19306 # 51.193183
# dMarker['geod']['lon'] = 4.15528 # 4.155056
# dMarker['UTM'] = {}
# dMarker['UTM.E'], dMarker['UTM.N'], dMarker['UTM.Z'], dMarker['UTM.L'] = utm.from_latlon(dMarker['geod']['lat'], dMarker['geod']['lon'])
# dSTF['marker'] = dMarker
# read in the STF file using included header information
dfGeod = readSTFGeodetic(stfFile=fileSTF, logger=logger)
amutils.logHeadTailDataFrame(df=dfGeod, dfName=dSTF['stf'], callerName=cFuncName, logger=logger)
# save to cvs file
dSTF['csv'] = os.path.splitext(dSTF['stf'])[0] + '.csv'
dfGeod.to_csv(dSTF['csv'])
# plot trajectory
logger.info('{func:s}: information:\n{dict!s}'.format(dict=amutils.pretty(dSTF), func=cFuncName))
plotcoords.plotUTMSuppressed(dStf=dSTF, dfCrd=dfGeod[['time', 'UTM.E', 'UTM.N', 'Error']], logger=logger)
# plot the UTM coordinates and #SVs
plotcoords.plotUTMCoords(dStf=dSTF, dfCrd=dfGeod[['time', 'UTM.E', 'UTM.N', 'Height[m]', 'NrSV', 'SignalInfo', 'dist', '2D/3D']], logger=logger)
# plot trajectory
plotcoords.plotUTMScatter(dStf=dSTF, dfCrd=dfGeod[['time', 'UTM.E', 'UTM.N', 'SignalInfo', '2D/3D']], logger=logger)
logger.info('{func:s}: information:\n{dict!s}'.format(dict=amutils.pretty(dSTF), func=cFuncName))
def main(argv):
"""
creates a combined SBF file from hourly or six-hourly SBF files
"""
amc.cBaseName = colored(os.path.basename(__file__), 'yellow')
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
# treat command line options
dirCSV, filesCSV, GNSSsyst, GNSSsignals, movAvg, logLevels = treatCmdOpts(
argv)
# create logging for better debugging
logger = amc.createLoggers(os.path.basename(__file__),
dir=dirCSV,
logLevels=logLevels)
# check if arguments are accepted
workDir = checkExistenceArgs(cvsDir=dirCSV,
csvFiles=filesCSV,
logger=logger)
# create dictionary with the current info
global dCSV
dCSV = {}
dCSV['dir'] = workDir
dCSV['gnss'] = GNSSsyst
for i, (signal, csv) in enumerate(zip(GNSSsignals, filesCSV)):
dCSV[i] = {'signal': signal, 'file': csv}
dCSV['movavg'] = movAvg
logger.info('{func:s}: information:\n{dict!s}'.format(dict=dCSV,
func=cFuncName))
# read and merge into a single dataframe
dfObsMerged = mergeSignals(csvFiles=filesCSV, logger=logger)
# create signalwise difference
dfObsMerged = signalDifference(dfObs=dfObsMerged, logger=logger)
amutils.logHeadTailDataFrame(df=dfObsMerged,
dfName='dfObsMerged',
callerName=cFuncName,
logger=logger)
# find max/min values for signals and for difference over all PRNs
dCSV['dMax'] = amutils.divround((dfObsMerged[dCSV['SVs']].max()).max(), 5,
2.5)
dCSV['dMin'] = amutils.divround((dfObsMerged[dCSV['SVs']].min()).min(), 5,
2.5)
for i in [0, 1]:
stCols = dCSV[i]['SVs'] + '-{st:s}'.format(st=dCSV[i]['signal'])
dCSV[i]['max'] = amutils.divround(dfObsMerged[stCols].max().max(), 5,
2.5)
dCSV[i]['min'] = amutils.divround(dfObsMerged[stCols].min().min(), 5,
2.5)
logger.info('{func:s}: information:\n{dict!s}'.format(dict=dCSV,
func=cFuncName))
# create plots per prn
signalDiffPlot.plotSignalDiff(dCsv=dCSV, dfSig=dfObsMerged, logger=logger)
def plotAGC(dStf: dict, dfAgc: pd.DataFrame, logger=logging.Logger):
"""
plots the UTM coordinates and #SVs on 4 different plots as a function of time
"""
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: start plotting AGC values'.format(func=cFuncName))
amutils.logHeadTailDataFrame(df=dfAgc, dfName='dfAgc', callerName=cFuncName, logger=logger)
# specify the style
mpl.style.use('seaborn')
colors = ['tab:green', 'tab:olive', 'tab:orange', 'tab:cyan', 'tab:blue', 'tab:red', 'tab:pink', 'tab:purple', 'tab:brown', 'tab:white']
# (re)set the color iterator
# colorsIter = iter(list(mcolors.TABLEAU_COLORS))
colorsIter = iter(list(colors))
fig, ax = plt.subplots(nrows=1, ncols=1, sharex=True)
fig.set_size_inches(14, 10)
# for setting the time on time-scale
dtFormat = plot_utils.determine_datetime_ticks(startDT=dfAgc['time'].iloc[0], endDT=dfAgc['time'].iloc[-1])
# x-axis properties
ax.set_xlim([dfAgc['time'].iloc[0], dfAgc['time'].iloc[-1]])
if dtFormat['minutes']:
ax.xaxis.set_major_locator(dates.MinuteLocator(byminute=[0, 15, 30, 45], interval=1))
else:
ax.xaxis.set_major_locator(dates.HourLocator(interval=dtFormat['hourInterval'])) # every 4 hours
ax.xaxis.set_major_formatter(dates.DateFormatter('%H:%M')) # hours and minutes
ax.xaxis.set_minor_locator(dates.DayLocator(interval=1)) # every day
ax.xaxis.set_minor_formatter(dates.DateFormatter('\n%d-%m-%Y'))
ax.xaxis.set_tick_params(rotation=0)
for tick in ax.xaxis.get_major_ticks():
# tick.tick1line.set_markersize(0)
# tick.tick2line.set_markersize(0)
tick.label1.set_horizontalalignment('center')
# get the index for the different frontends
dIdx = {} # dict with indices
for i, fe in enumerate(dStf['frontend']):
logger.info('{func:s}: ... plotting frontend[{nr:d}], SSNID = {ssnid:d}, name = {name:s}'.format(nr=i, ssnid=fe, name=dStf['frontend'][fe]['name'], func=cFuncName))
idx = dfAgc.index[dfAgc['FrontEnd'] == fe]
logger.info('{func:s}: ... indices found {idx!s} (#{len:d})'.format(idx=idx, len=len(idx), func=cFuncName))
# plot the AGC for this frontend
ax.plot(dfAgc['time'].loc[idx], dfAgc['AGCGain[dB]'].loc[idx], color=next(colorsIter), linestyle='', marker='.', label=dStf['frontend'][fe]['name'], markersize=3)
# name y-axis
ax.set_ylabel('AGC Gain [dB]', fontsize=14)
# add a legend the plot showing 2D/3D positioning displayed
ax.legend(loc='best', ncol=16, markerscale=6)
# title of plot
title = '{syst:s}: AGC Gain [dB]'.format(syst=dStf['gnss'])
fig.suptitle(title, fontsize=16)
# copyright this
ax.annotate(r'$\copyright$ Alain Muls ([email protected])', xy=(1, 0), xycoords='axes fraction', xytext=(0, -45), textcoords='offset pixels', horizontalalignment='right', verticalalignment='bottom', weight='strong', fontsize='medium')
# Save the file in dir png
pltDir = os.path.join(dStf['dir'], 'png')
os.makedirs(pltDir, exist_ok=True)
pltName = '{syst:s}-AGC.png'.format(syst=dStf['gnss'].replace(' ', '-'))
pltName = os.path.join(pltDir, pltName)
fig.savefig(pltName, dpi=100)
logger.info('{func:s}: plot saved as {name:s}'.format(name=pltName, func=cFuncName))
plt.show(block=True)
def plot_elev_distribution(dRtk: dict,
df: pd.DataFrame,
ds: pd.Series,
obs_name: str,
logger: logging.Logger,
showplot: bool = False):
"""
plot_elev_distribution plots the distribution of CN0 or PRres as function of elevation bins
"""
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: creating {obs:s} distribution plot'.format(
obs=obs_name, func=cFuncName))
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=df,
dfName=obs_name)
# set up the plot
plt.style.use('ggplot')
# possible GNSS systems in df
gnss_names = ('GAL', 'GPS')
colors = ('blue', 'red')
dgnss_avail = {}
col_width = 0.25
for gnss_name in gnss_names:
dgnss_avail[gnss_name] = any(
[True for col in df if col.startswith(gnss_name)])
nrCols = 3
col_move = 0
if all(dgnss_avail.values()):
tmpValue = divmod(len(df.columns) / 2, nrCols)
col_move = 0.125
else:
tmpValue = divmod(len(df.columns), nrCols)
syst_names = ' + '.join([k for k, v in dgnss_avail.items() if v == True])
if (tmpValue[1] == 0):
nrRows = int(tmpValue[0])
else:
nrRows = int(tmpValue[0]) + 1
# get the elevation bins used
elev_bins = list(set([col[3:] for col in df.columns]))
elev_bins.sort()
logger.info('{func:s}: elevation bins {bins!s}'.format(bins=elev_bins,
func=cFuncName))
logger.info('{func:s}: elevation bins sorted {bins!s}'.format(
bins=elev_bins.sort(), func=cFuncName))
fig, ax = plt.subplots(nrows=nrRows,
ncols=nrCols,
sharex=True,
sharey=True,
figsize=(20.0, 12.0))
fig.suptitle('{syst:s} - {posf:s} - {date:s}: {obs:s} Statistics'.format(
posf=dRtk['info']['rtkPosFile'],
syst=syst_names,
date=dRtk['Time']['date'],
obs=obs_name),
fontsize='xx-large')
for i, elev_bin, gnss, color in zip((-1, +1), elev_bins, dgnss_avail,
colors):
# plot if the gnss is avaiable
if dgnss_avail[gnss]:
logger.info('{func:s}: plotting for system {gnss:s}'.format(
gnss=gnss, func=cFuncName))
# the indexes on the x-axis
ind = np.arange(len(df.index))
logger.info('{func:s}: ind = {ind!s}'.format(ind=ind,
func=cFuncName))
# columns of this system
gnss_cols = [
'{gnss:s}{bin:s}'.format(gnss=gnss, bin=elev_bin)
for elev_bin in elev_bins
]
# calculate the total number of observations per system
obs_per_bin = df.loc[:, gnss_cols].sum()
logger.info('{func:s}: obs_per_bin = {nrobs!s}'.format(
nrobs=obs_per_bin, func=cFuncName))
if obs_name == 'PRres':
# get index numbers for PRres between -2 and +2
tmpValue = divmod(df.shape[0], 2)
if tmpValue[1] == 0:
mid_prres = tmpValue[0] - 0.5
else:
mid_prres = tmpValue
for axis, col in zip(ax.flat, gnss_cols):
# create a filled area for domain [-1, 1] if PRres plot
if obs_name == 'PRres':
axis.axvspan(mid_prres - 2,
mid_prres + 2,
alpha=0.1,
color='green')
# draw a bar plot
axis.bar(ind + (i * col_move),
df[col] / obs_per_bin.sum() * 100,
alpha=0.5,
color=color,
edgecolor='none')
# rotate the ticks on this axis
idx = np.asarray([i for i in range(len(df.index))])
axis.set_xticks(idx)
axis.set_xticklabels(df.index.tolist(), rotation='vertical')
axis.annotate('# = {:.0f} ({:.2f}%)'.format(
ds[col], ds[col] / ds.sum() * 100),
xy=(1, 1),
xycoords='axes fraction',
xytext=(0, -25),
textcoords='offset pixels',
horizontalalignment='right',
verticalalignment='bottom',
weight='strong',
fontsize='large')
# set the title for sub-plot
axis.set_title(
label='Elevation bin {bin:s}'.format(bin=col[3:]),
fontsize='x-large')
# set the title for the Y axis
axis.set_ylabel(
'{obs:s} statistics in [%]'.format(obs=obs_name))
# save the plot in subdir png of GNSSSystem
amutils.mkdir_p(os.path.join(dRtk['info']['dir'], 'png'))
pngName = os.path.join(
dRtk['info']['dir'], 'png',
os.path.splitext(dRtk['info']['rtkPosFile'])[0] +
'-{syst:s}-{obs:s}-dist.png'.format(syst=syst_names.replace(" ", ""),
obs=obs_name))
fig.savefig(pngName, dpi=fig.dpi)
if showplot:
plt.show(block=True)
else:
plt.close(fig)
def calcDOPs(dfSats: pd.DataFrame, logger: logging.Logger) -> pd.DataFrame:
"""
calculates the number of SVs used and corresponding DOP values
"""
cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored(sys._getframe().f_code.co_name, 'green')
logger.info('{func:s}: calculating number of SVs in PVT and DOP values'.format(func=cFuncName))
# calculate sin/cos of elevation/azimuth
dfSats['sinEl'] = np.sin(np.deg2rad(dfSats.Elev))
dfSats['cosEl'] = np.cos(np.deg2rad(dfSats.Elev))
dfSats['sinAz'] = np.sin(np.deg2rad(dfSats.Azim))
dfSats['cosAz'] = np.cos(np.deg2rad(dfSats.Azim))
# calculate the direction cosines for each satellite
dfSats['alpha'] = dfSats['cosEl'] * dfSats['sinAz']
dfSats['beta'] = dfSats['cosEl'] * dfSats['cosAz']
dfSats['gamma'] = dfSats['sinEl']
amc.logDataframeInfo(df=dfSats, dfName='dfSats', callerName=cFuncName, logger=logger)
# get count of SVs
dfSVCount = countSVs(dfSVs=dfSats, logger=logger)
amc.logDataframeInfo(df=dfSVCount, dfName='dfSVCount', callerName=cFuncName, logger=logger)
# calculating DOP is time consumig, so thin down the TOWs
naTOWs4DOP = getTOWs4DOP(dfNrSVs=dfSVCount, logger=logger)
logger.debug('{func:s} TOWs for calculating DOPs = {array!s}'.format(func=cFuncName, array=naTOWs4DOP))
# create a dataframe for DOP values containing the DateTime column (unique values)
dfDOPs = pd.DataFrame(naTOWs4DOP, columns=['DT'])
amc.logDataframeInfo(df=dfDOPs, dfName='dfDOPs start', callerName=cFuncName, logger=logger)
# select the #SVs from dfSVCount for the intervals we use for DOP calculation
# amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfSVCount, dfName='dfSVCount')
dfNrSVs4DOP = dfSVCount.loc[dfSVCount['DT'].isin(naTOWs4DOP)]
dfNrSVs4DOP.reset_index(inplace=True)
amc.logDataframeInfo(df=dfNrSVs4DOP, dfName='dfNrSVs4DOP', callerName=cFuncName, logger=logger)
# merge last column with #SVs into dfDops
dfDOPs.loc[:, '#SVs'] = dfNrSVs4DOP['#SVs']
# add NA columns for xDOP values
dfDOPs = dfDOPs.reindex(columns=dfDOPs.columns.tolist() + ['HDOP', 'VDOP', 'PDOP', 'GDOP'])
# amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfDOPs, dfName='dfDOPs')
# iterate over all unique TOWs to determine corresponding xDOP values
logger.info('{func:s}: calculating xDOP values for {epochs:d} epochs'.format(func=cFuncName, epochs=len(naTOWs4DOP)))
for i, DT in enumerate(naTOWs4DOP):
# get the index for each DT we have so that we can select the direction cosines associated
# DT = '2019-04-10 00:00:00'
# dt = DT.strptime('%Y-%m-%d %H:%M:%S')
# print('DT = {!s} {!s}'.format(DT, type(DT)))
# print('np.datetime64(DT) = {!s} {!s}'.format(np.datetime64(DT), type(np.datetime64(DT))))
# print('dfSats[DT].iloc[0] = {!s} {!s}'.format(dfSats['DT'].iloc[0], type(dfSats['DT'].iloc[0])))
towIndices = dfSats.index[dfSats['DT'] == np.datetime64(DT)].tolist()
# print('towIndices = {!s}'.format(towIndices))
# create matrix with the direction cosines
dfTOW = dfSats[['alpha', 'beta', 'gamma']].iloc[towIndices]
dfTOW['delta'] = 1.
A = dfTOW.to_numpy()
# print('dfTOW = {!s}'.format(dfTOW))
# print('dfTOW = {!s}'.format(type(dfTOW)))
# invert ATA and retain the diagonal squared
ATAinvDiag = np.linalg.inv(A.transpose().dot(A)).diagonal()
sqDiag = np.square(ATAinvDiag)
# print('ATAinvDiag = \n{!s} \n{!s}'.format(ATAinvDiag, type(ATAinvDiag)))
# print('sqDiag = \n{!s} \n{!s}'.format(sqDiag, type(sqDiag)))
# get the index for this DT into the dfDOPs
indexTOW = dfDOPs.index[dfDOPs['DT'] == DT].tolist()[0]
# print('index DT = {!s}'.format(indexTOW))
# calculate the xDOP values and store them in the dfDOPs
PDOP = np.sqrt(sqDiag[0] + sqDiag[1] + sqDiag[2])
dfDOPs.HDOP.iloc[indexTOW] = np.sqrt(sqDiag[0] + sqDiag[1])
dfDOPs.VDOP.iloc[indexTOW] = ATAinvDiag[2]
dfDOPs.PDOP.iloc[indexTOW] = PDOP
dfDOPs.GDOP.iloc[indexTOW] = np.sqrt(sqDiag[0] + sqDiag[1] + sqDiag[2] + sqDiag[3])
# print('dfDOPS.iloc[indexTOW] = {!s}'.format(dfDOPs.iloc[indexTOW]))
# show progress bar
progbar(i, len(naTOWs4DOP), 60)
print() # empty print statement for ending progbar
# drop the cos/sin & direction cosines columns from dfSats
dfSats.drop(['sinEl', 'cosEl', 'sinAz', 'cosAz', 'alpha', 'beta', 'gamma'], axis=1, inplace=True)
amc.logDataframeInfo(df=dfDOPs, dfName='dfDOPs (end)', callerName=cFuncName, logger=logger)
amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=dfDOPs, dfName='dfDOPs')
return dfDOPs
def parsePosFile(logger: logging.Logger) -> pd.DataFrame:
"""
parses 'posn' file created by pyrtklib.py
"""
# set current function name
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
posFilePath = os.path.join(amc.dRTK['posDir'], amc.dRTK['posFile'])
logger.info('{func:s} parsing rtk-pos file {posf:s}'.format(
func=cFuncName, posf=posFilePath))
# looking for start times of observation file
for line in open(posFilePath):
rec = line.strip()
if rec.startswith('% obs start'):
amc.dRTK['obsStart'] = datetime.strptime(rec[14:33],
'%Y/%m/%d %H:%M:%S')
break
# looking for end times of observation file
for line in open(posFilePath):
rec = line.strip()
if rec.startswith('% obs end'):
amc.dRTK['obsEnd'] = datetime.strptime(rec[14:33],
'%Y/%m/%d %H:%M:%S')
break
# looking for ref pos of observation file
foundRefPos = False
for line in open(posFilePath):
rec = line.strip()
if rec.startswith('% ref pos'):
amc.dRTK['RefPos'] = [float(x) for x in rec.split(':')[1].split()]
amc.dRTK['RefPosUTM'] = utm.from_latlon(amc.dRTK['RefPos'][0],
amc.dRTK['RefPos'][1])
logger.info(
'{func:s}: reference station coordinates are LLH={llh!s} UTM={utm!s}'
.format(func=cFuncName,
llh=amc.dRTK['RefPos'],
utm=amc.dRTK['RefPosUTM']))
foundRefPos = True
break
if not foundRefPos:
amc.dRTK['RefPos'] = [np.NaN, np.NaN, np.NaN]
amc.dRTK['RefPosUTM'] = (np.NaN, np.NaN, np.NaN, np.NaN)
logger.info(
'{func:s}: no reference station used'.format(func=cFuncName))
# find start of results in rtk file
endHeaderLine = amutils.line_num_for_phrase_in_file('% GPST', posFilePath)
dfPos = pd.read_csv(posFilePath,
header=endHeaderLine,
delim_whitespace=True)
dfPos = dfPos.rename(
columns={
'%': 'WNC',
'GPST': 'TOW',
'latitude(deg)': 'lat',
'longitude(deg)': 'lon',
'height(m)': 'ellH',
'sdn(m)': 'sdn',
'sde(m)': 'sde',
'sdu(m)': 'sdu',
'sdne(m)': 'sdne',
'sdeu(m)': 'sdeu',
'sdun(m)': 'sdun',
'age(s)': 'age'
})
# check if we have records for this mode in the data, else exit
if dfPos.shape[0] == 0:
logger.info('{func:s}: found no data in pos-file {pos:s}'.format(
func=cFuncName, pos=amc.dRTK['posFile']))
sys.exit(amc.E_FAILURE)
# store total number of observations
amc.dRTK['#obs'] = dfPos.shape[0]
# store number of calculated positions for requested rtk quality
amc.dRTK['#obsQual'] = len(dfPos.loc[dfPos['Q'] == amc.dRTK['iQual']])
logger.info('{func:s}: amc.dRTK = \n{drtk!s}'.format(func=cFuncName,
drtk=amc.dRTK))
# convert the time in seconds
dfPos['DT'] = dfPos.apply(lambda x: gpstime.UTCFromWT(x['WNC'], x['TOW']),
axis=1)
# add UTM coordinates
dfPos['UTM.E'], dfPos['UTM.N'], dfPos['UTM.Z'], dfPos[
'UTM.L'] = utm.from_latlon(dfPos['lat'].to_numpy(),
dfPos['lon'].to_numpy())
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=dfPos,
dfName='{posf:s}'.format(posf=posFilePath))
amc.logDataframeInfo(df=dfPos,
dfName='dfPos',
callerName=cFuncName,
logger=logger)
return dfPos
def main(argv):
"""
pyRTKPlot adds UTM coordinates to output of rnx2rtkp.
If 'stat' file is available, calculates xDOP values, and make plots of statictics
"""
amc.cBaseName = colored(os.path.basename(__file__), 'yellow')
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
# some options for diasplay of dataframes
pd.set_option('display.max_columns', None) # or 1000
pd.set_option('display.max_rows', None) # or 1000
pd.set_option('display.max_colwidth', -1) # or 199
# limit float precision
json.encoder.FLOAT_REPR = lambda o: format(o, '.3f')
np.set_printoptions(precision=4)
# treat command line options
rtkPosFile, rtkDir, crdMarker, showPlots, overwrite, logLevels = treatCmdOpts(
argv)
# create logging for better debugging
logger, log_name = amc.createLoggers(baseName=os.path.basename(__file__),
dir=rtkDir,
logLevels=logLevels)
# change to selected directory if exists
# print('rtkDir = %s' % rtkDir)
if not os.path.exists(rtkDir):
logger.error('{func:s}: directory {dir:s} does not exists'.format(
func=cFuncName, dir=colored(rtkDir, 'red')))
sys.exit(amc.E_DIR_NOT_EXIST)
else:
os.chdir(rtkDir)
logger.info('{func:s}: changed to dir {dir:s}'.format(func=cFuncName,
dir=colored(
rtkDir,
'green')))
# store information
dInfo = {}
dInfo['dir'] = rtkDir
dInfo['rtkPosFile'] = rtkPosFile
dInfo['rtkStatFile'] = dInfo['rtkPosFile'] + '.stat'
dInfo['posn'] = dInfo['rtkPosFile'] + '.posn'
dInfo['posnstat'] = dInfo['posn'] + '.html'
amc.dRTK['info'] = dInfo
# GNSS system is last part of root directory
amc.dRTK['syst'] = 'UNKNOWN'
for _, syst in enumerate(['GAL', 'GPS', 'COM']):
if syst.lower() in amc.dRTK['info']['dir'].lower():
amc.dRTK['syst'] = syst
# print('amc.dRTK['syst'] = {:s}'.format(amc.dRTK['syst']))
# info about PDOP bins and statistics
dPDOP = {}
dPDOP['bins'] = [0, 2, 3, 4, 5, 6, math.inf]
amc.dRTK['PDOP'] = dPDOP
# set the reference point
dMarker = {}
dMarker['lat'], dMarker['lon'], dMarker['ellH'] = map(float, crdMarker)
print('crdMarker = {!s}'.format(crdMarker))
if [dMarker['lat'], dMarker['lon'], dMarker['ellH']] == [0, 0, 0]:
dMarker['lat'] = dMarker['lon'] = dMarker['ellH'] = np.NaN
dMarker['UTM.E'] = dMarker['UTM.N'] = np.NaN
dMarker['UTM.Z'] = dMarker['UTM.L'] = ''
else:
dMarker['UTM.E'], dMarker['UTM.N'], dMarker['UTM.Z'], dMarker[
'UTM.L'] = utm.from_latlon(dMarker['lat'], dMarker['lon'])
logger.info('{func:s}: marker coordinates = {crd!s}'.format(func=cFuncName,
crd=dMarker))
amc.dRTK['marker'] = dMarker
# check wether pos and stat file are present, else exit
if not os.access(os.path.join(rtkDir, amc.dRTK['info']['rtkPosFile']),
os.R_OK) or not os.access(
os.path.join(rtkDir, amc.dRTK['info']['rtkStatFile']),
os.R_OK):
logger.error(
'{func:s}: file {pos:s} or {stat:s} is not accessible'.format(
func=cFuncName,
pos=os.path.join(rtkDir, amc.dRTK['info']['rtkPosFile']),
stat=os.path.join(rtkDir, amc.dRTK['info']['rtkStatFile'])))
sys.exit(amc.E_FILE_NOT_EXIST)
# read the position file into a dataframe and add dUTM coordinates
logger.info('{func:s}: parsing RTKLib pos file {pos:s}'.format(
pos=amc.dRTK['info']['rtkPosFile'], func=cFuncName))
dfPosn = parse_rtk_files.parseRTKLibPositionFile(logger=logger)
# calculate the weighted avergae of llh & enu
amc.dRTK['WAvg'] = parse_rtk_files.weightedAverage(dfPos=dfPosn,
logger=logger)
# find difference with reference and ax/min limits for UTM plot
logger.info(
'{func:s}: calculating coordinate difference with reference/mean position'
.format(func=cFuncName))
dfCrd, dCrdLim = plot_position.crdDiff(
dMarker=amc.dRTK['marker'],
dfUTMh=dfPosn[['UTM.E', 'UTM.N', 'ellH']],
plotCrds=['UTM.E', 'UTM.N', 'ellH'],
logger=logger)
# merge dfCrd into dfPosn
dfPosn[['dUTM.E', 'dUTM.N', 'dEllH']] = dfCrd[['UTM.E', 'UTM.N', 'ellH']]
# work on the statistics file
# split it in relavant parts
dTmpFiles = parse_rtk_files.splitStatusFile(
amc.dRTK['info']['rtkStatFile'], logger=logger)
# parse the satellite file (contains Az, El, PRRes, CN0)
dfSats = parse_rtk_files.parseSatelliteStatistics(dTmpFiles['sat'],
logger=logger)
store_to_cvs(df=dfSats, ext='sats', dInfo=amc.dRTK, logger=logger)
# determine statistics on PR residuals for all satellites per elevation bin
dfDistCN0, dsDistCN0, dfDistPRres, dsDistPRRes = parse_rtk_files.parse_elevation_distribution(
dRtk=amc.dRTK, dfSat=dfSats, logger=logger)
store_to_cvs(df=dfDistCN0, ext='CN0.dist', dInfo=amc.dRTK, logger=logger)
store_to_cvs(df=dfDistPRres,
ext='PRres.dist',
dInfo=amc.dRTK,
logger=logger)
# BEGIN DEBUG
# END DEBUG
# determine statistics of PR residuals for each satellite
amc.dRTK['PRres'] = parse_rtk_files.parse_sv_residuals(dfSat=dfSats,
logger=logger)
# calculate DOP values from El, Az info for each TOW
dfDOPs = parse_rtk_files.calcDOPs(dfSats, logger=logger)
store_to_cvs(df=dfDOPs, ext='XDOP', dInfo=amc.dRTK, logger=logger)
# merge the PDOP column of dfDOPs into dfPosn and interpolate the PDOP column
dfResults = pd.merge(left=dfPosn,
right=dfDOPs[['DT', 'PDOP', 'HDOP', 'VDOP', 'GDOP']],
left_on='DT',
right_on='DT',
how='left')
dfPosn = dfResults.interpolate()
store_to_cvs(df=dfPosn, ext='posn', dInfo=amc.dRTK, logger=logger)
# calculate per DOP bin the statistics of PDOP
parse_rtk_files.addPDOPStatistics(dRtk=amc.dRTK,
dfPos=dfPosn,
logger=logger)
# add statistics for the E,N,U coordinate differences
dfStatENU = enu_stat.enu_statistics(
dRtk=amc.dRTK,
dfENU=dfPosn[['DT', 'dUTM.E', 'dUTM.N', 'dEllH']],
logger=logger)
# add statistics for the E,N,U coordinate differences
dfDistENU, dfDistXDOP = enu_stat.enupdop_distribution(dRtk=amc.dRTK,
dfENU=dfPosn[[
'DT', 'dUTM.E',
'dUTM.N',
'dEllH', 'PDOP',
'HDOP', 'VDOP',
'GDOP'
]],
logger=logger)
store_to_cvs(df=dfDistENU, ext='ENU.dist', dInfo=amc.dRTK, logger=logger)
store_to_cvs(df=dfDistXDOP, ext='XDOP.dist', dInfo=amc.dRTK, logger=logger)
logger.info('{func:s}: dRTK =\n{settings!s}'.format(func=cFuncName,
settings=json.dumps(
amc.dRTK,
sort_keys=False,
indent=4)))
# # store statistics for dfPosn
# logger.info('{func:s}: creating pandas profile report {ppname:s} for dfPosn, {help:s}'.format(ppname=colored(amc.dRTK['info']['posnstat'], 'green'), help=colored('be patient', 'red'), func=cFuncName))
# dfProfile = dfPosn[['DT', 'ns', 'dUTM.E', 'dUTM.N', 'dEllH', 'sdn', 'sde', 'sdu', 'PDOP']]
# ppTitle = 'Report on {posn:s} - {syst:s} - {date:s}'.format(posn=amc.dRTK['info']['posn'], syst=amc.dRTK['syst'], date=amc.dRTK['Time']['date'])
# profile = pp.ProfileReport(df=dfProfile, check_correlation_pearson=False, correlations={'pearson': False, 'spearman': False, 'kendall': False, 'phi_k': False, 'cramers': False, 'recoded': False}, title=ppTitle)
# profile.to_file(output_file=amc.dRTK['info']['posnstat'])
# parse the clock stats
dfCLKs = parse_rtk_files.parseClockBias(dTmpFiles['clk'], logger=logger)
store_to_cvs(df=dfCLKs, ext='clks', dInfo=amc.dRTK, logger=logger)
# BEGIN debug
dfs = (dfPosn, dfSats, dfCLKs, dfCrd, dfDOPs, dfStatENU, dfDistENU,
dfDistXDOP, dfDistPRres, dfDistCN0)
dfsNames = ('dfPosn', 'dfSats', 'dfCLKs', 'dfCrd', 'dfDOPs', 'dfStatENU',
'dfDistENU', 'dfDistXDOP')
for df, dfName in zip(dfs, dfsNames):
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=df,
dfName=dfName)
amc.logDataframeInfo(df=df,
dfName=dfName,
callerName=cFuncName,
logger=logger)
# EOF debug
# create the position plot (use DOP to color segments)
plot_position.plotUTMOffset(dRtk=amc.dRTK,
dfPos=dfPosn,
dfCrd=dfCrd,
dCrdLim=dCrdLim,
logger=logger,
showplot=showPlots)
# create the UTM N-E scatter plot
plot_scatter.plotUTMScatter(dRtk=amc.dRTK,
dfPos=dfPosn,
dfCrd=dfCrd,
dCrdLim=dCrdLim,
logger=logger,
showplot=showPlots)
plot_scatter.plotUTMScatterBin(dRtk=amc.dRTK,
dfPos=dfPosn,
dfCrd=dfCrd,
dCrdLim=dCrdLim,
logger=logger,
showplot=showPlots)
# create ENU distribution plots
plot_distributions_crds.plot_enu_distribution(dRtk=amc.dRTK,
dfENUdist=dfDistENU,
dfENUstat=dfStatENU,
logger=logger,
showplot=showPlots)
# create XDOP plots
plot_distributions_crds.plot_xdop_distribution(dRtk=amc.dRTK,
dfXDOP=dfDOPs,
dfXDOPdisp=dfDistXDOP,
logger=logger,
showplot=showPlots)
# plot pseudo-range residus
dPRResInfo = {
'name': 'PRres',
'yrange': [-6, 6],
'title': 'PR Residuals',
'unit': 'm',
'linestyle': '-'
}
logger.info(
'{func:s}: creating dPRRes plots based on dict {dict!s}'.format(
func=cFuncName, dict=dPRResInfo))
plot_sats_column.plotRTKLibSatsColumn(dCol=dPRResInfo,
dRtk=amc.dRTK,
dfSVs=dfSats,
logger=logger,
showplot=showPlots)
# plot CN0
dCN0Info = {
'name': 'CN0',
'yrange': [20, 60],
'title': 'CN0 Ratio',
'unit': 'dBHz',
'linestyle': '-'
}
logger.info('{func:s}: creating CN0 plots based on dict {dict!s}'.format(
func=cFuncName, dict=dCN0Info))
plot_sats_column.plotRTKLibSatsColumn(dCol=dCN0Info,
dRtk=amc.dRTK,
dfSVs=dfSats,
logger=logger,
showplot=showPlots)
# create plots for elevation distribution of CN0 and PRres
plot_distributions_elev.plot_elev_distribution(dRtk=amc.dRTK,
df=dfDistCN0,
ds=dsDistCN0,
obs_name='CN0',
logger=logger,
showplot=showPlots)
plot_distributions_elev.plot_elev_distribution(dRtk=amc.dRTK,
df=dfDistPRres,
ds=dsDistPRRes,
obs_name='PRres',
logger=logger,
showplot=showPlots)
# # plot elevation
dElevInfo = {
'name': 'Elev',
'yrange': [0, 90],
'title': 'Elevation',
'unit': 'Deg',
'linestyle': '-'
}
logger.info('{func:s}: creating Elev plots based on dict {dict!s}'.format(
func=cFuncName, dict=dElevInfo))
plot_sats_column.plotRTKLibSatsColumn(dCol=dElevInfo,
dRtk=amc.dRTK,
dfSVs=dfSats,
logger=logger,
showplot=showPlots)
# # plot the receiver clock
logger.info('{func:s}: creating Clock plots'.format(func=cFuncName))
plot_clock.plotClock(dfClk=dfCLKs,
dRtk=amc.dRTK,
logger=logger,
showplot=showPlots)
logger.info('{func:s}: final amc.dRTK =\n{settings!s}'.format(
func=cFuncName,
settings=json.dumps(amc.dRTK, sort_keys=False, indent=4)))
jsonName = amc.dRTK['info']['rtkPosFile'] + '.json'
with open(jsonName, 'w') as f:
json.dump(amc.dRTK, f, ensure_ascii=False, indent=4)
logger.info('{func:s}: created json file {json:s}'.format(func=cFuncName,
json=colored(
jsonName,
'green')))
# copy temp log file to the YYDOY directory
copyfile(
log_name,
os.path.join(
amc.dRTK['info']['dir'], '{obs:s}-{prog:s}'.format(
obs=amc.dRTK['info']['rtkPosFile'].replace(';', '_'),
prog='plot.log')))
os.remove(log_name)
def addRTKResult(logger: logging.Logger):
"""
adds the result to campaign csv file
"""
# write formatted output of result
cFuncName = amc.cBaseName + ': ' + colored(sys._getframe().f_code.co_name,
'green')
amc.dRTK['csvFile'] = os.path.join(
amc.dRTK['rootDir'],
'{campaign:s}.csv'.format(campaign=amc.dRTK['campaign']))
# if campaign file exists, read in the data
if os.access(amc.dRTK['csvFile'], os.R_OK):
logger.info('{func:s}: reading campaign csv file {csv:s}'.format(
func=cFuncName, csv=amc.dRTK['csvFile']))
# read in the campaign data
dfCampaign = pd.read_csv(amc.dRTK['csvFile'])
else:
# create an empty dataframe
dfCampaign = pd.DataFrame(columns=[
'campaign', 'marker', 'date', 'start', 'end', 'rtkqual',
'#obsTotal', '#obsQual', 'lat', 'lon', 'ellH', 'sdu', 'UTM.E',
'sde', 'UTM.N', 'sdn', 'UTM.Z', 'UTM.L', 'Ref_lat', 'Ref_lon',
'Ref_ellH', 'Ref_UTM.E', 'Ref_UTM.N', 'Ref_UTM.Z', 'Ref_UTM.L',
'Distance', 'DeltaH'
])
# check whether the current processing has been saved to csv file
index = dfCampaign.loc[(dfCampaign['campaign'] == amc.dRTK['campaign'])
& (dfCampaign['marker'] == amc.dRTK['marker']) &
(dfCampaign['rtkqual']
== amc.dRTK['rtkqual'])].index
# if index is not empty => delete dataframe current info
if len(index) > 0:
dfCampaign.drop(index, inplace=True)
# calculate the slant distance to Reference if available
try:
crdWAvg = (amc.dRTK['WAVG']['lat'], amc.dRTK['WAVG']['lon'])
crdRefPt = (amc.dRTK['RefPos'][0], amc.dRTK['RefPos'][1])
distance = geopy.distance.vincenty(crdWAvg, crdRefPt).m
DeltaH = amc.dRTK['WAVG']['ellH'] - amc.dRTK['RefPos'][2]
except ValueError:
distance = np.NaN
DeltaH = np.NaN
# add the new info to the csv file
dfCampaign = dfCampaign.append(pd.Series([
amc.dRTK['campaign'], amc.dRTK['marker'],
amc.dRTK['obsStart'].strftime("%d/%m/%Y"),
amc.dRTK['obsStart'].strftime('%H:%M:%S'),
amc.dRTK['obsEnd'].strftime('%H:%M:%S'), amc.dRTK['rtkqual'],
amc.dRTK['#obs'], amc.dRTK['#obsQual'], amc.dRTK['WAVG']['lat'],
amc.dRTK['WAVG']['lon'], amc.dRTK['WAVG']['ellH'],
amc.dRTK['WAVG']['sdellH'], amc.dRTK['WAVG']['UTM.E'],
amc.dRTK['WAVG']['sdUTM.E'], amc.dRTK['WAVG']['UTM.N'],
amc.dRTK['WAVG']['sdUTM.N'], amc.dRTK['WAVG']['UTM.Z'],
amc.dRTK['WAVG']['UTM.L'], amc.dRTK['RefPos'][0],
amc.dRTK['RefPos'][1], amc.dRTK['RefPos'][2], amc.dRTK['RefPosUTM'][0],
amc.dRTK['RefPosUTM'][1], amc.dRTK['RefPosUTM'][2],
amc.dRTK['RefPosUTM'][3], distance, DeltaH
],
index=dfCampaign.columns),
ignore_index=True)
# sort the dataframe
dfCampaign.sort_values(['campaign', 'date', 'rtkqual', 'marker'],
ascending=[True, True, True, True],
inplace=True)
# format the output of data for importing in excel workbook
# for col in ['ellH', 'UTM.E', 'UTM.N', 'Ref_ellH', 'Ref_UTM.N', 'Ref_UTM.E', 'Distance']:
# dfCampaign[col] = dfCampaign[col].map('{:.3f}'.format)
# for col in ['lat', 'lon', 'Ref_lat', 'Ref_lon']:
# dfCampaign[col] = dfCampaign[col].map('{:.9f}'.format)
# info logging for campaign
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=dfCampaign,
dfName='dfCampaign updated')
# add campaign sheet write to excel workbook
if amc.dRTK['excel']:
df2excel.append_df_to_excel(filename=amc.dRTK['xlsName'],
df=dfCampaign,
sheet_name=amc.dRTK['campaign'],
truncate_sheet=True,
startrow=0,
index=False,
float_format="%.9f")
logger.info(
'{func:s}: added sheet {sheet:s} to workbook {wb:s}'.format(
func=cFuncName,
sheet=amc.dRTK['campaign'],
wb=amc.dRTK['xlsName']))
# save the dataframe by overwriting the amc.dRTK['csvfile']
dfCampaign.to_csv(amc.dRTK['csvFile'], index=None, header=True)
def main(argv):
"""
pyconvbin converts raw data from SBF/UBlox to RINEX
"""
amc.cBaseName = colored(os.path.basename(__file__), 'yellow')
cFuncName = colored(os.path.basename(__file__),
'yellow') + ' - ' + colored(
sys._getframe().f_code.co_name, 'green')
# treat command line options
rnx_dir, gnss, cutoff, multiplier, showPlots, logLevels = treatCmdOpts(
argv)
# create logging for better debugging
logger, log_name = amc.createLoggers(os.path.basename(__file__),
dir=rnx_dir,
logLevels=logLevels)
logger.info('{func:s}: arguments processed: {args!s}'.format(
args=rnx_dir, func=cFuncName))
# check validity of passed arguments
retCode = checkValidityArgs(dir_rnx=rnx_dir, logger=logger)
if retCode != amc.E_SUCCESS:
logger.error('{func:s}: Program exits with code {error:s}'.format(
error=colored('{!s}'.format(retCode), 'red'), func=cFuncName))
sys.exit(retCode)
# store parameters
amc.dRTK = {}
# get the information from pyconvbin created json file
read_json(dir_rnx=rnx_dir, logger=logger)
# load the requested OBSTAB file into a pandas dataframe
df_obs = rnxobs_tabular.read_obs_tabular(gnss=gnss, logger=logger)
df_obs['gap'] = np.nan
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=df_obs,
dfName='df_obs')
# get unique list of PRNs in dataframe
prn_lst = sorted(df_obs['PRN'].unique())
logger.info('{func:s}: observed PRNs are {prns!s} (#{total:d})'.format(
prns=prn_lst, total=len(prn_lst), func=cFuncName))
logger.info(
'{func:s}; getting corresponding NORAD info'.format(func=cFuncName))
# read the files galileo-NORAD-PRN.t and gps-ops-NORAD-PRN.t
dfNORAD = tle_parser.read_norad2prn(logger=logger)
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=dfNORAD,
dfName='dfNORAD')
# get the corresponding NORAD nrs for the given PRNs
dNORADs = tle_parser.get_norad_numbers(prns=prn_lst,
dfNorad=dfNORAD,
logger=logger)
logger.info('{func:s}: corresponding NORAD nrs (#{count:d}):'.format(
count=len(dNORADs), func=cFuncName))
# load a time scale and set RMA as Topo
# loader = sf.Loader(dir_tle, expire=True) # loads the needed data files into the tle dir
ts = sf.load.timescale()
RMA = sf.Topos('50.8438 N', '4.3928 E')
logger.info('{func:s}: Earth station RMA @ {topo!s}'.format(
topo=colored(RMA, 'green'), func=cFuncName))
# get the datetime that corresponds to yydoy
date_yydoy = datetime.strptime(amc.dRTK['rnx']['times']['DT'],
'%Y-%m-%d %H:%M:%S')
yydoy = date_yydoy.strftime('%y%j')
logger.info(
'{func:s}: calculating rise / set times for {date:s} ({yydoy:s})'.
format(date=colored(date_yydoy.strftime('%d-%m-%Y'), 'green'),
yydoy=yydoy,
func=cFuncName))
t0 = ts.utc(int(date_yydoy.strftime('%Y')), int(date_yydoy.strftime('%m')),
int(date_yydoy.strftime('%d')))
date_tomorrow = date_yydoy + timedelta(days=1)
t1 = ts.utc(int(date_tomorrow.strftime('%Y')),
int(date_tomorrow.strftime('%m')),
int(date_tomorrow.strftime('%d')))
# find corresponding TLE record for NORAD nrs
df_tles = tle_parser.find_norad_tle_yydoy(dNorads=dNORADs,
yydoy=yydoy,
logger=logger)
# list of rise / set times by observation / TLEs
lst_obs_rise = []
# find in observations and by TLEs what the riuse/set times are and number of observations
for prn in prn_lst:
# find rise & set times for each SV and store into list dt_obs_rise_set and dt_obs_set
nom_interval, dt_obs_rise, dt_obs_set, obs_arc_count = rnxobs_tabular.rise_set_times(
prn=prn, df_obstab=df_obs, nomint_multi=multiplier, logger=logger)
# find rise:set times using TLEs
dt_tle_rise, dt_tle_set, dt_tle_cul, tle_arc_count = tle_parser.tle_rise_set_times(
prn=prn,
df_tle=df_tles,
marker=RMA,
t0=t0,
t1=t1,
elev_min=cutoff,
obs_int=nom_interval,
logger=logger)
# add to list for creating dataframe
lst_obs_rise.append([
dt_obs_rise, dt_obs_set, obs_arc_count, dt_tle_rise, dt_tle_set,
dt_tle_cul, tle_arc_count
])
# test to import in dataframe
df_rise_set_tmp = pd.DataFrame(lst_obs_rise,
columns=[
'obs_rise', 'obs_set', 'obs_arc_count',
'tle_rise', 'tle_set', 'tle_cul',
'tle_arc_count'
],
index=prn_lst)
# find corresponding arcs between observation and predicted TLE
max_arcs, df_rise_set = rnxobs_tabular.intersect_arcs(
df_rs=df_rise_set_tmp, logger=logger)
# inform user
amutils.logHeadTailDataFrame(logger=logger,
callerName=cFuncName,
df=df_rise_set,
dfName='df_rise_set')
# write to csv file
csvName = os.path.join(amc.dRTK['gfzrnxDir'],
amc.dRTK['rnx']['gnss'][gnss]['marker'],
'rise-set-dt.csv')
df_rise_set.to_csv(csvName, index=None, header=True)
# create a new dataframe that has PRNs as index and the max_arcs columns with number of obs / TLEs
df_obs_arcs = rnxobs_tabular.rearrange_arcs(nr_arcs=max_arcs,
df_rs=df_rise_set,
logger=logger)
# write to csv file
csvName = os.path.join(amc.dRTK['gfzrnxDir'],
amc.dRTK['rnx']['gnss'][gnss]['marker'],
'obs_arcs.csv')
df_obs_arcs.to_csv(csvName, index=None, header=True)
# plot the statistics of observed vs TLE predicted
plot_obstab.plot_rise_set_times(gnss=gnss,
df_rs=df_rise_set,
logger=logger,
showplot=showPlots)
plot_obstab.plot_rise_set_stats(gnss=gnss,
df_arcs=df_obs_arcs,
nr_arcs=max_arcs,
logger=logger,
showplot=showPlots)
# amutils.logHeadTailDataFrame(logger=logger, callerName=cFuncName, df=df_obs[(df_obs['gap'] > 1.) | (df_obs['gap'].isna())], dfName='df_obs', head=50)
# logger.info('{func:s}: amc.dRTK =\n{json!s}'.format(json=json.dumps(amc.dRTK, sort_keys=False, indent=4, default=amutils.DT_convertor), func=cFuncName))
# copy temp log file to the YYDOY directory
copyfile(
log_name,
os.path.join(
os.path.join(amc.dRTK['gfzrnxDir'],
amc.dRTK['rnx']['gnss'][gnss]['marker']),
'pyobstab.log'))
os.remove(log_name)
