def read_eop_C04(file_path_in):
"""
read EOP C04 file
Parameters
----------
file_path_in : str
path of the EOP C04 file.
Returns
-------
DF : DataFrame
out DataFrame.
"""
DF = pd.read_csv(file_path_in,
delim_whitespace=True,
skip_blank_lines=True,
skiprows=13,
header=None)
cols = [
'yyyy', 'mm', 'dd', 'MJD', 'x', 'y', 'UT1-UTC', 'LOD', 'dX', 'dY',
'x_Err', 'y_Err', 'UT1-UTC_Err', 'LOD_Err', 'dX_Err', 'dY_Err'
]
DF.columns = cols
DF["epoch"] = conv.MJD2dt(DF.MJD)
return DF
def stations_in_EPOS_sta_coords_file_mono(coords_file_path):
"""
Gives stations in a EPOS coords. file (YYYY_DDD_sta_coordinates)
Parameters
----------
coords_file_path : str
path of the EPOS coords. file.
Returns
-------
epoch : datetime
the main mean epoch in the EPOS coords. file.
stats_list : list
list of 4 char station list.
"""
SITE_line_list = utils.grep(coords_file_path, " SITE m")
stats_list = []
mean_mjd_list = []
for l in SITE_line_list:
stat = l.split()[8].lower()
stats_list.append(stat)
mean_mjd = np.mean([float(l.split()[6]), float(l.split()[7])])
mean_mjd_list.append(mean_mjd)
mjd_final = utils.most_common(mean_mjd_list)
epoch = conv.MJD2dt(mjd_final)
return epoch, stats_list
def read_combi_sum_full(sum_full_file,
RMS_lines_output=True,
set_PRN_as_index=True):
Vals_stk = []
for l in open(sum_full_file):
F = l.split()
if "|" in F:
F.remove("|")
### Find date line
if "MJD:" in l:
date_line = l
### Skip useless lines
if not "|" in l or "------" in l:
continue
### Find AC list
if "PRN" in l:
ACs_list = F
ACs_list.append("RMS_sat")
ACs_list.append("PRN_str")
ACs_list.append("CONST")
elif F[0].isnumeric():
Fout = [float(f) for f in F]
Fout[0] = int(Fout[0])
#Add the PRN string and the constellation
Fout.append(prn_int_2_prn_str(int(Fout[0])))
Fout.append(Fout[-1][0])
Vals_stk.append(Fout)
elif "RMS" in F[0] and RMS_lines_output:
Fout = [float(f) for f in F[1:]]
Fout.append(np.nan)
Fout.insert(0, F[0])
#Add FAKE the PRN string and the constellation
Fout.append(F[0])
Fout.append(None)
Vals_stk.append(Fout)
DF = pd.DataFrame(Vals_stk, columns=ACs_list)
### Date management
mjd = float(date_line.split("MJD:")[1].split()[0])
date_dt = conv.MJD2dt(mjd)
DF.date_mjd = mjd
DF.date_dt = date_dt
DF.date_gps = utils.join_improved("", conv.dt2gpstime(date_dt))
if set_PRN_as_index:
DF.set_index("PRN_str", inplace=True)
return DF
def read_gfz_trop(trpfile):
"""
This function is reading GFZ troposphere sinex into Pandas DataFrame
Parameters
----------
trpfile : Str
File name of GFZ troposphere sinex.
Returns
-------
DF : Pandas DataFrame
Pandas Dataframe of GFZ troposphere sinex.
"""
fields = []
flagtrop = False
for line in open(trpfile,"r",encoding = "ISO-8859-1"):
if re.compile('TROP/SOLUTION').search(line):
flagtrop = not flagtrop
continue
if flagtrop ==True and line[0] == ' ':
field = line.split()
fields.append(field)
else:
continue
DF = pd.DataFrame(fields)
DF.drop(columns=[0,8,9,10,11,12,18,19,20],inplace=True)
DF.columns = ['STAT','epoc','year','doy','secofday','ztd_est','ztd_est_std','num_sat','tgn_est','tgn_est_std','tge_est','tge_est_std']
cols_numeric = ['epoc','ztd_est','ztd_est_std','num_sat','tgn_est','tgn_est_std','tge_est','tge_est_std']
DF[cols_numeric] = DF[cols_numeric].apply(pd.to_numeric, errors='coerce')
DF['epoc'] = conv.MJD2dt(DF['epoc'].values)
DF['epoc'] = DF['epoc'].dt.floor('H')
return DF
def ESMGFZ_extrapolator(path_or_netcdf_object_in,
time_xtrp,
lat_xtrp,
lon_xtrp,
wished_values=("duV","duNS","duEW"),
output_type = "DataFrame",
debug=False,verbose=True,
time_smart=True,
interp_method="splinef2d"):
"""
Extrapolate loading values from the EMSGFZ models
esmdata.gfz-potsdam.de:8080/
Parameters
----------
path_or_netcdf_object_in : string, list of strings or NetCDF object
Input
can be a file path (string), a list of string (will be concatenated)
or direcly the NetCDF object (faster).
time_xtrp : float or float iterable
time for the wished extrapolated values
for daily files: hours of day [0..23].
for yearly files: day of years [0..364].
lat_xtrp : float or float iterable
latitude component for the wished extrapolated values
ranging from [-90..90]
lon_xtrp : float or float iterable
longitude component for the wished extrapolated values.
ranging from [-180..180]
wished_values : tuple of string, optional
the components of the extrapolated values.
The default is ("duV","duNS","duEW").
output_type : str, optional
Choose the output type.
"DataFrame","dict","array","tuple","list"
The default is "DataFrame".
debug : bool, optional
returns the NetCDF object for debug purposes
Returns
-------
Points_out : see output_type
The extrapolated values.
"""
if not utils.is_iterable(time_xtrp):
time_xtrp = [time_xtrp]
if not utils.is_iterable(lat_xtrp):
lat_xtrp = [lat_xtrp]
if not utils.is_iterable(lon_xtrp):
lon_xtrp = [lon_xtrp]
Points_xtrp = (time_xtrp,lat_xtrp,lon_xtrp)
if type(path_or_netcdf_object_in) is str:
NC = nc.Dataset(path_or_netcdf_object_in)
elif type(path_or_netcdf_object_in) in (nc.Dataset,nc.MFDataset):
NC = path_or_netcdf_object_in
else:
NC = nc.MFDataset(sorted(path_or_netcdf_object_in))
if debug:
return NC
time_orig = np.array(NC['time'][:])
if time_smart:
# we work in MJD
start_date = conv.dt2MJD(conv.str_date2dt(NC['time'].units[11:]))
if len(time_orig) <= 366:
time = start_date - .0 + time_orig
else:
time = start_date - .0 + np.array(range(len(time_orig)))
lat = np.flip(np.array(NC['lat'][:])) ### we flip the lat because not ascending !
lon = np.array(NC['lon'][:])
Points = (time,lat,lon)
WishVals_Stk = []
WishVals_dic = dict()
#### prepare dedicated time, lat, lon columns
Points_xtrp_array = np.array(list(itertools.product(*Points_xtrp)))
WishVals_dic['time'] = Points_xtrp_array[:,0]
WishVals_dic['lat'] = Points_xtrp_array[:,1]
WishVals_dic['lon'] = Points_xtrp_array[:,2]
### do the interpolation for the wished value
for wishval in wished_values:
if verbose:
print("INFO:",wishval,"start interpolation at",dt.datetime.now())
#### Val = np.array(NC[wishval]) ### Slow
Val = NC[wishval][:]
if verbose:
print("INFO:",wishval,"grid loaded at",dt.datetime.now())
Val = np.flip(Val,1) ### we flip the lat because not ascending !
Val_xtrp = scipy.interpolate.interpn(Points,Val,
Points_xtrp,
method=interp_method)
WishVals_Stk.append(Val_xtrp)
WishVals_dic[wishval] = Val_xtrp
#### choose the output
if output_type == "DataFrame":
Points_out = pd.DataFrame(WishVals_dic)
if time_smart:
Points_out['time_dt'] = conv.MJD2dt(Points_out['time'])
elif output_type == "dict":
Points_out = WishVals_dic
elif output_type == "array":
Points_out = np.column_stack(WishVals_Stk)
elif output_type == "tuple":
Points_out = tuple(WishVals_Stk)
elif output_type == "list":
Points_out = list(WishVals_Stk)
else:
Points_out = WishVals_Stk
return Points_out
def timeline_plotter(datadico,start = dt.datetime(1980,1,1) ,
end = dt.datetime(2099,1,1),dots_plot=False,
jul_date_plot=False,datadico_anex_list = [],
use_only_stats_of_main_datadico=False,
colordico_for_main_datadico=None):
"""
A simpler version has been commited to geodezyx toolbox for archive
on 20180118 15:59A
"""
fig , ax = plt.subplots()
ax.xaxis.grid(True)
ax.yaxis.grid(True)
if not use_only_stats_of_main_datadico:
stats_concat_list = list(datadico.keys()) + sum([list(e.keys()) for e in datadico_anex_list], [])
stats_concat_list = list(reversed(sorted(list(set(stats_concat_list)))))
else:
stats_concat_list = list(reversed(sorted(list(datadico.keys()))))
# the plot has not the same behavior if it is the morning or not
# (rinexs timelines wont be ploted if it is the morning)
if dt.datetime.now().hour < 12:
morning_shift = dt.timedelta(days=1)
else:
morning_shift = dt.timedelta(days=0)
legend_list = [] # must be here before the loop
for i,stat in enumerate(stats_concat_list):
# PART 1 : PLOT MAIN DATADICO
if not stat in datadico.keys():
continue
#T = Time, O = Station name (Observation)
Torig = [ e[-1] for e in datadico[stat] ]
Oorig = [ e[1] for e in datadico[stat] ]
# Time windowing
T , O = [] , []
for t , o in zip(Torig , Oorig):
if ( start - morning_shift ) <= t <= end:
T.append(t)
O.append(o)
T,O = utils.sort_binom_list(T,O)
TMJD=conv.dt2MJD(T)
TGrpAll = utils.consecutive_groupIt(TMJD,True) # Tuples (start,end) of continue period
for tgrp in TGrpAll:
color1 = 'C0'
color2 = ''
extra_archive = False
###*** managing colors
if colordico_for_main_datadico:
igrpstart = TMJD.index(tgrp[0])
igrpend = TMJD.index(tgrp[1])
Ogrp = O[igrpstart:igrpend+1]
Tgrp = TMJD[igrpstart:igrpend+1] # all dates in the current continue period
opt_set = list(set(Ogrp))
if len(opt_set) == 1: # Regular case : only one archive
if opt_set[0] in colordico_for_main_datadico:
color1 = colordico_for_main_datadico[opt_set[0]]
else: # several archives ... so the line has to be splited in segments
extra_archive = True
OSubgrp = utils.identical_groupIt(Ogrp)
TSubgrp = utils.sublistsIt(Tgrp , [len(e) for e in OSubgrp])
Tgrp_plt = [ (e[0] , e[-1] + 1) for e in TSubgrp ] # +1 because the end boundary day is not included
Ogrp_plt = [ list(set(e))[0] for e in OSubgrp ]
Color_grp_plt = [ colordico_for_main_datadico[e] if e in colordico_for_main_datadico else color2 for e in Ogrp_plt ]
###*** End of managing colors
tgrp = (tgrp[0] , tgrp[1] + 1) # +1 because the end boundary day is not included
# must stay there, in case of not colordico_for_main_datadico
if not jul_date_plot:
tgrp = conv.MJD2dt(tgrp)
if extra_archive:
Tgrp_plt = [ (conv.MJD2dt(e[0]) , conv.MJD2dt(e[1])) for e in Tgrp_plt ]
#PLOT part
if tgrp[0] == tgrp[1] + dt.timedelta(days=1): # CASE NO PERIOD, ONLY ONE DAY
ax.plot(tgrp[0],i , '.' + color1)
else: # CASE PERIOD
if not extra_archive: # ONE ARCHIVE REGULAR CASE
ax.plot(tgrp,[i]*2, '-' + color1)
else: # SUBCASE "EXTRA" : SEVERAL ARCHIVE
for tt , cc in zip(Tgrp_plt , Color_grp_plt):
ax.plot(tt,[i]*2 , '-' + cc)
# PART 2 : PLOT ANEX DATADICO
for idatadico_anex,datadico_anex in enumerate(datadico_anex_list):
if stat in list(datadico_anex.keys()):
T = datadico_anex[stat]
T = [ t for t in T if start <= t <= end ]
T = sorted(T)
if jul_date_plot:
T = conv.MJD2dt(T)
pale_blue_dot , = ax.plot(T,i*np.ones(len(T)), 'o', color='skyblue',label="final SNX")
legend_list = [pale_blue_dot]
#### LEGEND
if colordico_for_main_datadico:
import matplotlib.lines as mlines
for arcnam , col in colordico_for_main_datadico.items():
legend_list.append(mlines.Line2D([], [], color=col,
label=arcnam))
plt.legend(handles=legend_list,loc='upper left',ncol=3,
columnspacing=1)
# ax.set_yticks(np.arange(0,len(plotstat_lis)-1),plotstat_lis)
plt.yticks(np.arange(0,len(stats_concat_list)),stats_concat_list)
fig.autofmt_xdate()
koef = np.sqrt(2) * 1
fig.set_size_inches(koef*11.69,koef*len(stats_concat_list) * 0.28) #16.53
ax.set_ylim([-1 , len(stats_concat_list) + 1])
return fig
def read_eop_finals(file_path_in,
precnut_model=2000,
simplified_EOP_DF="mixed"):
"""
read EOP finals file
Parameters
----------
file_path_in : str
path of the EOP finals file.
precnut_model : int, optional
IAU Precession-Nutation Model. Valid values are 1980 and 2000.
The default is 2000.
Returns
-------
DF : DataFrame
out DataFrame - Complete content of the finals file.
DF2 : DataFrame
out DataFrame - Simplified content of the finals file.
Based on C04 EOP DataFrame structure.
If no Bulletin-B values provided, use Bulletin-A.
Note
----
Format description
https://github.com/marando/phpIERS/blob/master/src/Marando/IERS/default/readme.finals
"""
if precnut_model == 2000:
x_or_psi = "X"
y_or_eps = "Y"
elif precnut_model == 1980:
x_or_psi = "PSI"
y_or_eps = "EPS"
else:
raise Exception
cols = ((0, 2), (2, 4), (4, 6), (6, 7), (7, 15), (15, 16), (16, 17),
(17, 18), (18, 27), (27, 36), (36, 37), (37, 46), (46, 55),
(55, 57), (57, 58), (58, 68), (68, 78), (78, 79), (79, 86),
(86, 93), (93, 95), (95, 96), (96, 97), (97, 106), (106, 115),
(115, 116), (116, 125), (125, 134), (134, 144), (144, 154),
(154, 165), (165, 175), (175, 185))
col_names = [
'yyyy', 'mm', 'dd', '[blank]', 'MJD', '[blank]', 'flg_xy', '[blank]',
'x_A', 'x_A_Err', '[blank]', 'y_A', 'y_A_Err', '[blanks]',
'flg_UT1-UTC', 'UT1-UTC_A', 'UT1-UTC_A_Err', '[blank]', 'LOD_A',
'LOD_A_Err', '[blanks]', 'flg_nut', '[blank]', 'd' + x_or_psi + '_A',
'd' + x_or_psi + '_A_Err', '[blank]', 'd' + y_or_eps + '_A',
'd' + y_or_eps + '_A_Err', 'x_B', 'y_B', 'UT1-UTC_B',
'd' + x_or_psi + '_B', 'd' + y_or_eps + '_B'
]
DF = pd.read_fwf(file_path_in, colspecs=cols, header=None)
DF.columns = col_names
### remove everything with NaN
DF.dropna(axis=1, how="all", inplace=True)
### if not x_A provided, then its a blank line
DF = DF[np.logical_not(np.isnan(DF['x_A']))]
### set mjd as int
DF["MJD"] = DF["MJD"].astype(np.int64)
#### DF 2 is a simplified version, base on the C04 DF
DF2 = pd.DataFrame()
DF2["epoch"] = conv.MJD2dt(DF["MJD"])
if simplified_EOP_DF == 'mixed':
### Create epoch and use B-values per default
DF2[['MJD', 'x', 'y', 'UT1-UTC', 'LOD', 'dX', 'dY']] = DF[[
'MJD', 'x_B', 'y_B', 'UT1-UTC_B', 'LOD_A', 'dX_B', 'dY_B'
]]
DF2["Bul"] = "B"
### If no B-values, use A-values
DF2.loc[np.isnan(DF.x_B), 'Bul'] = "A"
DF2.loc[np.isnan(DF.x_B), 'x'] = DF.loc[np.isnan(DF.x_B), 'x_A']
DF2.loc[np.isnan(DF.y_B), 'y'] = DF.loc[np.isnan(DF.y_B), 'y_A']
DF2.loc[np.isnan(DF['UT1-UTC_B']),
'UT1-UTC'] = DF.loc[np.isnan(DF['UT1-UTC_B']), 'UT1-UTC_A']
DF2.loc[np.isnan(DF.dX_B), 'dX'] = DF.loc[np.isnan(DF.dX_B), 'dX_A']
DF2.loc[np.isnan(DF.dX_B), 'dY'] = DF.loc[np.isnan(DF.dY_B), 'dY_A']
elif simplified_EOP_DF == "A":
DF2[['MJD', 'x', 'y', 'UT1-UTC', 'LOD', 'dX', 'dY']] = DF[[
'MJD', 'x_A', 'y_A', 'UT1-UTC_A', 'LOD_A', 'dX_A', 'dY_A'
]]
DF2["Bul"] = "A"
DF2.dropna(axis=1, how="all", inplace=True)
elif simplified_EOP_DF == "B":
DF2[['MJD', 'x', 'y', 'UT1-UTC', 'LOD', 'dX', 'dY']] = DF[[
'MJD', 'x_B', 'y_B', 'UT1-UTC_B', 'LOD_A', 'dX_B', 'dY_B'
]]
DF2["Bul"] = "B"
DF2.dropna(axis=1, how="all", inplace=True)
else:
print("ERR: read_eop_C04: check the simplified_EOP_DF parameter !!!")
raise Exception
return DF, DF2
def read_sp3_header(sp3_in, ac_name_in=None):
"""
Read a SP3 file header and return a Pandas DataFrame
with sat. PRNs and sigmas contained in the header
Parameters
----------
sp3_in : str or list
path of the SP3 file
can handle gzip-compressed file (with .gz/.GZ extension)
can also handle the sp3 content as a list of strings
(useful when read_sp3_header is used as a subfunction of read_sp3)
ac_name_in : str
force the AC name
(necessary when read_sp3_header is used as a subfunction of read_sp3)
Returns
-------
Header_DF : Pandas DataFrame
2 columns "sat", "sigma"
Note
-------
More infos about the sigma
http://acc.igs.org/orbacc.txt
"""
if type(sp3_in) is list:
### case when read_sp3_header is used as a subfunction of read_sp3
Lines = sp3_in
elif sp3_in[-2:] in ("gz", "GZ"):
F = gzip.open(sp3_in, "r+")
Lines = [e.decode('utf-8') for e in F]
else:
F = open(sp3_in, 'r+')
Lines = F.readlines()
if not ac_name_in:
ac_name = os.path.basename(sp3_in)[:3]
else:
ac_name = ac_name_in
Sat_prn_list = []
Sat_sig_list = []
for il, l in enumerate(Lines):
if il == 1:
date = conv.MJD2dt(int(l.split()[4]))
if l[:2] == "+ ":
Sat_prn_list.append(l)
if l[:2] == "++":
Sat_sig_list.append(l)
if l[0] == "*":
break
### PRN part
Sat_prn_list_clean = []
for prn_line in Sat_prn_list:
prn_line_splited = prn_line.split()
prn_line_splited = [e for e in prn_line_splited if not "+" in e]
prn_line_splited = [e for e in prn_line_splited if not e == "0"]
Sat_prn_list_clean = Sat_prn_list_clean + prn_line_splited
try:
sat_nbr = int(Sat_prn_list_clean[0])
except:
sat_nbr = 0
Sat_prn_list_clean = Sat_prn_list_clean[1:]
Sat_prn_string = "".join(Sat_prn_list_clean)
Sat_prn_list_final = []
for i in range(sat_nbr):
Sat_prn_list_final.append(Sat_prn_string[i * 3:i * 3 + 3])
### Sigma part
Sat_sig_list_clean = []
for sig_line in Sat_sig_list:
sig_line_splited = sig_line.split()
sig_line_splited = [e for e in sig_line_splited if not "+" in e]
Sat_sig_list_clean = Sat_sig_list_clean + sig_line_splited
Sat_sig_list_final = [int(e) for e in Sat_sig_list_clean[:sat_nbr]]
### Export part
AC_name_list = [ac_name] * sat_nbr
Date_list = [date] * sat_nbr
Header_DF = pd.DataFrame(list(
zip(AC_name_list, Sat_prn_list_final, Sat_sig_list_final, Date_list)),
columns=["AC", "sat", "sigma", "epoch"])
return Header_DF