def read_combi_sum_exclu(sum_file, return_as_df=True, use_intuitive_bool=True):
t_dt = conv.sp3name2dt(sum_file)
with open(sum_file) as f:
cont = f.readlines()
excluded_dic = dict()
useful_ssection = False
useful_ssection_k = 0
for l in cont:
f = l.split()
if "---|---" in l and useful_ssection_k < 2:
useful_ssection = not useful_ssection
useful_ssection_k += 1
continue
if not useful_ssection:
continue
prn_raw = f[0]
if "X" in prn_raw:
exclu = True
else:
exclu = False
if use_intuitive_bool:
exclu = not exclu
prn_int = int(f[0].replace("X", "").split()[0])
prn_good = prn_int_2_prn_str(prn_int)
excluded_dic[prn_good] = exclu
if return_as_df:
return pd.DataFrame(excluded_dic, index=[t_dt])
else:
return excluded_dic
def read_erp(file_path_in, ac=None):
"""
Read IGS Analysis Center ERP files
Parameters
----------
file_path_in : str
Path of the file in the local machine.
ac : str
The analysis center that will be used.
If not precised, will be the first 3 letters of the input name
Returns
-------
out1 : pandas table
Returns a panda table format with the data extracted from the file.
Note
----
Columns name
('MJD','X-P (arcsec)', 'Y-P (arcsec)', 'UT1UTC (E-7S)','LOD (E-7S/D)','S-X (E-6" arcsec)','S-Y (E-6" arcsec)',
'S-UT (E-7S)','S-LD (E-7S/D)','NR (E-6" arcsec)', 'NF (E-6" arcsec)', 'NT (E-6" arcsec)',
'X-RT (arcsec/D)','Y-RT (arcsec/D)','S-XR (E-6" arcsec/D)','S-YR (E-6" arcsec/D)', 'C-XY', 'C-XT',
'C-YT', 'DPSI', 'DEPS','S-DP','S-DE')
"""
caminho_arq = file_path_in
#### FIND DELIVERY DATE
name = os.path.basename(caminho_arq)
if not ac:
ac = name[:3]
if len(name) == 12:
dt_delivery = conv.sp3name2dt(caminho_arq)
elif len(name) == 38:
dt_delivery = conv.sp3name_v3_2dt(caminho_arq)
else:
dt_delivery = conv.posix2dt(0)
le = open(caminho_arq, 'r')
letudo = le.readlines()
le.close()
tamanho = len(letudo) #usado para saber quantas linhas tem o arquivo
#para = tamanho #onde o arquivo para de ser lido
numeros = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
#le = 0
#numlin = 0 #numero de linhas da matriz de epocas
#numcol = 16 #numero de colunas que a matriz final deve ter
ERP = []
if caminho_arq[-3:] in ('snx', 'ssc'):
file = open(caminho_arq)
Lines = file.readlines()
XPO_stk = []
XPO_std_stk = []
YPO_stk = []
YPO_std_stk = []
LOD_stk = []
LOD_std_stk = []
MJD_stk = []
marker = False
for i in range(len(Lines)):
if len(Lines[i].strip()) == 0:
continue
else:
if Lines[i].split()[0] == '+SOLUTION/ESTIMATE':
marker = True
if Lines[i].split()[0] == '-SOLUTION/ESTIMATE':
marker = False
if utils.contains_word(Lines[i], 'XPO') and marker:
# Doy = (Lines[i][30:33])
# Year = (Lines[i][27:29])
# Pref_year = '20'
# Year = int(Pref_year+Year)
# Date = conv.doy2dt(Year,Doy)
Date = conv.datestr_sinex_2_dt(Lines[i].split()[5])
XPO = float(Lines[i][47:68]) * (10**-3)
XPO_std = float(Lines[i][69:80]) * (10**-3)
XPO_stk.append(XPO)
XPO_std_stk.append(XPO_std)
MJD_stk.append(conv.dt2MJD(Date))
#MJD_stk.append(cmg.jd_to_mjd(cmg.date_to_jd(Date.year,Date.month,Date.day)))
if utils.contains_word(Lines[i], 'YPO') and marker:
# Doy = (Lines[i][30:33])
# Year = (Lines[i][27:29])
# Pref_year = '20'
# Year = int(Pref_year+Year)
# Date = conv.doy2dt(Year,Doy)
Date = conv.datestr_sinex_2_dt(Lines[i].split()[5])
YPO = float(Lines[i][47:68]) * (10**-3)
YPO_std = float(Lines[i][69:80]) * (10**-3)
YPO_stk.append(YPO)
YPO_std_stk.append(YPO_std)
MJD_stk.append(conv.dt2MJD(Date))
#MJD_stk.append(cmg.jd_to_mjd(cmg.date_to_jd(Date.year,Date.month,Date.day)))
if utils.contains_word(Lines[i], 'LOD') and marker:
# Doy = (Lines[i][30:33])
# Year = (Lines[i][27:29])
# Pref_year = '20'
# Year = int(Pref_year+Year)
# Date = conv.doy2dt(Year,Doy)
Date = conv.datestr_sinex_2_dt(Lines[i].split()[5])
LOD = float(Lines[i][47:68]) * (10**+4)
LOD_std = float(Lines[i][69:80]) * (10**+4)
LOD_stk.append(LOD)
LOD_std_stk.append(LOD_std)
#MJD_stk.append(cmg.jd_to_mjd(cmg.date_to_jd(Date.year,Date.month,Date.day)))
MJD_stk.append(conv.dt2MJD(Date))
MJD = list(sorted(set(MJD_stk)))
if len(LOD_stk) == 0:
LOD_stk = ['0'] * len(MJD)
LOD_std_stk = ['0'] * len(MJD)
for i in range(len(MJD)):
ERP_data = [
ac, MJD[i], XPO_stk[i], YPO_stk[i], 0, LOD_stk[i],
XPO_std_stk[i], YPO_std_stk[i], 0, LOD_std_stk[i], 0, 0, 0, 0,
0, 0, 0, dt_delivery
]
ERP.append(ERP_data)
if ac in ('COD', 'cod', 'com', 'cof', 'grg', 'mit', 'sio', 'igs', 'igr'):
for i in range(tamanho + 1):
linhaatual = linecache.getline(caminho_arq, i)
if linhaatual[0:1] in numeros:
ERP_data = linhaatual.split()
for j in range(len(ERP_data)):
ERP_data[j] = float(ERP_data[j])
ERP_data.insert(0, ac)
ERP_data[2] = ERP_data[2] * (10**-6)
ERP_data[3] = ERP_data[3] * (10**-6)
ERP_data[13] = ERP_data[13] * (10**-6)
ERP_data[14] = ERP_data[14] * (10**-6)
del ERP_data[17:]
ERP_data.append(dt_delivery)
ERP.append(ERP_data)
linecache.clearcache()
# Erp_end = pd.DataFrame(ERP, columns=['AC','MJD','X-P', 'Y-P', 'UT1UTC(UT1 -TAI)','LOD','S-X','S-Y','S-UT','S-LD','NR', 'NF', 'NT',
# 'X-RT','Y-RT','S-XR','S-YR',"Delivery_date"])
# return Erp_end
#
if ac in ('wum', 'grg', 'esa', 'mit', 'ngs', 'sio'):
for i in range(tamanho + 1):
linhaatual = linecache.getline(caminho_arq, i)
if linhaatual[0:1] in numeros:
ERP_data = linhaatual.split()
for j in range(len(ERP_data)):
ERP_data[j] = float(ERP_data[j])
ERP_data.insert(0, ac)
ERP_data[2] = ERP_data[2] * (10**-6)
ERP_data[3] = ERP_data[3] * (10**-6)
ERP_data[13] = ERP_data[13] * (10**-6)
ERP_data[14] = ERP_data[14] * (10**-6)
ERP_data.append(dt_delivery)
ERP.append(ERP_data)
linecache.clearcache()
# Erp_end = pd.DataFrame(ERP, columns=['AC','MJD','X-P', 'Y-P', 'UT1UTC(UT1 -TAI)','LOD','S-X','S-Y','S-UT','S-LD','NR', 'NF', 'NT',
# 'X-RT','Y-RT','S-XR','S-YR'])
# return Erp_end
#
if ac in ('gbm', 'gfz', 'gfr', "p1_", "p1r"):
for i in range(tamanho + 1):
linhaatual = linecache.getline(caminho_arq, i)
if linhaatual[0:1] in numeros:
ERP_data = linhaatual.split()
for j in range(len(ERP_data)):
ERP_data[j] = float(ERP_data[j])
ERP_data.insert(0, ac)
ERP_data[2] = ERP_data[2] * (10**-6)
ERP_data[3] = ERP_data[3] * (10**-6)
ERP_data[13] = ERP_data[13] * (10**-6)
ERP_data[14] = ERP_data[14] * (10**-6)
ERP_data.append(dt_delivery)
ERP.append(ERP_data)
linecache.clearcache()
# Erp_end = pd.DataFrame(ERP, columns=['AC','MJD','X-P', 'Y-P', 'UT1UTC(UT1 -TAI)','LOD','S-X','S-Y','S-UT','S-LD','NR', 'NF', 'NT',
# 'X-RT','Y-RT','S-XR','S-YR']) ##EH TBM O RATE XY POR DIA??????
# return Erp_end
header = []
if ac in ('emr'):
for i in range(tamanho + 1):
linhaatual = linecache.getline(caminho_arq, i)
if linhaatual == 'EOP SOLUTION':
del ERP_data[:]
header = ['EOP SOLUTION']
if linhaatual[0:1] in numeros and 'EOP SOLUTION' in header:
ERP_data = linhaatual.split()
for j in range(len(ERP_data)):
ERP_data[j] = float(ERP_data[j])
ERP_data.insert(0, ac)
ERP_data[2] = ERP_data[2] * (10**-6)
ERP_data[3] = ERP_data[3] * (10**-6)
ERP_data[13] = ERP_data[13] * (10**-6)
ERP_data[14] = ERP_data[14] * (10**-6)
del ERP_data[17:]
ERP_data.append(dt_delivery)
ERP.append(ERP_data)
linecache.clearcache()
Erp_end = pd.DataFrame(ERP,
columns=[
'AC', 'MJD', 'X-P', 'Y-P', 'UT1UTC(UT1 -TAI)',
'LOD', 'S-X', 'S-Y', 'S-UT', 'S-LD', 'NR', 'NF',
'NT', 'X-RT', 'Y-RT', 'S-XR', 'S-YR',
'Delivered_date'
])
return Erp_end
def sp3_overlap_creator(ac_list,
dir_in,
dir_out,
suffix_out_input=None,
overlap_size=7200,
force=False,
manage_missing_sats='common_sats_only',
eliminate_null_sat=True,
severe=False,
separated_systems_export=False,
first_date=None):
"""
Generate an SP3 Orbit file with overlap based on the SP3s of the
days before and after
Parameters
----------
ac_list : list
3-character codes of the ACs.
dir_in : str
where the input sp3 are.
dir_out : str
where the output sp3 will be outputed.
suffix_out_input : str, optional
last char of the 3-char. code. if None, then it is the same as input.
overlap_size : int, optional
Overlapsize. The default is 7200.
force : True, optional
force overwrite. The default is False.
manage_missing_sats : str, optional
'exclude' : generate a file with only the common sat
between the 3 days. Thus, exclude the missing sats
'extrapolate' : extrapolate the missing sats based on the first/last epoch
The default is 'common_sats_only'.
eliminate_null_sat : bool, optional
eliminate null sat. The default is True.
severe : bool, optional
raise an exception if problem. The default is False.
separated_systems_export : bool, optional
export different sp3 for different system. The default is False.
first_date : datetime, optional
exclude SP3 before this epoch
Returns
-------
None.
Note
----
start/end date are not implemented
the force option skips existing files
"""
Dict_Lfiles_ac = dict()
for ac in ac_list:
Dict_Lfiles_ac[ac] = []
Lfile = Dict_Lfiles_ac[ac]
Extlist = ["sp3", "SP3", "sp3.gz", "SP3.gz"]
for ext in Extlist:
Lfile = Lfile + utils.find_recursive(dir_in, "*" + ac + "*" + ext)
print("Nb of SP3 found for", ac, len(Lfile))
if not suffix_out_input:
suffix_out = ac
else:
suffix_out = ac[:2] + suffix_out_input
D = []
WWWWD = []
for sp3 in Lfile:
#wwwwd_str = os.path.basename(sp3)[3:8]
#D.append(conv.gpstime2dt(int(wwwwd_str[:4]),int(wwwwd_str[4:])))
dat = conv.sp3name2dt(sp3)
D.append(dat)
for dat in D[1:-1]: ####if selection manuel, zip > 2lists !!!
try:
print("***********", ac, dat)
if first_date:
if dat < first_date:
print("INFO: SKIP date", dat)
continue
wwwwd_str = conv.dt_2_sp3_datestr(dat).zfill(5)
dat_bef = dat - dt.timedelta(days=1)
dat_aft = dat + dt.timedelta(days=1)
wwwwd_str_bef = utils.join_improved(
"", *conv.dt2gpstime(dat_bef)).zfill(5)
wwwwd_str_aft = utils.join_improved(
"", *conv.dt2gpstime(dat_aft)).zfill(5)
###### check if exists
dir_out_wk = os.path.join(dir_out, "wk" + str(wwwwd_str)[:4])
utils.create_dir(dir_out_wk)
fil_out = dir_out_wk + "/" + suffix_out + wwwwd_str + ".sp3"
if not force and os.path.isfile(fil_out):
print("0))", fil_out, "exists, skipping...")
continue
### *************** STEP 1 ***************
print("1)) Search for the days before/after")
print("1))", dat_bef, dat_aft)
p1 = utils.find_regex_in_list(wwwwd_str + ".sp3", Lfile, True)
p_bef = utils.find_regex_in_list(wwwwd_str_bef + ".sp3", Lfile,
True)
p_aft = utils.find_regex_in_list(wwwwd_str_aft + ".sp3", Lfile,
True)
print("1)) Files found for the days before/after")
print("0b)", p_bef)
print("01)", p1)
print("0a)", p_aft)
if not p1 or not p_bef or not p_aft:
print("ERROR with day", dat)
continue
SP3 = files_rw.read_sp3(p1)
SP3_bef = files_rw.read_sp3(p_bef)
SP3_aft = files_rw.read_sp3(p_aft)
### Filtering to keep P only
SP3 = SP3[SP3.type == "P"]
SP3_bef = SP3_bef[SP3_bef.type == "P"]
SP3_aft = SP3_aft[SP3_aft.type == "P"]
SP3_bef = SP3_bef[SP3_bef["epoch"] < SP3["epoch"].min()]
SP3_aft = SP3_aft[SP3_aft["epoch"] > SP3["epoch"].max()]
SP3concat = pd.concat((SP3_bef, SP3, SP3_aft))
dat_filter_bef = dat - dt.timedelta(seconds=overlap_size)
dat_filter_aft = dat + dt.timedelta(
seconds=overlap_size) + dt.timedelta(days=1)
### *************** STEP 2 ***************
print("2)) dates of the overlap period before/after")
print("2))", dat_filter_bef, dat_filter_aft)
### *************** STEP 3 ***************
print("3)) Dates of: SP3 concatenated, before, current, after")
print("3))", SP3concat["epoch"].min(),
SP3concat["epoch"].max())
print("3b)", SP3_bef["epoch"].min(), SP3_bef["epoch"].max())
print("31)", SP3["epoch"].min(), SP3["epoch"].max())
print("3a)", SP3_aft["epoch"].min(), SP3_aft["epoch"].max())
SP3concat = SP3concat[(SP3concat["epoch"] >= dat_filter_bef)
& (SP3concat["epoch"] <= dat_filter_aft)]
########## HERE WE MANAGE THE MISSING SATS
if manage_missing_sats == "exclude":
print("4))", "remove missing sats ")
common_sats = set(SP3_bef["sat"]).intersection(
set(SP3["sat"])).intersection(set(SP3_aft["sat"]))
SP3concat = SP3concat[SP3concat["sat"].isin(common_sats)]
elif manage_missing_sats == "extrapolate":
print("4))", "extrapolate missing sats ")
for iovl, SP3_ovl in enumerate((SP3_bef, SP3_aft)):
if iovl == 0:
backward = True
forward = False
backfor = "backward"
elif iovl == 1:
backward = False
forward = True
backfor = "forward"
Sats = set(SP3["sat"])
Sats_ovl = set(SP3_ovl["sat"])
Sats_miss = Sats.difference(Sats_ovl)
if not Sats_miss:
continue
print("4a)", "extrapolate missing sats", backfor,
Sats_miss)
SP3extrapo_in = SP3concat[SP3concat["sat"].isin(
Sats_miss)]
#step = utils.most_common(SP3concat["epoch"].diff().dropna())
#step = step.astype('timedelta64[s]').astype(np.int32)
step = 900
#print(step)
#print("SP3extrapo_in",SP3extrapo_in)
SP3extrapo = reffram.extrapolate_sp3_DataFrame(
SP3extrapo_in,
step=step,
n_step=int(overlap_size / step),
backward=backward,
forward=forward,
until_backward=dat_filter_bef,
until_forward=dat_filter_aft,
return_all=False)
SP3concat = pd.concat((SP3concat, SP3extrapo))
print(SP3extrapo)
else:
print("ERR: check manage_missing_sats value")
raise Exception
if eliminate_null_sat:
GoodSats = []
for sat in SP3concat["sat"].unique():
XYZvals = SP3concat[SP3concat["sat"] == sat][[
"x", "y", "z"
]].sum(axis=1)
V = np.sum(np.isclose(XYZvals, 0)) / len(XYZvals)
if V < 0.50:
GoodSats.append(sat)
else:
print("6) eliminate because null position", sat)
SP3concat = SP3concat[SP3concat["sat"].isin(GoodSats)]
### *************** STEP 7 ***************
print("7))", "Start/End Epoch of the concatenated file ")
print("7))", SP3concat["epoch"].min(),
SP3concat["epoch"].max())
#### All systems
print("8)) outputed file")
print(fil_out)
write_sp3(SP3concat, fil_out)
#### system separated
if False:
for sys in SP3concat["const"].unique():
try:
SP3concat_sys = SP3concat[SP3concat["const"] ==
sys]
fil_out_sys = dir_out_wk + "/" + suffix_out[:2] + sys.lower(
) + wwwwd_str.zfill(5) + ".sp3"
print("9)) outputed file")
print(fil_out_sys)
write_sp3(SP3concat_sys, fil_out_sys)
except:
continue
except KeyboardInterrupt:
raise KeyboardInterrupt
except Exception as e:
if severe:
print("WARN:", e)
raise e
else:
print("WARN: Error", e, "but no severe mode, continue...")
continue
"""
def read_pdm_res_slr_mono(res_file_in, sol="sol"):
"""
Read a PDM7 res(idual) file for SLR Validation
Parameters
----------
res_file_in : str
path of the input res file.
sol : str or lambda fct
solution name
if it is a lambda fct, it will grab the sol name from the full residual path
e.g. : solnam = lambda x: x.split("/")[-5][4:]
Returns
-------
DFout : Pandas DataFrame
output DataFrame.
"""
dat = conv.sp3name2dt("xxx" + os.path.basename(res_file_in)[:5])
### get useful values
L = utils.extract_text_between_elements_2(res_file_in, "\+residuals",
"\-residuals")
L = L[3:-1]
output = io.StringIO()
output.write("".join(L))
output.seek(0)
###
if utils.is_lambda(sol):
sol_wrk = sol(res_file_in)
else:
sol_wrk = sol
### read
DFout = pd.read_csv(output, header=None, delim_whitespace=True)
### rename useful columns
DFout = DFout.rename(
columns={
0: 'time',
1: 'sta',
2: 'sat',
4: 'res',
5: 'elev',
6: 'azi',
7: 'amb',
8: 'slr',
9: 'dt_sta',
10: 'delay',
11: 'sig_sta'
})
DFout["day"] = dat
DFout["epoc"] = dat + DFout["time"].apply(
lambda x: dt.timedelta(microseconds=x * 86400 * 10 * 6))
DFout['sol'] = sol_wrk
DFout["sys"] = DFout["sat"].str[0]
return DFout