def test_date2gpswd(self):
"""Test date2gpswd function,
use date: 2017-5-17, GPS week: 1949, day of week: 3.
"""
date = datetime.date(2017, 5, 17)
gpsweek, dayofweek = gnsscal.date2gpswd(date)
self.assertEqual(gpsweek, 1949)
self.assertEqual(dayofweek, 3)
date = datetime.date(1980, 1, 1)
with self.assertRaises(ValueError):
gnsscal.date2gpswd(date)
def parse(date,
receiver_name,
constellations,
input_folder='',
output_folder=''):
# Print a message.
date = date_type(int(date[:4]), int(date[4:6]), int(date[6:]))
# Define PRNs.
PRNs_to_parse = []
if 'G' in constellations:
PRNs_to_parse.extend(['G' + str(i) for i in range(1, 33)])
if 'R' in constellations:
PRNs_to_parse.extend(['R' + str(i) for i in range(1, 25)])
if 'E' in constellations:
PRNs_to_parse.extend(['E' + str(i) for i in range(1, 31)])
# Define parsing parameters.
parameters = ParseSettings()
parameters.binary_dir = input_folder + filesep + receiver_name
parameters.CSV_dir = output_folder + filesep + receiver_name + filesep + 'CSV_FILES'
parameters.receiver_name = receiver_name
parameters.date_range = False
parameters.start_date = [date.year, date.month, date.day]
parameters.end_date = [date.year, date.month, date.day]
parameters.reduced = True
parameters.raw = False
parameters.PRNs_to_parse = PRNs_to_parse
parameters.set_time_range = False
# Binary dir and file.
binary_file = str(date2gpswd(date)[0]) + '_' + str(
date2gpswd(date)[1]) + '_00_' + receiver_name + '.GPS'
binary_dir = parameters.binary_dir + filesep + str(
date2gpswd(date)[0]) + filesep + binary_file
# If the binary file exists, parse.
if os.path.exists(binary_dir):
# Print status to command window.
print(
"\n---------------------------------------------------------------------"
)
print(
"PART 1: EISA PARSING. Receiver: {}. Date: ({}, {}, {})\n".format(
receiver_name, date.year, date.month, date.day))
# Parse.
run_parsing(parameters, cwd + filesep + "Parsing")
else:
print("The following binary file does not exist: {}. Receiver: {}.".
format(binary_dir, receiver_name))
def run_parsing(model, exe_dir):
# Process the dates. Obtain the names of the binary files.
start_year, start_month, start_day = model.start_date
end_year, end_month, end_day = model.end_date
number_of_days = (date(end_year, end_month, end_day) - date(start_year, start_month, start_day)).days
if number_of_days < 0:
print('Error: The selected end date must be after the start date.')
days = [date(start_year, start_month, start_day) + timedelta(days=i) for i in range(number_of_days + 1)]
binary_files = [str(date2gpswd(day)[0]) + '_' + str(date2gpswd(day)[1]) + '_00_' + model.receiver_name + '.GPS' for
day in days]
# Parse the binary files.
for binary_file in binary_files:
# Parse file.
success, error = parse_binary_file(binary_file, exe_dir, model)
if not success:
print(error)
def __init__(self, dt):
self.__dt = dt
self._year = self.__dt.year
self._month = self.__dt.month
self._day = self.__dt.day
self._doy = dt.timetuple().tm_yday
gpswd = date2gpswd(date(self._year, self._month, self._day))
self._gpsweek = gpswd[0]
self._gpsweekday = gpswd[1]
def download_corr_files(out_path, base_stn, obs, nav, sevenzip):
"""
This function downloads the necessary base station and satellite files (broadcast/orbits)
for correcting raw rinex files using rtkpost.
"""
# file root name
name = obs.split('.obs')[0]
# first we need the year, doy, and GPS week from the observations file
f = open(out_path + obs, 'r').read()
print("Downloading correction data for {}".format(obs))
lines = f.strip().splitlines()
for l in lines:
# read from RINEX header
if l.endswith('TIME OF FIRST OBS'):
var = l.split()
yr = var[0]
# use datetime and gnsscal to convert the date to doy and gps week
date = datetime.date(int(yr), int(var[1]), int(var[2]))
doy = str(gnsscal.date2doy(date))
if not len(doy) == 3: # add the leading 0 if DOY is < 3 digits
doy = '0' + doy
week = str(gnsscal.date2gpswd(date)[0]) + str(
gnsscal.date2gpswd(date)[1])
# create a new folder to hold the correction data and move the rinex files into it
path = out_path + yr + doy + '_' + name + '/'
if not os.path.exists(path):
os.mkdir(path)
shutil.move(out_path + obs, path + obs)
shutil.move(out_path + nav, path + nav)
# download the base station observations, broadcast navigation, and satellite orbital clocks
# NOTE: we log into and out of the FTP server each time to avoid some hang-ups in downloading
# also this could allow for replacement of the server to different locations for each file
# connect to the FTP for getting the base station observations
f = ftp.FTP('igs.ensg.ign.fr')
try:
f.login()
print('logged into ftp')
except:
print('hmm couldn\'t connect to base station ftp. no internet?')
sys.exit()
# navigate to the directory
f.cwd('/pub/igs/data/' + yr + '/' + doy + '/')
# download the base station using the leading identifier and wildcards
# note we use the 30 second decimated data (*30S* wildcard below), this is
# available for most sites
filematch = base_stn + '*30S*.crx.gz'
for filename in f.nlst(filematch):
target_file_name = os.path.join(path, os.path.basename(filename))
with open(target_file_name, 'wb') as fhandle:
f.retrbinary('RETR %s' % filename, fhandle.write)
# quit and close server connection
f.quit()
f.close()
#also unzip the file with 7zip
cmd = '"' + sevenzip + '"' + ' e ' + path + filename + ' -o' + path
subprocess.call(cmd, shell=False)
# change the file name and convert to rinex with crnx2rnx
shutil.move(path + filename.split('.gz')[0],
path + filename.split('crx')[0] + yr[2:] + 'd')
cmd = crnx2rnx + ' ' + path + filename.split('crx')[0] + yr[2:] + 'd'
subprocess.call(cmd, shell=False)
# final filename
baseSTN = path + filename.split('crx')[0] + yr[2:] + 'o'
# grab the broadcast navigation data from the same directory
f = ftp.FTP('igs.ensg.ign.fr')
try:
f.login()
print('logged into ftp')
except:
print('hmm couldn\'t connect to base station ftp. no internet?')
sys.exit()
# navigate to the directory
f.cwd('/pub/igs/data/' + yr + '/' + doy + '/')
# get the filematch
filename = 'brdc' + doy + '0.' + yr[2:] + 'n.Z'
target_file_name = os.path.join(path, os.path.basename(filename))
with open(target_file_name, 'wb') as fhandle:
f.retrbinary('RETR %s' % filename, fhandle.write)
# quit and close server connection
f.quit()
f.close()
# unzip with 7zip
cmd = '"' + sevenzip + '"' + ' e ' + path + filename + ' -o' + path
subprocess.call(cmd, shell=False)
# final filename
brdc = target_file_name.split('.Z')[0]
# finally grab the satellite precise orbits from a different directory
f = ftp.FTP('igs.ensg.ign.fr')
try:
f.login()
print('logged into ftp')
except:
print('hmm couldn\'t connect to base station ftp. no internet?')
sys.exit()
# navigate to the directory
f.cwd('/pub/igs/products/' + week[0:4] + '/')
# we try with the rapid orbits, if they're available
try:
filename = 'igr' + week + '.sp3.Z'
target_file_name = os.path.join(path, os.path.basename(filename))
with open(target_file_name, 'wb') as fhandle:
f.retrbinary('RETR %s' % filename, fhandle.write)
# retry with the ultra-rapid orbits if that didn't work
except:
filename = 'igu' + week + '_18.sp3.Z' # arbitrarily taking the final ultra-rapid file (18:00)
target_file_name = os.path.join(path, os.path.basename(filename))
with open(target_file_name, 'wb') as fhandle:
f.retrbinary('RETR %s' % filename, fhandle.write)
# unzip with 7zip
cmd = '"' + sevenzip + '"' + ' e ' + path + filename + ' -o' + path
subprocess.call(cmd, shell=False)
# final filename
orbits = target_file_name.split('.Z')[0]
# quit and close server connection for good
f.quit()
f.close()
return yr, doy, week, baseSTN, brdc, orbits
def ML_event_detection(date,
receiver_name,
constellations,
threshold,
location,
input_folder='',
output_folder=''):
# Determine the date, CSV dir, and GRAPHS dir.
year, month, day = date[:4], date[4:6], date[6:]
binary_dir = input_folder + filesep + receiver_name
CSV_dir = output_folder + filesep + receiver_name + filesep + 'CSV_FILES'
graphs_dir = output_folder + filesep + receiver_name + filesep + 'GRAPHS'
# Define PRNs.
PRNs_to_process = []
if 'G' in constellations:
PRNs_to_process.extend(['G' + str(i) for i in range(1, 33)])
if 'R' in constellations:
PRNs_to_process.extend(['R' + str(i) for i in range(1, 25)])
if 'E' in constellations:
PRNs_to_process.extend(['E' + str(i) for i in range(1, 31)])
# Continue only if the path containing the csv files of the corresponding date exists.
if os.path.exists(CSV_dir + filesep + date):
# Print status to command window.
print(
"\n---------------------------------------------------------------------"
)
print("PART 3: EISA ML MODULE. Receiver: {}. Date: ({}, {}, {})\n".
format(receiver_name, year, month, day))
# Create S4 Neural Network, and load the weights.
S4_model = NNModel('S4')
S4_model.load_weights('ML' + filesep + 's4_scintillation.h5')
# Create sigma Neural Network, and load the weights.
sigma_model = NNModel('sigma')
sigma_model.load_weights('ML' + filesep + 'sigma_scintillation.h5')
# Identify binary file name.
day = date_type(int(year), int(month), int(day))
week_number, week_day_number = int(date2gpswd(day)[0]), int(
date2gpswd(day)[1])
binary_file_name = "{}_{}_00_{}.GPS".format(str(week_number),
str(week_day_number),
receiver_name)
binary_file = binary_dir + filesep + str(
week_number) + filesep + binary_file_name
# Ionospheric scintillation detection.
for prn in PRNs_to_process:
# Files.
input_file = CSV_dir + filesep + date + filesep + 'REDOBS_{}_{}.csv'.format(
prn, date)
output_file = CSV_dir + filesep + date + filesep + r'\ML_Detection\REDOBS_{}_{}_ML'.format(
prn, date)
# Convert date to list format (which is the input format for the run_ML function).
date_list = [date[:4], date[4:6], date[6:]]
# Directory to the new (ML) plots.
graphs_output_dir = graphs_dir + filesep + date + filesep + 'ML'
# Check that the input file exists.
if not os.path.isfile(input_file):
continue
# ML Detection. Use try except, so that the code continues running even in the case of an error.
try:
# ML Detection: S4 scintillation.
print(
'\n ----- Ionospheric Scintillation Event Detection (ML Module). Date: {}. PRN: {}. -----'
.format(date, prn))
output_files_s4 = run_ML(input_file,
output_file,
S4_model,
prn,
date_list,
scintillation_type='S4',
save_plot=True,
save_plot_dir=graphs_output_dir +
filesep + 'Amplitude',
threshold=threshold,
location=location)
# ML Detection: sigma scintillation.
output_files_sigma = run_ML(input_file,
output_file,
sigma_model,
prn,
date_list,
scintillation_type='sigma',
save_plot=True,
save_plot_dir=graphs_output_dir +
filesep + 'Phase',
threshold=threshold,
location=location)
# Raw data period processing.
output_files = output_files_s4 + output_files_sigma
if output_files:
success, msg = parse_file(binary_file,
CSV_dir,
os.getcwd() + filesep +
'parsing', [prn],
week_number,
week_day_number,
reduced_or_raw='raw',
print_header=False)
if not success:
print(msg)
# If no output files were generated, print a message.
else:
print(
'No scintillation event was found for prn {}.'.format(
prn))
# Print exception if an error is raised.
except Exception as e:
print(
'Could not process prn {}. The following exception occurred: {}.'
.format(prn, e.message))
d['predicted'] = df_predict_prn['yhat'].values
d['index'] = df_expect_prn['epoch_time'].values
dc[column] = d
original_dataframe[column] = df_expect_prn[column]
final_dataframe[column] = pd.DataFrame(
data=df_predict_prn['yhat'].values, columns=['m'])
elif column == "OMEGA":
df = pd.DataFrame()
df['epoch_time'] = df_expect_prn['epoch_time']
t_len = df_final_prn.shape[0]
df['OMEGA'] = df_final_prn['OMEGA'][t_len - 1:t_len][t_len -
1:t_len]
prev_week, _ = gnsscal.date2gpswd(
df_final_prn['epoch_time'][t_len - 1:t_len].dt.date.values[0])
this_week, _ = gnsscal.date2gpswd(
df_expect_prn['epoch_time'][0:1].dt.date.values[0])
if this_week != prev_week:
hdiff = (24 -
df_final_prn['epoch_time'][t_len -
1:t_len].dt.hour.values[0]
) + df_expect_prn['epoch_time'][:1].dt.hour.values[0]
df['OMEGA'][:1] = df_final_prn['OMEGA'][
t_len - 1:t_len].values[0] - hdiff * 6 * (10**-6) - 0.12
else:
hdiff = (24 -
df_final_prn['epoch_time'][t_len -
1:t_len].dt.hour.values[0]
) + df_expect_prn['epoch_time'][:1].dt.hour.values[0]
def Accepter():
while 1:
global conn
if ABS.accept()[0] or PPP.accept()[0] or RTK.accept()[0]:
conn = {
'ABS': ABS.accept()[0],
'PPP': PPP.accept()[0],
'RTK': RTK.accept()[0]
}
eventConnection.set()
today = datetime.date.today()
GPSweek = gnsscal.date2gpswd(today)
# init threads
AccepterConnection = threading.Thread(target=Accepter)
Recivers = threading.Thread(target=Reciver)
RTKOptimization = threading.Thread(target=ReciveRTK)
# start threads
AccepterConnection.start()
Recivers.start()
RTKOptimization.start()
def main():
time.sleep(10)
RequestRTK(AproxPos)