def test_invalid_name(tmp_path, rnx_sample): sample_path = tmp_path / 'sample' shutil.copy(rnx_sample, sample_path) with pytest.raises(ValueError) as excinfo: decompress_on_disk(sample_path) msg = excinfo.value.args[0] assert msg.endswith('is not a valid RINEX file name')
def test_on_disk_invalid_input(tmp_path): path = tmp_path / 'sample.crx' path.write_bytes(b'blah' * 100) with pytest.raises(ValueError) as excinfo: decompress_on_disk(path) msg = excinfo.value.args[0] assert 'not a valid RINEX file' in msg assert not get_compressed_path(path).exists()
def test_on_disk_truncated_input(tmp_path): in_file = 'sample.crx' sample_path = tmp_path / in_file sample_path.write_bytes(get_data_path(in_file).read_bytes()[:200]) with pytest.raises(HatanakaException) as excinfo: decompress_on_disk(sample_path) msg = excinfo.value.args[0] assert 'truncated in the middle' in msg assert not get_compressed_path(sample_path).exists()
def test_on_disk_empty_input(tmp_path, crx_str, rnx_bytes): path = tmp_path / 'sample.crx' path.write_bytes(b'') with pytest.raises(ValueError) as excinfo: decompress_on_disk(path) assert "empty file" in str(excinfo.value) assert not get_decompressed_path(path).exists() path.unlink() path = tmp_path / 'sample.rnx' path.write_bytes(b'') with pytest.raises(ValueError) as excinfo: compress_on_disk(path) assert "file is too short" in str(excinfo.value) assert not get_compressed_path(path, is_obs=True).exists()
def rnpath(inps): # Let us start by parsing out all relevant user configuration inputs. yy = inps['dtstart_yy'] # The starting two-digit year number doy = inps['dtstart_doy'] # The starting day of year in GPST cwd = Path(inps['cwd']) # Current working directory iwd = cwd / 'input' name1 = inps['name1'] # 4-letter ID of the first spacecraft name2 = inps['name2'] # 4-letter ID of the second spacecraft # Now, we get all necessary file paths, in the input folder. rnx1file = iwd / (name1 + doy + '0.' + yy + 'O') rnx2file = iwd / (name2 + doy + '0.' + yy + 'O') crx1file = iwd / (name1 + doy + '0.' + yy + 'D') crx2file = iwd / (name2 + doy + '0.' + yy + 'D') # Check if there is a need for hatanaka decompression for LEO 1. if rnx1file.exists(): print('Decompressed RINEX obs file observed for ' + name1 + '\n') elif crx1file.exists(): print('Hatanaka compressed file observed for ' + name1) hatanaka.decompress_on_disk(crx1file) if rnx1file.exists(): print('Hatanaka decompression successful for LEO1! \n') else: print('Decompression failed. Did you rename any folders? \n') else: print('Neither compressed nor decompressed RINEX files were found! \n') # Check if there is a need for hatanaka decompression for LEO 2. if rnx2file.exists(): print('Decompressed RINEX obs file observed for ' + name2 + '\n') elif crx2file.exists(): print('Hatanaka compressed file observed for ' + name2) hatanaka.decompress_on_disk(crx2file) if rnx2file.exists(): print('Hatanaka decompression successful for LEO2! \n') else: print('Decompression failed. Did you rename any folders? \n') else: print('Neither compressed nor decompressed RINEX files were found! \n') if rnx1file.exists() and rnx2file.exists(): return rnx1file, rnx2file else: print('Error, somehow RINEX observation files still not found!') print('Did you rename any folders accidentally? \n') return False
def ftp_download_files(url_base, folder_path, cacheDir, filenames, compression='', overwrite=False): """ Like download file, but more of them. Keeps a persistent FTP connection open to be more efficient. """ folder_path_abs = os.path.join(cacheDir, folder_path) ftp = ftp_connect(url_base + folder_path) filepaths = [] for filename in filenames: filename_zipped = filename + compression filepath = str(hatanaka.get_decompressed_path(os.path.join(folder_path_abs, filename))) filepath_zipped = os.path.join(folder_path_abs, filename_zipped) print("pulling from", url_base, "to", filepath) if not os.path.isfile(filepath) or overwrite: if not os.path.exists(folder_path_abs): os.makedirs(folder_path_abs) try: ftp.retrbinary('RETR ' + filename_zipped, open(filepath_zipped, 'wb').write) except (ftplib.error_perm): raise IOError("Could not download file from: " + url_base + folder_path + filename_zipped) except (socket.timeout): raise IOError("Read timed out from: " + url_base + folder_path + filename_zipped) filepaths.append(str(hatanaka.decompress_on_disk(filepath_zipped))) else: filepaths.append(filepath) return filepaths
def download_and_cache_file(url_base, folder_path, cacheDir, filename, compression='', overwrite=False): folder_path_abs = os.path.join(cacheDir, folder_path) filename_zipped = filename + compression filepath = str(hatanaka.get_decompressed_path(os.path.join(folder_path_abs, filename))) filepath_attempt = filepath + '.attempt_time' filepath_zipped = os.path.join(folder_path_abs, filename_zipped) if os.path.exists(filepath_attempt): with open(filepath_attempt, 'rb') as rf: last_attempt_time = float(rf.read().decode()) if time.time() - last_attempt_time < SECS_IN_HR: raise IOError(f"Too soon to try {folder_path + filename_zipped} from {url_base} ") if not os.path.isfile(filepath) or overwrite: if not os.path.exists(folder_path_abs): os.makedirs(folder_path_abs) try: data_zipped = download_file(url_base, folder_path, filename_zipped) except (IOError, pycurl.error, socket.timeout): unix_time = time.time() if not os.path.exists(cacheDir + 'tmp/'): os.makedirs(cacheDir + '/tmp') with tempfile.NamedTemporaryFile(delete=False, dir=cacheDir+'tmp/') as fout: fout.write(str.encode(str(unix_time))) os.replace(fout.name, filepath + '.attempt_time') raise IOError(f"Could not download {folder_path + filename_zipped} from {url_base} ") with open(filepath_zipped, 'wb') as wf: wf.write(data_zipped) filepath = str(hatanaka.decompress_on_disk(filepath_zipped)) return filepath
def test_decompress_on_disk_delete(tmp_path, rnx_bytes): # prepare in_file = 'sample.crx.gz' sample_path = tmp_path / in_file shutil.copy(get_data_path(in_file), sample_path) # decompress and delete out_path = decompress_on_disk(sample_path, delete=True) # check expected_path = tmp_path / 'sample.rnx' assert not sample_path.exists() assert out_path == expected_path assert expected_path.exists() assert clean(decompress(expected_path)) == clean(rnx_bytes) # check that already decompressed is not deleted out_path = decompress_on_disk(expected_path, delete=True) assert out_path == expected_path assert out_path.exists()
def test_decompress_non_obs(tmp_path, rnx_bytes, input_suffix): # prepare txt = make_nav(rnx_bytes) sample_path = tmp_path / ('sample' + input_suffix + '.gz') sample_path.write_bytes(gzip.compress(txt)) # decompress out_path = decompress_on_disk(sample_path) # check assert out_path.exists() assert out_path == tmp_path / ('sample' + input_suffix) assert clean(out_path.read_bytes()) == clean(txt)
def test_decompress_on_disk(tmp_path, crx_sample, rnx_bytes, input_suffix, expected_suffix): # prepare sample_path = tmp_path / ('sample' + input_suffix) in_file = 'sample' + input_suffix shutil.copy(get_data_path(in_file), sample_path) # decompress out_path = decompress_on_disk(sample_path) # check assert out_path.exists() print(list(tmp_path.glob('*'))) assert out_path == tmp_path / ('sample' + expected_suffix) assert clean(out_path.read_bytes()) == clean(rnx_bytes)
def download_corr_files(out_path, base_stn, obs, nav): """ This function downloads the necessary base station and satellite files (broadcast/orbits) for correcting raw rinex files using rtkpost. """ # file root name name = obs.split('.obs')[0] # first we need the year, doy, and GPS week from the observations file f = open(out_path + obs, 'r').read() print("Downloading correction data for {}".format(obs)) lines = f.strip().splitlines() for l in lines: # read from RINEX header if l.endswith('TIME OF FIRST OBS'): var = l.split() yr = var[0] # use datetime and gnsscal to convert the date to doy and gps week date = datetime.date(int(yr), int(var[1]), int(var[2])) doy = str(gnsscal.date2doy(date)) if not len(doy) == 3: # add the leading 0 if DOY is < 3 digits doy = '0' + doy week = str(gnsscal.date2gpswd(date)[0]) + str( gnsscal.date2gpswd(date)[1]) # create a new folder to hold the correction data and move the rinex files into it path = out_path + yr + doy + '_' + name + '/' if not os.path.exists(path): os.mkdir(path) shutil.move(out_path + obs, path + obs) shutil.move(out_path + nav, path + nav) # download the base station observations, broadcast navigation, and satellite orbital clocks # NOTE: we log into and out of the FTP server each time to avoid some hang-ups in downloading # also this could allow for replacement of the server to different locations for each file # connect to the FTP for getting the base station observations f = ftp.FTP('igs.ensg.ign.fr') try: f.login() print('logged into ftp') except: print('hmm couldn\'t connect to base station ftp. no internet?') sys.exit() # navigate to the directory f.cwd('/pub/igs/data/' + yr + '/' + doy + '/') # download the base station using the leading identifier and wildcards # note we use the 30 second decimated data (*30S* wildcard below), this is # available for most sites filematch = base_stn + '*30S*.crx.gz' for filename in f.nlst(filematch): target_file_name = os.path.join(path, os.path.basename(filename)) with open(target_file_name, 'wb') as fhandle: f.retrbinary('RETR %s' % filename, fhandle.write) # quit and close server connection f.quit() f.close() # also decompress the file decompressed_path = hatanaka.decompress_on_disk(path + filename, delete=True) # final filename baseSTN = path + filename.split('crx')[0] + yr[2:] + 'o' shutil.move(decompressed_path, baseSTN) # grab the broadcast navigation data from the same directory f = ftp.FTP('igs.ensg.ign.fr') try: f.login() print('logged into ftp') except: print('hmm couldn\'t connect to base station ftp. no internet?') sys.exit() # navigate to the directory f.cwd('/pub/igs/data/' + yr + '/' + doy + '/') # get the filematch filename = 'brdc' + doy + '0.' + yr[2:] + 'n.Z' target_file_name = os.path.join(path, os.path.basename(filename)) with open(target_file_name, 'wb') as fhandle: f.retrbinary('RETR %s' % filename, fhandle.write) # quit and close server connection f.quit() f.close() # decompress the file decompressed_path = hatanaka.decompress_on_disk(path + filename, delete=True) # final filename brdc = str(decompressed_path) # finally grab the satellite precise orbits from a different directory f = ftp.FTP('igs.ensg.ign.fr') try: f.login() print('logged into ftp') except: print('hmm couldn\'t connect to base station ftp. no internet?') sys.exit() # navigate to the directory f.cwd('/pub/igs/products/' + week[0:4] + '/') # we try with the rapid orbits, if they're available try: filename = 'igr' + week + '.sp3.Z' target_file_name = os.path.join(path, os.path.basename(filename)) with open(target_file_name, 'wb') as fhandle: f.retrbinary('RETR %s' % filename, fhandle.write) # retry with the ultra-rapid orbits if that didn't work except: filename = 'igu' + week + '_18.sp3.Z' # arbitrarily taking the final ultra-rapid file (18:00) target_file_name = os.path.join(path, os.path.basename(filename)) with open(target_file_name, 'wb') as fhandle: f.retrbinary('RETR %s' % filename, fhandle.write) # decompress the file decompressed_path = hatanaka.decompress_on_disk(path + filename, delete=True) # final filename orbits = str(decompressed_path) # quit and close server connection for good f.quit() f.close() return yr, doy, week, baseSTN, brdc, orbits