def process_raw_log(date, bucket_name=STATIC_TEST_S3_BUCKET, local_dest=DEFAULT_SWIFT_TMP_DIR, verbose=False): base_prefix = '/builds/' path = local_dest + bucket_name + base_prefix + "/" + date new_files = [] gps_time_tabs = [ 'rover_iar_state', 'rover_logs', 'rover_tracking', 'rover_acq', 'rover_thread_state', 'rover_uart_state', 'rover_ephemerides', 'base_ephemerides' ] for root, dirnames, filenames in os.walk(path): for filename in fnmatch.filter(filenames, 'serial*.json'): if verbose: print "Processing %s to hdf5" % filename nf = hdf5_write(root + "/" + filename, root + "/" + filename + '.hdf5', verbose) with pd.HDFStore(nf) as store: if not store.rover_spp.empty: get_gps_time_col(store, gps_time_tabs, verbose=verbose) reindex_tables(store, ['rover_iar_state', 'rover_logs'], verbose=verbose) new_files.append(nf) return sorted(new_files)
def test_hdf5(): log_datafile \ = "./data/serial_link_log_20150314-190228_dl_sat_fail_test1.log.json.dat" filename = log_datafile + ".hdf5" processor = StoreToHDF5() hdf5_write(log_datafile, filename) assert os.path.isfile(filename) with pd.HDFStore(filename) as store: assert store assert isinstance(store.base_obs, pd.Panel) assert store.base_obs.shape == (91, 6, 5) assert store.base_obs[0, :, :].to_dict()\ == {10: {'L': 10374.46875, 'P': 22165269.800000001, 'cn0': 22.0, 'lock': 3381.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}, 12: {'L': 64486.11328125, 'P': 20302353.989999998, 'cn0': 46.0, 'lock': 63239.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}, 14: {'L': 120079.03125, 'P': 22980000.0, 'cn0': 18.0, 'lock': 64615.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}, 16: {'L': 99532.98828125, 'P': 20753378.68, 'cn0': 48.0, 'lock': 31377.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}, 27: {'L': -5162.3359375, 'P': 19956188.879999999, 'cn0': 18.0, 'lock': 1482.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}} assert isinstance(store.ephemerides, pd.Panel) assert store.ephemerides.shape == (2, 29, 3) assert isinstance(store.rover_obs, pd.Panel) assert store.rover_obs.shape == (3, 6, 8) assert store.rover_obs[0, :, :].to_dict() \ == {0: {'L': 49613.0, 'P': 22980000.0, 'cn0': 14.0, 'lock': 10583.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 6: {'L': -154459.46484375, 'P': 21443899.43, 'cn0': 29.0, 'lock': 24867.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 10: {'L': -4688.3203125, 'P': 22089093.379999999, 'cn0': 14.0, 'lock': 24238.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 12: {'L': 82371.0859375, 'P': 20239084.489999998, 'cn0': 41.0, 'lock': 55100.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 14: {'L': 161655.17578125, 'P': 22931626.530000001, 'cn0': 21.0, 'lock': 6171.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 16: {'L': 165409.55078125, 'P': 20705699.739999998, 'cn0': 37.0, 'lock': 60164.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 27: {'L': -33727.6875, 'P': 19874306.629999999, 'cn0': 39.0, 'lock': 21822.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 29: {'L': -80021.1796875, 'P': 19864244.440000001, 'cn0': 50.0, 'lock': 42862.0, 'host_offset': 0.0, 'host_time': 1426384948.0}} assert isinstance(store.rover_rtk_ned, pd.DataFrame) assert store.rover_rtk_ned.shape == (0, 0) assert isinstance(store.rover_rtk_ecef, pd.DataFrame) assert store.rover_rtk_ecef.shape == (0, 0) assert isinstance(store.rover_spp, pd.DataFrame) assert store.rover_spp.shape == (9, 5) assert store.rover_spp.ix[['n_sats', 'x']].to_dict() \ == {Timestamp('2015-03-15 02:02:23.600000'): {'n_sats': 8.0, 'x': -2704372.4505344247}, Timestamp('2015-03-15 02:02:23.700000'): {'n_sats': 8.0, 'x': -2704372.2245847895}, Timestamp('2015-03-15 02:02:23.800000'): {'n_sats': 8.0, 'x': -2704372.5723111397}, Timestamp('2015-03-15 02:02:23.900000'): {'n_sats': 8.0, 'x': -2704371.0836650538}, Timestamp('2015-03-15 02:02:24'): {'n_sats': 8.0, 'x': -2704370.9813769697}} assert isinstance(store.rover_tracking, pd.Panel) assert store.rover_tracking.shape == (19, 5, 318) rd = store.rover_tracking[:, :, 0].to_dict() assert rd.keys() == [0, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 19, 20, 21, 25, 27, 29] assert rd[0] \ == {'cn0': 3.7272727489471436, 'prn': 0.0, 'state': 1.0, 'host_offset': 0.0, 'host_time': 1426384948.0} assert rd[29] \ == {'cn0': 12.685534477233887, 'prn': 29.0, 'state': 1.0, 'host_offset': 0.0, 'host_time': 1426384948.0} assert isinstance(store.rover_iar_state, pd.DataFrame) store.rover_iar_state.shape == (0, 0)
def test_hdf5(): log_datafile \ = "./data/serial_link_log_20150314-190228_dl_sat_fail_test1.log.json.dat" filename = log_datafile + ".hdf5" processor = StoreToHDF5() hdf5_write(log_datafile, filename) assert os.path.isfile(filename) with pd.HDFStore(filename) as store: assert store assert isinstance(store.base_obs, pd.Panel) assert store.base_obs.shape == (91, 6, 5) assert store.base_obs[0, :, :].to_dict()\ == {10: {'L': 10374.46875, 'P': 22165269.800000001, 'cn0': 22.0, 'lock': 3381.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}, 12: {'L': 64486.11328125, 'P': 20302353.989999998, 'cn0': 46.0, 'lock': 63239.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}, 14: {'L': 120079.03125, 'P': 22980000.0, 'cn0': 18.0, 'lock': 64615.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}, 16: {'L': 99532.98828125, 'P': 20753378.68, 'cn0': 48.0, 'lock': 31377.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}, 27: {'L': -5162.3359375, 'P': 19956188.879999999, 'cn0': 18.0, 'lock': 1482.0, 'host_offset': 58353.0, 'host_time': 1426385006.0}} assert isinstance(store.ephemerides, pd.Panel) assert store.ephemerides.shape == (2, 29, 3) assert isinstance(store.rover_obs, pd.Panel) assert store.rover_obs.shape == (3, 6, 8) assert store.rover_obs[0, :, :].to_dict() \ == {0: {'L': 49613.0, 'P': 22980000.0, 'cn0': 14.0, 'lock': 10583.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 6: {'L': -154459.46484375, 'P': 21443899.43, 'cn0': 29.0, 'lock': 24867.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 10: {'L': -4688.3203125, 'P': 22089093.379999999, 'cn0': 14.0, 'lock': 24238.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 12: {'L': 82371.0859375, 'P': 20239084.489999998, 'cn0': 41.0, 'lock': 55100.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 14: {'L': 161655.17578125, 'P': 22931626.530000001, 'cn0': 21.0, 'lock': 6171.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 16: {'L': 165409.55078125, 'P': 20705699.739999998, 'cn0': 37.0, 'lock': 60164.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 27: {'L': -33727.6875, 'P': 19874306.629999999, 'cn0': 39.0, 'lock': 21822.0, 'host_offset': 0.0, 'host_time': 1426384948.0}, 29: {'L': -80021.1796875, 'P': 19864244.440000001, 'cn0': 50.0, 'lock': 42862.0, 'host_offset': 0.0, 'host_time': 1426384948.0}} assert isinstance(store.rover_rtk_ned, pd.DataFrame) assert store.rover_rtk_ned.shape == (0, 0) assert isinstance(store.rover_rtk_ecef, pd.DataFrame) assert store.rover_rtk_ecef.shape == (0, 0) assert isinstance(store.rover_spp, pd.DataFrame) assert store.rover_spp.shape == (9, 5) assert store.rover_spp.ix[['n_sats', 'x']].to_dict() \ == {Timestamp('2015-03-15 02:02:23.600000'): {'n_sats': 8.0, 'x': -2704372.4505344247}, Timestamp('2015-03-15 02:02:23.700000'): {'n_sats': 8.0, 'x': -2704372.2245847895}, Timestamp('2015-03-15 02:02:23.800000'): {'n_sats': 8.0, 'x': -2704372.5723111397}, Timestamp('2015-03-15 02:02:23.900000'): {'n_sats': 8.0, 'x': -2704371.0836650538}, Timestamp('2015-03-15 02:02:24'): {'n_sats': 8.0, 'x': -2704370.9813769697}} assert isinstance(store.rover_tracking, pd.Panel) assert store.rover_tracking.shape == (19, 5, 318) rd = store.rover_tracking[:, :, 0].to_dict() assert rd.keys() == [ 0, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 19, 20, 21, 25, 27, 29 ] assert rd[0] \ == {'cn0': 3.7272727489471436, 'prn': 0.0, 'state': 1.0, 'host_offset': 0.0, 'host_time': 1426384948.0} assert rd[29] \ == {'cn0': 12.685534477233887, 'prn': 29.0, 'state': 1.0, 'host_offset': 0.0, 'host_time': 1426384948.0} assert isinstance(store.rover_iar_state, pd.DataFrame) store.rover_iar_state.shape == (0, 0)