def cfg(): path_cruise = path_on_drive_d( '/mnt/D/Work/_Python3/And0K/h5toGrid/test/csv2h5/data') # get_cfg = partial( main, [ os.path.join(scripts_path, 'ini/csv_inclin_Kondrashov.ini'), '--path', os.path.join(path_cruise, 'inclin_Kondrashov_180430.txt'), '--b_interact', 'False', '--return', '<gen_names_and_log>', '--log', os.path.join(scripts_path, 'log/csv2h5_inclin_Kondrashov.log'), '--b_skip_if_up_to_date', 'False', # to not use store '--min_date', '30.04.2018 23:59:51', # ; UTC, not output data < min_date '--max_date', '01.05.2018 00:00:05', # ; UTC, not output data > max_date ]) return get_cfg
because of zero calibration in vsz too and because it not uses Vabs coefs. """ import numpy as np from utils2init import path_on_drive_d from inclinometer.h5from_veusz_coef import main as h5from_veusz_coef from inclinometer.h5inclinometer_coef import h5copy_coef from veuszPropagate import __file__ as file_veuszPropagate probes = np.arange(10,40) #[23,30,32] 17,18 [3,12,15,19,1,13,14,16] [1,4,5,7,11,12] # [4,5,11,12] #[29, 30, 33] # [3, 14, 15, 16, 19] channels_list = ['M', 'A'] # [] # Output coefficients here: # 1. stand data (same as input for 1st step) db_path_calibr_scalling = path_on_drive_d( r'D:\WorkData\~configuration~\inclinometr\190710incl.h5' # r'd:\WorkData\_experiment\inclinometer\200807_Schukas\mag_components_calibration\200807_calibr-lab#b.h5' ) # 2. tank data db_path_tank = path_on_drive_d( # path to load calibration data: newer first r'd:\WorkData\_experiment\inclinometer\210331_tank[4,5,9,10,11,19,28,33,36,37,38]\210331incl.h5' # r'd:\WorkData\_experiment\inclinometer\200807_Schukas\200807_calibr-tank-b.h5' ) vsz_data = {'veusze': None} for i, probe in enumerate(probes): # incl_calibr not supports multiple timeranges so calculate one by one probe tbl = f'incl{probe:0>2}' # note: regex result from veusz name by re_tbl_from_vsz_name below must be same # f'incl_b{probe:0>2}' vsz_path = db_path_tank.with_name(f'{tbl}.vsz') # {db_path_tank.stem} vsz_data = h5from_veusz_coef( [str(Path(file_veuszPropagate).with_name('veuszPropagate.ini')),
from utils2init import path_on_drive_d, init_logging, open_csv_or_archive_of_them, st # l = logging.getLogger(__name__) l = init_logging(logging, None, None, 'INFO') if True: # False. Experimental speedup but takes memory from dask.cache import Cache cache = Cache(2e9) # Leverage two gigabytes of memory cache.register() # Turn cache on globally if False: # True: # False: # l.warning('using "synchronous" scheduler for debugging') import dask dask.config.set(scheduler='synchronous') # Directory where inclinometer data will be stored path_cruise = path_on_drive_d( r'd:\WorkData\BalticSea\200628_Pregolya,Lagoon-inclinometer') r""" d:\WorkData\BalticSea\200630_AI55\inclinometer d:\WorkData\_experiment\inclinometer\200610_tank_ex[4,5,7,9,10,11][3,12,13,14,15,16,19] d:\WorkData\BalticSea\200514_Pregolya,Lagoon-inclinometer d:\WorkData\BalticSea\200317_Pregolya,Lagoon-inclinometer d:\WorkData\BalticSea\191210_Pregolya,Lagoon-inclinometer d:\WorkData\_experiment\inclinometer\200117_tank[23,30,32] d:\workData\BalticSea\191119_Filino\inclinometer d:\workData\BalticSea\191108_Filino\inclinometer d:\WorkData\BlackSea\191029_Katsiveli\inclinometer d:\WorkData\_experiment\inclinometer\191106_tank_ex1[1,13,14,16] d:\WorkData\BalticSea\190806_Yantarniy\inclinometer d:\workData\BalticSea\190801inclinometer_Schuka d:\workData\BalticSea\190713_ABP45\inclinometer
# datafile = '/mnt/D/Work/_Python3/And0K/h5toGrid/test/csv2h5/data/inclin_Kondrashov.txt' import os import sys import unittest, pytest from functools import partial from to_pandas_hdf5.csv2h5 import * # main as csv2h5, __file__ as file_csv2h5, read_csv from to_pandas_hdf5.csv_specific_proc import * from to_pandas_hdf5.h5toh5 import h5init, h5del_obsolete from to_pandas_hdf5.h5_dask_pandas import filterGlobal_minmax, h5_append_dummy_row from utils2init import path_on_drive_d # import imp; imp.reload(csv2h5) r'd:/WorkData/BalticSea/180418_Svetlogorsk/inclinometer' scripts_path = path_on_drive_d( '/mnt/D/Work/_Python3/And0K/h5toGrid/scripts') # to find ini test_path = path_on_drive_d('/mnt/D/Work/_Python3/And0K/h5toGrid/test') sys.path.append(str(Path(test_path).parent.resolve())) # g.es(sys.argv[0]) # sys.argv[0] = scripts_path # to can write log to ".\log" directory # os.chdir(os.path.dirname(scripts_path)) def test_proc_loaded_nav_HYPACK_SES2000( a: Union[pd.DataFrame, np.ndarray], cfg_in: Mapping[str, Any]) -> pd.DatetimeIndex: """ Specified prep&proc of SES2000 data from program "HYPACK": - Time calc: gets string for time in current zone - Lat, Lon to degrees conversion
from inclinometer.incl_calibr import str_range, main as incl_calibr, zeroing_azimuth from inclinometer.incl_h5clc import h5_names_gen from inclinometer.h5inclinometer_coef import h5copy_coef from inclinometer.h5from_veusz_coef import main as h5from_veusz_coef from veuszPropagate import __file__ as file_veuszPropagate l = logging.getLogger(__name__) probes = [ 4, 5, 7, 9, 10, 11, 3, 12, 13, 14, 15, 16, 19 ] # [23,30,32] 17,18 [3,12,15,19,1,13,14,16] [1,4,5,7,11,12] # [4,5,11,12] #[29, 30, 33] # [3, 14, 15, 16, 19] channels_list = ['M', 'A'] # [] # stand data - input for 1st step db_path_calibr_scalling = path_on_drive_d( r'd:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe\190710incl.h5' ) r""" d:\WorkData\_experiment\_2018\inclinometer\181004_tank[1-20]\181004_KTIz.h5 # r'd:/workData/_experiment/_2019/inclinometer/190704_tank_ex2/190704incl.h5' # r'd:\WorkData\_experiment\inclinometer\190711_tank\190711incl.h5' # r'd:\WorkData\_experiment\inclinometer\190711_tank\190711incl.h5' # r'/mnt/D/workData/_experiment/_2019/inclinometer/190704_tank_ex2/190704incl.h5' """ step = 2 # one step for one program's run # --------------------------------------------------------------------------------------------- for i, probe in enumerate( probes ): # incl_calibr not supports multiple time_ranges so calculate one by one probe # tank data - used to output coefficients in both steps
import pandas as pd # import my scripts drive_d = Path('D:/' if sys.platform == 'win32' else '/mnt/D') # allows to run on both my Linux and Windows systems: scripts_path = drive_d.joinpath('Work/_Python3/And0K/h5toGrid/scripts') sys.path.append(str(Path(scripts_path).parent.resolve())) # os.getcwd() from to_pandas_hdf5.csv2h5 import main as csv2h5 from to_pandas_hdf5.csv_specific_proc import correct_kondrashov_txt from inclinometer.h5inclinometer_coef import h5copy_coef import veuszPropagate from utils_time import intervals_from_period from utils2init import path_on_drive_d path_cruise = path_on_drive_d( r'd:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe\_raw\201218' ) r""" d:\WorkData\_experiment\inclinometer\200610_tank_ex[4,5,7,9,10,11][3,12,13,14,15,16,19] d:\WorkData\_experiment\inclinometer\200117_tank[23,30,32] d:\workData\BalticSea\191119_Filino d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe\190902[3,14,15,16,19] d:\WorkData\_experiment\inclinometer\190711_tank[1,4,5,7,11,12] d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe\191009incl15_directions_test d:\WorkData\BalticSea\190806_Yantarniy\inclinometer d:\WorkData\_experiment\inclinometer\190704_tank_ex2[12,22,27,28,30,31,35] d:\WorkData\_experiment\inclinometer\190704_tank_ex1[21,23,24,25,26,29,32,34] d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe
pd.DataFrame.from_dict( dict(zip(list(txt_results['fit1result'].keys()), vsz_data[n]))).to_csv( Path(cfg['out']['path']).with_name( f'average_for_fitting-{n}.txt'), sep='\t', header=txt_results['fit1result'].keys, mode='a') return {**vsz_data, 'veusze': veusze} if __name__ == '__main__': # Example call cfg_out_db_path = path_on_drive_d( r'd:\WorkData\_experiment\inclinometer\190711_tank\190711incl.h5' # r'/mnt/D/workData/_experiment/_2019/inclinometer/190704_tank_ex2/190704incl.h5' ) # r'd:\WorkData\_experiment\_2019\inclinometer\190704\190704incl.h5' # r'd:\workData\BalticSea\190713_ABP45\inclinometer\190816incl.h5' # 190711incl.h5 cfg['out']['db_path'] cfg_in = { 'path': cfg_out_db_path.with_name( '190711incl12.vsz'), # '190704incl[0-9][0-9].vsz' # r'd:\WorkData\_experiment\_2019\inclinometer\190704\190704incl[0-9][0-9].vsz', # r'd:\WorkData\_experiment\_2019\inclinometer\190711_tank\190711incl[0-9][0-9].vsz', # r'd:\WorkData\_experiment\_2018\inclinometr\181004_KTI\incl09.vsz' 'widget': '/fitV(force)/grid1/graph/fit1/values' } # , 'out': { 'db_path': }
""" import numpy as np from utils2init import path_on_drive_d from inclinometer.h5from_veusz_coef import main as h5from_veusz_coef from inclinometer.h5inclinometer_coef import h5copy_coef from veuszPropagate import __file__ as file_veuszPropagate probes = np.arange( 1, 31 ) #[23,30,32] 17,18 [3,12,15,19,1,13,14,16] [1,4,5,7,11,12] # [4,5,11,12] #[29, 30, 33] # [3, 14, 15, 16, 19] channels_list = ['M', 'A'] # [] # Output coefficients here: # 1. stand data (same as input for 1st step) db_path_calibr_scalling = path_on_drive_d( # r'D:\WorkData\~configuration~\inclinometr\190710incl.h5' r'd:\WorkData\_experiment\inclinometer\_Schukas\210603_lab\_raw\210603incl.h5' ) # 2. tank data db_path_tank = path_on_drive_d( # path to load calibration data: newer first # r'd:\WorkData\_experiment\inclinometer\210331_tank[4,5,9,10,11,19,28,33,36,37,38]\210331incl.h5' r'd:\WorkData\_experiment\inclinometer\_Schukas\200807_tank[b01-b30]\200807_calibr-tank-b.h5' ) vsz_data = {'veusze': None} for i, probe in enumerate(probes): # incl_calibr not supports multiple time_ranges so calculate one by one probe tbl = f'incl_b{probe:0>2}' # note: regex result from veusz name by re_tbl_from_vsz_name below must be same # f'incl_b{probe:0>2}' vsz_path = db_path_tank.with_name(f'{tbl}.vsz') # {db_path_tank.stem} vsz_data = h5from_veusz_coef( [