def cfg():
path_cruise = path_on_drive_d(
'/mnt/D/Work/_Python3/And0K/h5toGrid/test/csv2h5/data') #
get_cfg = partial(
main,
[
os.path.join(scripts_path, 'ini/csv_inclin_Kondrashov.ini'),
'--path',
os.path.join(path_cruise, 'inclin_Kondrashov_180430.txt'),
'--b_interact',
'False',
'--return',
'<gen_names_and_log>',
'--log',
os.path.join(scripts_path, 'log/csv2h5_inclin_Kondrashov.log'),
'--b_skip_if_up_to_date',
'False', # to not use store
'--min_date',
'30.04.2018 23:59:51', # ; UTC, not output data < min_date
'--max_date',
'01.05.2018 00:00:05', # ; UTC, not output data > max_date
])
return get_cfg
because of zero calibration in vsz too and because it not uses Vabs coefs.
"""
import numpy as np
from utils2init import path_on_drive_d
from inclinometer.h5from_veusz_coef import main as h5from_veusz_coef
from inclinometer.h5inclinometer_coef import h5copy_coef
from veuszPropagate import __file__ as file_veuszPropagate
probes = np.arange(10,40) #[23,30,32] 17,18 [3,12,15,19,1,13,14,16] [1,4,5,7,11,12] # [4,5,11,12] #[29, 30, 33] # [3, 14, 15, 16, 19]
channels_list = ['M', 'A'] # []
# Output coefficients here:
# 1. stand data (same as input for 1st step)
db_path_calibr_scalling = path_on_drive_d(
r'D:\WorkData\~configuration~\inclinometr\190710incl.h5'
# r'd:\WorkData\_experiment\inclinometer\200807_Schukas\mag_components_calibration\200807_calibr-lab#b.h5'
)
# 2. tank data
db_path_tank = path_on_drive_d( # path to load calibration data: newer first
r'd:\WorkData\_experiment\inclinometer\210331_tank[4,5,9,10,11,19,28,33,36,37,38]\210331incl.h5'
# r'd:\WorkData\_experiment\inclinometer\200807_Schukas\200807_calibr-tank-b.h5'
)
vsz_data = {'veusze': None}
for i, probe in enumerate(probes):
# incl_calibr not supports multiple timeranges so calculate one by one probe
tbl = f'incl{probe:0>2}' # note: regex result from veusz name by re_tbl_from_vsz_name below must be same
# f'incl_b{probe:0>2}'
vsz_path = db_path_tank.with_name(f'{tbl}.vsz') # {db_path_tank.stem}
vsz_data = h5from_veusz_coef(
[str(Path(file_veuszPropagate).with_name('veuszPropagate.ini')),
from utils2init import path_on_drive_d, init_logging, open_csv_or_archive_of_them, st
# l = logging.getLogger(__name__)
l = init_logging(logging, None, None, 'INFO')
if True: # False. Experimental speedup but takes memory
from dask.cache import Cache
cache = Cache(2e9) # Leverage two gigabytes of memory
cache.register() # Turn cache on globally
if False: # True: # False: #
l.warning('using "synchronous" scheduler for debugging')
import dask
dask.config.set(scheduler='synchronous')
# Directory where inclinometer data will be stored
path_cruise = path_on_drive_d(
r'd:\WorkData\BalticSea\200628_Pregolya,Lagoon-inclinometer')
r"""
d:\WorkData\BalticSea\200630_AI55\inclinometer
d:\WorkData\_experiment\inclinometer\200610_tank_ex[4,5,7,9,10,11][3,12,13,14,15,16,19]
d:\WorkData\BalticSea\200514_Pregolya,Lagoon-inclinometer
d:\WorkData\BalticSea\200317_Pregolya,Lagoon-inclinometer
d:\WorkData\BalticSea\191210_Pregolya,Lagoon-inclinometer
d:\WorkData\_experiment\inclinometer\200117_tank[23,30,32]
d:\workData\BalticSea\191119_Filino\inclinometer
d:\workData\BalticSea\191108_Filino\inclinometer
d:\WorkData\BlackSea\191029_Katsiveli\inclinometer
d:\WorkData\_experiment\inclinometer\191106_tank_ex1[1,13,14,16]
d:\WorkData\BalticSea\190806_Yantarniy\inclinometer
d:\workData\BalticSea\190801inclinometer_Schuka
d:\workData\BalticSea\190713_ABP45\inclinometer
# datafile = '/mnt/D/Work/_Python3/And0K/h5toGrid/test/csv2h5/data/inclin_Kondrashov.txt'
import os
import sys
import unittest, pytest
from functools import partial
from to_pandas_hdf5.csv2h5 import * # main as csv2h5, __file__ as file_csv2h5, read_csv
from to_pandas_hdf5.csv_specific_proc import *
from to_pandas_hdf5.h5toh5 import h5init, h5del_obsolete
from to_pandas_hdf5.h5_dask_pandas import filterGlobal_minmax, h5_append_dummy_row
from utils2init import path_on_drive_d
# import imp; imp.reload(csv2h5)
r'd:/WorkData/BalticSea/180418_Svetlogorsk/inclinometer'
scripts_path = path_on_drive_d(
'/mnt/D/Work/_Python3/And0K/h5toGrid/scripts') # to find ini
test_path = path_on_drive_d('/mnt/D/Work/_Python3/And0K/h5toGrid/test')
sys.path.append(str(Path(test_path).parent.resolve()))
# g.es(sys.argv[0])
# sys.argv[0] = scripts_path # to can write log to ".\log" directory
# os.chdir(os.path.dirname(scripts_path))
def test_proc_loaded_nav_HYPACK_SES2000(
a: Union[pd.DataFrame,
np.ndarray], cfg_in: Mapping[str, Any]) -> pd.DatetimeIndex:
"""
Specified prep&proc of SES2000 data from program "HYPACK":
- Time calc: gets string for time in current zone
- Lat, Lon to degrees conversion
from inclinometer.incl_calibr import str_range, main as incl_calibr, zeroing_azimuth
from inclinometer.incl_h5clc import h5_names_gen
from inclinometer.h5inclinometer_coef import h5copy_coef
from inclinometer.h5from_veusz_coef import main as h5from_veusz_coef
from veuszPropagate import __file__ as file_veuszPropagate
l = logging.getLogger(__name__)
probes = [
4, 5, 7, 9, 10, 11, 3, 12, 13, 14, 15, 16, 19
] # [23,30,32] 17,18 [3,12,15,19,1,13,14,16] [1,4,5,7,11,12] # [4,5,11,12] #[29, 30, 33] # [3, 14, 15, 16, 19]
channels_list = ['M', 'A'] # []
# stand data - input for 1st step
db_path_calibr_scalling = path_on_drive_d(
r'd:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe\190710incl.h5'
)
r"""
d:\WorkData\_experiment\_2018\inclinometer\181004_tank[1-20]\181004_KTIz.h5
# r'd:/workData/_experiment/_2019/inclinometer/190704_tank_ex2/190704incl.h5'
# r'd:\WorkData\_experiment\inclinometer\190711_tank\190711incl.h5'
# r'd:\WorkData\_experiment\inclinometer\190711_tank\190711incl.h5'
# r'/mnt/D/workData/_experiment/_2019/inclinometer/190704_tank_ex2/190704incl.h5'
"""
step = 2 # one step for one program's run
# ---------------------------------------------------------------------------------------------
for i, probe in enumerate(
probes
): # incl_calibr not supports multiple time_ranges so calculate one by one probe
# tank data - used to output coefficients in both steps
import pandas as pd
# import my scripts
drive_d = Path('D:/' if sys.platform == 'win32' else
'/mnt/D') # allows to run on both my Linux and Windows systems:
scripts_path = drive_d.joinpath('Work/_Python3/And0K/h5toGrid/scripts')
sys.path.append(str(Path(scripts_path).parent.resolve())) # os.getcwd()
from to_pandas_hdf5.csv2h5 import main as csv2h5
from to_pandas_hdf5.csv_specific_proc import correct_kondrashov_txt
from inclinometer.h5inclinometer_coef import h5copy_coef
import veuszPropagate
from utils_time import intervals_from_period
from utils2init import path_on_drive_d
path_cruise = path_on_drive_d(
r'd:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe\_raw\201218'
)
r"""
d:\WorkData\_experiment\inclinometer\200610_tank_ex[4,5,7,9,10,11][3,12,13,14,15,16,19]
d:\WorkData\_experiment\inclinometer\200117_tank[23,30,32]
d:\workData\BalticSea\191119_Filino
d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe
d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe\190902[3,14,15,16,19]
d:\WorkData\_experiment\inclinometer\190711_tank[1,4,5,7,11,12]
d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe
d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe\191009incl15_directions_test
d:\WorkData\BalticSea\190806_Yantarniy\inclinometer
d:\WorkData\_experiment\inclinometer\190704_tank_ex2[12,22,27,28,30,31,35]
d:\WorkData\_experiment\inclinometer\190704_tank_ex1[21,23,24,25,26,29,32,34]
d:\WorkData\_experiment\inclinometer\190710_compas_calibr-byMe
pd.DataFrame.from_dict(
dict(zip(list(txt_results['fit1result'].keys()),
vsz_data[n]))).to_csv(
Path(cfg['out']['path']).with_name(
f'average_for_fitting-{n}.txt'),
sep='\t',
header=txt_results['fit1result'].keys,
mode='a')
return {**vsz_data, 'veusze': veusze}
if __name__ == '__main__':
# Example call
cfg_out_db_path = path_on_drive_d(
r'd:\WorkData\_experiment\inclinometer\190711_tank\190711incl.h5'
# r'/mnt/D/workData/_experiment/_2019/inclinometer/190704_tank_ex2/190704incl.h5'
)
# r'd:\WorkData\_experiment\_2019\inclinometer\190704\190704incl.h5'
# r'd:\workData\BalticSea\190713_ABP45\inclinometer\190816incl.h5'
# 190711incl.h5 cfg['out']['db_path']
cfg_in = {
'path': cfg_out_db_path.with_name(
'190711incl12.vsz'), # '190704incl[0-9][0-9].vsz'
# r'd:\WorkData\_experiment\_2019\inclinometer\190704\190704incl[0-9][0-9].vsz',
# r'd:\WorkData\_experiment\_2019\inclinometer\190711_tank\190711incl[0-9][0-9].vsz',
# r'd:\WorkData\_experiment\_2018\inclinometr\181004_KTI\incl09.vsz'
'widget': '/fitV(force)/grid1/graph/fit1/values'
}
# , 'out': { 'db_path': }
"""
import numpy as np
from utils2init import path_on_drive_d
from inclinometer.h5from_veusz_coef import main as h5from_veusz_coef
from inclinometer.h5inclinometer_coef import h5copy_coef
from veuszPropagate import __file__ as file_veuszPropagate
probes = np.arange(
1, 31
) #[23,30,32] 17,18 [3,12,15,19,1,13,14,16] [1,4,5,7,11,12] # [4,5,11,12] #[29, 30, 33] # [3, 14, 15, 16, 19]
channels_list = ['M', 'A'] # []
# Output coefficients here:
# 1. stand data (same as input for 1st step)
db_path_calibr_scalling = path_on_drive_d(
# r'D:\WorkData\~configuration~\inclinometr\190710incl.h5'
r'd:\WorkData\_experiment\inclinometer\_Schukas\210603_lab\_raw\210603incl.h5'
)
# 2. tank data
db_path_tank = path_on_drive_d( # path to load calibration data: newer first
# r'd:\WorkData\_experiment\inclinometer\210331_tank[4,5,9,10,11,19,28,33,36,37,38]\210331incl.h5'
r'd:\WorkData\_experiment\inclinometer\_Schukas\200807_tank[b01-b30]\200807_calibr-tank-b.h5'
)
vsz_data = {'veusze': None}
for i, probe in enumerate(probes):
# incl_calibr not supports multiple time_ranges so calculate one by one probe
tbl = f'incl_b{probe:0>2}' # note: regex result from veusz name by re_tbl_from_vsz_name below must be same
# f'incl_b{probe:0>2}'
vsz_path = db_path_tank.with_name(f'{tbl}.vsz') # {db_path_tank.stem}
vsz_data = h5from_veusz_coef(
[
