def main(new_arg):
cfg = cfg_from_args(my_argparser(), new_arg)
if not cfg:
return
if cfg['program']['return'] == '<cfg_from_args>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'],
cfg['program']['verbose'])
print('\n' + this_prog_basename(__file__), end=' started. ')
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(**cfg['in'],
b_interact=cfg['program']['b_interact'])
except Ex_nothing_done as e:
print(e.message)
return ()
# cfg = {'in': {}}
# cfg['in']['path'] = \
# r'd:\workData\BalticSea\181005_ABP44\navigation\2018-10-06tracks_copy.gpx'
# r'd:\WorkData\_experiment\_2017\tracker\170502.gpx'
# r'd:\workData\_experiment\2016\GPS_tracker\sms_backup\sms-20160225135922.gpx'
for ifile, nameFull in enumerate(cfg['in']['paths'], start=1):
print('{}. {}'.format(ifile, nameFull), end=', ')
gpx2csv(nameFull)
def main_cfg(cfg: DictConfig): # hydra required arg, not use when call
"""
----------------------------
Add data from CSV-like files
to Pandas HDF5 store*.h5
----------------------------
"""
#print(OmegaConf.to_yaml(cfg))
global lf
# cfg = cfg_from_args(argparser_files(), **kwargs)
if not cfg.program.return_:
print('Can not initialise')
return cfg
elif cfg.program.return_ == '<cfg_from_args>': # to help testing
return cfg
lf = LoggingStyleAdapter(init_logging(logging, None, cfg.program.log, cfg.rogram.verbose))
print('\n' + this_prog_basename(__file__), end=' started. ')
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in']['path'] = init_file_names(
**cfg['in'], b_interact=cfg['program']['b_interact'])
except Ex_nothing_done as e:
print(e.message)
return ()
return cfg
def main_init(cfg: DictConfig) -> DictConfig:
"""
Common startup initializer
:param cfg:
:return:
"""
#global lf
# if cfg.search_path is not None:
# override_path = hydra.utils.to_absolute_path(cfg.search_path)
# override_conf = OmegaConf.load(override_path)
# cfg = OmegaConf.merge(cfg, override_conf)
print("Working directory : {}".format(os.getcwd()))
print(OmegaConf.to_yaml(cfg))
# cfg = cfg_from_args(argparser_files(), **kwargs)
if not cfg.program.return_:
print('Can not initialise')
return cfg
elif cfg.program.return_ == '<cfg_from_args>': # to help testing
return cfg
hydra.verbose = 1 if cfg.program.verbose == 'DEBUG' else 0 # made compatible to my old cfg
print('\n' + this_prog_basename(__file__), end=' started. ')
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(**cfg.input,
b_interact=cfg.program.b_interact)
except Ex_nothing_done as e:
print(e.message)
return ()
return cfg
def main(new_arg=None):
new_arg = [
r'.\h5toGpx_CTDs.ini', '--db_path',
r'd:\workData\BalticSea\170614_ANS34\170614Strahov.h5', '--path',
r'd:\workData\BalticSea\170614_ANS34\Baklan\2017*p1d5.txt',
'--gpx_names_fun_format', '+{:02d}', '--gpx_symbols_list',
"'Navaid, Orange'"
] # 'db_path', r'd:\workData\BalticSea\171003_ANS36\171003Strahov.h5'
cfg = cfg_from_args(my_argparser(), new_arg)
if not cfg:
return
if new_arg == '<return_cfg>': # to help testing
return cfg
print('\n' + this_prog_basename(__file__), 'started', end=' ')
if not cfg['out']['path'].is_absolute():
cfg['out']['path'] = cfg['in']['db_path'].parent / cfg['out'][
'path'] # set relative to cfg['in']['db_path']
try:
print(end='Data ')
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(**cfg['in']) # may interact
except Ex_nothing_done as e:
print(e.message)
return # or raise FileNotFoundError?
itbl = 0
# compile functions if defined in cfg or assign default
gpx_symbols = init_gpx_symbols_fun(cfg['out'])
gpx_names_funs = ["i+1"]
gpx_names_fun = eval(
compile(
"lambda i, row: '{}'.format({})".format(
cfg['out']['gpx_names_fun_format'], gpx_names_funs[itbl]), [],
'eval'))
tim = filename2date([f for f in ge_names(cfg)])
with pd.HDFStore(cfg['in']['db_path'], mode='r') as storeIn:
# dfL = storeIn[tblD + '/logFiles']
nav2add = h5select(storeIn, cfg['in']['table_nav'],
['Lat', 'Lon', 'DepEcho'], tim)[0]
rnav_df_join = nav2add.assign(
itbl=itbl) # copy/append on first/next cycle
# Save to gpx waypoints
# if 'gpx_names_funs' in cfg['out'] and \
# len(cfg['out']['gpx_names_funs'])>itbl:
#
# gpx_names = eval(compile('lambda i: str({})'.format(
# cfg['out']['gpx_names_funs'][itbl]), [], 'eval'))
#
save_to_gpx(rnav_df_join[-len(nav2add):],
cfg['out']['path'].with_name('fileNames'),
gpx_obj_namef=gpx_names_fun,
waypoint_symbf=gpx_symbols,
cfg_proc=cfg['process'])
def main():
print('\n' + this_prog_basename(__file__), end=' started. ')
cfg = veuszPropagate.main([
Path(veuszPropagate.__file__).parent.with_name('veuszPropagate.ini'),
'--data_yield_prefix',
'-',
'--path',
str(path_db
), # use for custom loading from db and some source is required
'--pattern_path',
str(
path_db.parent.joinpath(
r'CTD_S&S48M#1253\profiles\181005_1810-1813.vsz')),
# '--before_next', 'restore_config',
# '--add_custom_list',
# 'USEtime',
# '--add_custom_expressions_list',
# """
# "[['2018-10-03T17:23:00', '2018-10-03T18:25:00']]"
# """,
'--b_update_existed',
'True',
# '--export_pages_int_list', '',
'--b_interact',
'0',
# '--b_images_only', 'True'
])
if not cfg:
return 1
# Custom loading from db
cfg['in']['table_log'] = '/CTD_SST_48M/logRuns'
cfg['in']['db_path'] = path_db
gen_veusz_and_logs = veuszPropagate.load_to_veusz(
h5log_names_gen(cfg['in']), cfg)
cor_savings = veuszPropagate.co_savings(cfg)
cor_savings.send(None)
cor_send_data = veuszPropagate.co_send_data(gen_veusz_and_logs, cfg,
cor_savings)
cfgin_update = None
while True: # for vsz_data, log in cor_send_data.send(cfgin_update):
try:
vsz_data, log = cor_send_data.send(cfgin_update)
# will delete cfg['in']['tables']
except (GeneratorExit, StopIteration, Ex_nothing_done):
print('ok>')
break
custom_expressions_use_time = "[['{Index:%Y-%m-%dT%H:%M:%S}', '{DateEnd:%Y-%m-%dT%H:%M:%S}']]".format_map(
cfg['log_row'])
if not cfg['out']['b_images_only']:
cfgin_update = {
'add_custom': ['USE_timeRange'],
'add_custom_expressions': [custom_expressions_use_time]
}
def main_init(
cfg,
cs_store_name,
__file__=None,
):
"""
- prints parameters
- prints message that program (__file__ or cs_store_name) started
- converts cfg parameters to types according to its prefixes/suffixes names (see ini2dict())
:param cfg:
:param cs_store_name:
:param __file__:
:return:
"""
# global lf
# if cfg.search_path is not None:
# override_path = hydra.utils.to_absolute_path(cfg.search_path)
# override_conf = OmegaConf.load(override_path)
# cfg = OmegaConf.merge(cfg, override_conf)
print("Working directory : {}".format(os.getcwd()))
# todo: print only if config changed
print(OmegaConf.to_yaml(cfg))
# cfg = cfg_from_args(argparser_files(), **kwargs)
if not cfg.program.return_:
print('Can not initialise')
return cfg
elif cfg.program.return_ == '<cfg_from_args>': # to help testing
return cfg
hydra.verbose = 1 if cfg.program.verbose == 'DEBUG' else 0 # made compatible to my old cfg
print('\n' + this_prog_basename(__file__) if __file__ else cs_store_name,
end=' started. ')
try:
cfg_t = ini2dict(
cfg) # fields named with type pre/suffixes are converted
except MissingMandatoryValue as e:
lf.error(standard_error_info(e))
raise Ex_nothing_done()
except Exception:
lf.exception('startup error')
# OmegaConf.update(cfg, "in", cfg.input, merge=False) # error
# to allow non primitive types (cfg.out['db']) and special words field names ('in'):
# cfg = omegaconf.OmegaConf.to_container(cfg)
return cfg_t
def main_init(
cfg,
cs_store_name,
__file__=None,
):
"""
Common startup initializer:
- finds input files
- asks user to proceed if need
- prints message that program (__file__ or cs_store_name) started
:param cfg:
:return:
Note: if new_arg=='<cfg_from_args>' returns cfg but it will be None if argument
argv[1:] == '-h' or '-v' passed to this code
argv[1] is cfgFile. It was used with cfg files:
'csv2h5_nav_supervisor.ini'
'csv2h5_IdrRedas.ini'
'csv2h5_Idronaut.ini'
"""
# global lf
# if cfg.search_path is not None:
# override_path = hydra.utils.to_absolute_path(cfg.search_path)
# override_conf = OmegaConf.load(override_path)
# cfg = OmegaConf.merge(cfg, override_conf)
print("Working directory : {}".format(os.getcwd()))
print(OmegaConf.to_yaml(cfg))
# cfg = cfg_from_args(argparser_files(), **kwargs)
if not cfg.program.return_:
print('Can not initialise')
return cfg
elif cfg.program.return_ == '<cfg_from_args>': # to help testing
return cfg
hydra.verbose = 1 if cfg.program.verbose == 'DEBUG' else 0 # made compatible to my old cfg
print('\n' + this_prog_basename(__file__) if __file__ else cs_store_name,
end=' started. ')
cfg_t = ini2dict(cfg) # fields named with type pre/suffixes are converted
# OmegaConf.update(cfg, "in", cfg.input, merge=False) # error
# to allow non primitive types (cfg.out['db']) and special words field names ('in'):
# cfg = omegaconf.OmegaConf.to_container(cfg)
return cfg_t
def main_cfg(cfg: DictConfig):
global l
# cfg = cfg_from_args(argparser_files(), **kwargs)
if not cfg or not cfg['program'].get('return'):
print('Can not initialise')
return cfg
elif cfg['program']['return'] == '<cfg_from_args>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose'])
print('\n' + this_prog_basename(__file__), end=' started. ')
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in']['path'] = init_file_names(
**cfg['in'], b_interact=cfg['program']['b_interact'])
except Ex_nothing_done as e:
print(e.message)
return ()
return cfg
def main(new_arg=None):
"""
:param new_arg: returns cfg if new_arg=='<cfg_from_args>' but it will be None if argument
argv[1:] == '-h' or '-v' passed to this code
argv[1] is cfgFile. It was used with cfg files:
'csv2h5_nav_supervisor.ini'
'csv2h5_IdrRedas.ini'
'csv2h5_Idronaut.ini'
:return:
"""
global l
cfg = cfg_from_args(my_argparser(), new_arg)
if not cfg or not cfg['program'].get('return'):
print('Can not initialise')
return cfg
elif cfg['program']['return'] == '<cfg_from_args>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'],
cfg['program']['verbose'])
print('\n' + this_prog_basename(__file__), end=' started. ')
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(**{
**cfg['in'], 'path': cfg['in']['db_path']
},
b_interact=cfg['program']['b_interact'])
set_field_if_no(
cfg['in'], 'tables_log', '{}/logFiles'
) # will be filled by each table from cfg['in']['tables']
cfg['in']['query'] = query_time_range(**cfg['in'])
set_field_if_no(cfg['out'], 'db_path', cfg['in']['db_path'])
# cfg['out'] = init_file_names(cfg['out'], , path_field='db_path')
except Ex_nothing_done as e:
print(e.message)
return ()
# args = parser.parse_args()
# args.verbose= args.verbose[0]
# try:
# cfg= ini2dict(args.cfgFile)
# cfg['in']['cfgFile']= args.cfgFile
# except IOError as e:
# print('\n==> '.join([a for a in e.args if isinstance(a,str)])) #e.message
# raise(e)
# Open text log
if 'log' in cfg['program'].keys():
dir_create_if_need(os_path.dirname(cfg['program']['log']))
flog = open(cfg['program']['log'], 'a+', encoding='cp1251')
cfg['out']['log'] = OrderedDict({'fileName': None, 'fileChangeTime': None})
# Prepare saving to csv
if 'file_names_add_fun' in cfg['out']:
file_names_add = eval(
compile(cfg['out']['file_names_add_fun'], '', 'eval'))
else:
file_names_add = lambda i: '.csv' # f'_{i}.csv'
# Prepare data for output store and open it
if cfg['out']['tables'] == ['None']:
# will not write new data table and its log
cfg['out']['tables'] = None
# cfg['out']['tables_log'] = None # for _runs cfg will be redefined (this only None case that have sense?)
h5init(cfg['in'], cfg['out'])
# store, dfLogOld = h5temp_open(**cfg['out'])
cfg_fileN = os_path.splitext(cfg['in']['cfgFile'])[0]
out_tables_log = cfg['out'].get('tables_log')
if cfg_fileN.endswith('_runs') or (bool(out_tables_log)
and 'logRuns' in out_tables_log[0]):
# Will calculate only after filter # todo: calculate derived parameters before were they are bad (or replace all of them if any bad?)
func_before_cycle = lambda x: None
func_before_filter = lambda df, log_row, cfg: df
func_after_filter = lambda df, cfg: log_runs(df, cfg, cfg['out']['log']
)
# this table will be added:
cfg['out']['tables_log'] = [cfg['out']['tables'][0] + '/logRuns']
cfg['out'][
'b_log_ready'] = True # to not apdate time range in h5_append()
# Settings to not affect main data table and switch off not compatible options:
cfg['out']['tables'] = []
cfg['out'][
'b_skip_if_up_to_date'] = False # todo: If False check it: need delete all previous result of CTD_calc() or set min_time > its last log time. True not implemented?
cfg['program'][
'b_log_display'] = False # can not display multiple rows log
if 'b_save_images' in cfg['extract_runs']:
cfg['extract_runs']['path_images'] = cfg['out'][
'db_path'].with_name('_subproduct')
dir_create_if_need(cfg['extract_runs']['path_images'])
else:
if 'brown' in cfg_fileN.lower():
func_before_cycle = load_coef
if 'Lat' in cfg['in']:
func_before_filter = lambda *args, **kwargs: add_ctd_params(
process_brown(*args, **kwargs), kwargs['cfg'])
else:
func_before_filter = process_brown
else:
func_before_cycle = lambda x: None
def ctd_coord_and_params(df: pd.DataFrame, log_row, cfg):
coord_data_col_ensure(df, log_row)
return add_ctd_params(df, cfg)
func_before_filter = ctd_coord_and_params
func_after_filter = lambda df, cfg: df # nothing after filter
func_before_cycle(cfg) # prepare: usually assign data to cfg['for']
if cfg['out'].get('path_csv'):
dir_create_if_need(cfg['out']['path_csv'])
# Load data Main circle #########################################
# Open input store and cicle through input table log records
qstr_trange_pattern = "index>=Timestamp('{}') & index<=Timestamp('{}')"
iSt = 1
dfLogOld, cfg['out']['db'], cfg['out'][
'b_skip_if_up_to_date'] = h5temp_open(**cfg['out'])
b_out_db_is_different = cfg['out']['db'] is not None and cfg['out'][
'db_path_temp'] != cfg['in']['db_path']
# Cycle for each table, for each row in log:
# for path_csv in gen_names_and_log(cfg['out'], dfLogOld):
with FakeContextIfOpen(
lambda f: pd.HDFStore(f, mode='r'), cfg['in']['db_path'],
None if b_out_db_is_different else cfg['out']['db']
) as cfg['in']['db']: # not opens ['in']['db'] if already opened to write
for tbl in cfg['in']['tables']:
if False: # Show table info
nodes = sorted(
cfg['out']['db'].root.__members__) # , key=number_key
print(nodes)
print(tbl, end='. ')
df_log = cfg['in']['db'].select(cfg['in']['tables_log'].format(tbl)
or tbl,
where=cfg['in']['query'])
if True: # try:
if 'log' in cfg['program'].keys():
nRows = df_log.rows.size
flog.writelines(datetime.now().strftime(
'\n\n%d.%m.%Y %H:%M:%S> processed ') + f'{nRows} row' +
('s:' if nRows > 1 else ':'))
for ifile, r in enumerate(df_log.itertuples(),
start=iSt): # name=None
print('.', end='')
sys_stdout.flush()
path_raw = PurePath(r.fileName)
cfg['out']['log'].update(fileName=path_raw.name,
fileChangeTime=r.fileChangeTime)
# save current state
cfg['in']['file_stem'] = cfg['out']['log'][
'fileName'] # for exmple to can extract date in subprogram
cfg['in']['fileChangeTime'] = cfg['out']['log'][
'fileChangeTime']
if cfg['in']['b_skip_if_up_to_date']:
have_older_data, have_duplicates = h5del_obsolete(
cfg['out'], cfg['out']['log'], dfLogOld)
if have_older_data:
continue
if have_duplicates:
cfg['out']['b_remove_duplicates'] = True
print('{}. {}'.format(ifile, path_raw.name), end=': ')
# Load data
qstr = qstr_trange_pattern.format(r.Index, r.DateEnd)
df_raw = cfg['in']['db'].select(tbl, qstr)
cols = df_raw.columns.tolist()
# cfg['in']['lat'] and ['lon'] may be need in add_ctd_params() if Lat not in df_raw
if 'Lat_en' in df_log.columns and 'Lat' not in cols:
cfg['in']['lat'] = np.nanmean((r.Lat_st, r.Lat_en))
cfg['in']['lon'] = np.nanmean((r.Lon_st, r.Lon_en))
df = func_before_filter(df_raw, log_row=r, cfg=cfg)
if df.size: # size is zero means save only log but not data
# filter, updates cfg['out']['log']['rows']
df, _ = set_filterGlobal_minmax(
df, cfg['filter'], cfg['out']['log'])
if 'rows' not in cfg['out']['log']:
l.warning('no data!')
continue
elif isinstance(cfg['out']['log']['rows'], int):
print('filtered out {rows_filtered}, remains {rows}'.
format_map(cfg['out']['log']))
if cfg['out']['log']['rows']:
print('.', end='')
else:
l.warning('no data!')
continue
df = func_after_filter(df, cfg=cfg)
# Append to Store
h5_append(cfg['out'],
df,
cfg['out']['log'],
log_dt_from_utc=cfg['in']['dt_from_utc'])
# Copy to csv
if cfg['out'].get('path_csv'):
fname = '{:%y%m%d_%H%M}-{:%d_%H%M}'.format(
r.Index, r.DateEnd) + file_names_add(ifile)
if not 'data_columns' in cfg['out']:
cfg['out']['data_columns'] = slice(0,
-1) # all cols
df.to_csv( # [cfg['out']['data_columns']]
cfg['out']['path_csv'] / fname,
date_format=cfg['out']['text_date_format'],
float_format='%5.6g',
index_label='Time'
) # to_string, line_terminator='\r\n'
# Log to screen (if not prohibited explicitly)
if cfg['out']['log'].get('Date0') is not None and (
('b_log_display' not in cfg['program'])
or cfg['program']['b_log_display']):
str_log = '{fileName}:\t{Date0:%d.%m.%Y %H:%M:%S}-' \
'{DateEnd:%d. %H:%M:%S%z}\t{rows}rows'.format_map(
cfg['out']['log']) # \t{Lat}\t{Lon}\t{strOldVal}->\t{mag}
l.info(str_log)
else:
str_log = str(cfg['out']['log'].get('rows', '0'))
# Log to logfile
if 'log' in cfg['program'].keys():
flog.writelines('\n' + str_log)
if b_out_db_is_different:
try:
if cfg['out']['tables'] is not None:
print('')
if cfg['out']['b_remove_duplicates']:
h5remove_duplicates(cfg['out'],
cfg_table_keys=('tables',
'tables_log'))
# Create full indexes. Must be done because of using ptprepack in h5move_tables() below
l.debug('Create index')
for tblName in (cfg['out']['tables'] +
cfg['out']['tables_log']):
try:
cfg['out']['db'].create_table_index(tblName,
columns=['index'],
kind='full')
except Exception as e:
l.warning(
': table {}. Index not created - error'.format(
tblName), '\n==> '.join(
[s for s in e.args if isinstance(s, str)]))
except Exception as e:
l.exception('The end. There are error ')
import traceback, code
from sys import exc_info as sys_exc_info
tb = sys_exc_info()[2] # type, value,
traceback.print_exc()
last_frame = lambda tb=tb: last_frame(tb.tb_next
) if tb.tb_next else tb
frame = last_frame().tb_frame
ns = dict(frame.f_globals)
ns.update(frame.f_locals)
code.interact(local=ns)
finally:
cfg['out']['db'].close()
if cfg['program']['log']:
flog.close()
if cfg['out']['db'].is_open:
print('Wait store is closing...')
sleep(2)
failed_storages = h5move_tables(cfg['out'])
print('Finishing...' if failed_storages else 'Ok.', end=' ')
h5index_sort(
cfg['out'],
out_storage_name=f"{cfg['out']['db_path'].stem}-resorted.h5",
in_storages=failed_storages)
def main():
print('\n' + this_prog_basename(__file__), end=' started. ')
import_pattern = """
ImportFileHDF5(u'180905_clockCheck.h5', [
u'/{probe}/coef/G/A', u'/{probe}/coef/G/C', u'/{probe}/coef/H/A', u'/{probe}/coef/H/C', u'/{probe}/table'],
linked=True,
namemap={{u'/{probe}/coef/G/A': u'Ag_old', u'/{probe}/coef/G/C': u'Cg', u'/{probe}/coef/H/A': u'Ah_old',
u'/{probe}/coef/H/C': u'Ch', u'/{probe}/table/Ax': u'countsAx',
u'/{probe}/table/Ay': u'countsAy', u'/{probe}/table/Az': u'countsAz',
u'/{probe}/table/Mx': u'countsMx', u'/{probe}/table/My': u'countsMy',
u'/{probe}/table/Mz': u'countsMz'}}, renames={{u'Temp': u'sT'}})
""".replace('{{', '{{{{').replace('}}', '}}}}')
cfg = veuszPropagate.main([
Path(veuszPropagate.__file__).parent.with_name('veuszPropagate.ini'),
'--data_yield_prefix',
'-',
'--path',
str(path_cruise.joinpath(r'incl*.txt')),
'--pattern_path',
str(path_cruise.with_name('180905_1320incl01.vsz')),
# '--import_method', 'ImportFileCSV',
'--before_next',
'restore_config',
# '--add_custom_list',
# 'USEtime',
# '--add_custom_expressions_list',
# """
# "[['2018-09-05T13:19:55', '2018-09-05T13:20:25']]"
# """,
'--export_pages_int_list',
'6,7',
'--b_interact',
'0',
'--b_images_only',
'True'
])
if not cfg:
return 1
# veuszPropagate.load_vsz = cfg['load_vsz']
log_times = pd.to_datetime(
[
'05.09.2018 13:20', # 05.09.2018 13:20-00 первое качание
'07.09.2018 15:48', # 07.09.2018 15:48-00 второе качание
'11.09.2018 12:59' # 11.09.2018 12:59-00 третье качание
],
dayfirst=True)
t_log_list = pd.to_datetime(log_times)
dt_to_st = pd.Timedelta(seconds=-5)
dt_to_en = pd.Timedelta(seconds=30)
custom_expressions_use_time = [
"[['{:%Y-%m-%dT%H:%M:%S}', '{:%Y-%m-%dT%H:%M:%S}']]".format(
t_log + dt_to_st, t_log + dt_to_en) for t_log in t_log_list
]
# u"[['2018-09-05T13:19:55', '2018-09-05T13:20:25']]",
# u"[['2018-09-07T15:47:55', '2018-09-07T15:48:25']]",
# u"[['2018-09-11T13:58:55', '2018-09-11T13:59:55']]",
cor_savings = veuszPropagate.co_savings(cfg)
cor_savings.send(None)
for t_log, expr in zip(t_log_list, custom_expressions_use_time):
name_add_time = '_{:%y%m%d_%H%M}-{:%H%M}'.format(
t_log + dt_to_st, t_log + dt_to_en)
print('Processing group {}...'.format(name_add_time))
def f_file_name(file_name):
p = Path(file_name)
return str(p.with_name(p.stem + name_add_time + p.suffix))
gen_veusz_and_logs = veuszPropagate.load_to_veusz(
veuszPropagate.ge_names(cfg, f_file_name), cfg)
cor_send_data = veuszPropagate.co_send_data(gen_veusz_and_logs, cfg,
cor_savings)
cfgin_update = None
while True: # for vsz_data, log in cor_send_data.send(cfgin_update):
try:
vsz_data, log = cor_send_data.send(cfgin_update)
except (GeneratorExit, StopIteration, Ex_nothing_done):
print('ok>')
break
# except Exception as e:
# print('There are error: ', standard_error_info(e))
# break # continue
probe = log['fileName'][:log['fileName'].find('_')]
if not cfg['out']['b_images_only']:
cfgin_update = {
'add_custom': ['USEtime'],
'add_custom_expressions': [expr],
'eval': [import_pattern.format(probe=probe)]
}
def main(new_arg=None):
cfg = cfg_from_args(my_argparser(), new_arg)
if not cfg or not cfg['program'].get('return'):
print('Can not initialise')
return cfg
elif cfg['program']['return'] == '<cfg_from_args>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'],
cfg['program']['verbose'])
print('\n' + this_prog_basename(__file__), end=' started. ')
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(**cfg['in'],
b_interact=cfg['program']['b_interact'],
cfg_search_parent=cfg['out'])
h5init(cfg['in'], cfg['out'])
except Ex_nothing_done as e:
print(e.message)
exit()
df_dummy = pd.DataFrame(
np.full(1,
np.NaN,
dtype=np.dtype({
'formats': ['float64', 'float64'],
'names': cfg['out']['tracks_cols'][1:]
})),
index=(pd.NaT, )) # used for insert separator lines
if 'routes_cols' not in cfg['in']:
cfg['in']['routes_cols'] = cfg['in']['waypoints_cols']
if 'routes_cols' not in cfg['out']:
cfg['out']['routes_cols'] = cfg['out'][
'waypoints_cols'] # cfg['in']['routes_cols'] #
# Writing
if True: # try:
l.warning('processing ' + str(cfg['in']['nfiles']) + ' file' +
's:' if cfg['in']['nfiles'] > 1 else ':')
cfg['out']['log'] = {}
set_field_if_no(cfg['out'], 'table_prefix',
PurePath(cfg['in']['path']).stem)
cfg['out']['table_prefix'] = cfg['out']['table_prefix'].replace(
'-', '')
if len([t for t in cfg['out']['tables'] if len(t)]) > 1:
cfg['out']['tables'] = \
[cfg['out']['table_prefix'] + '_' + s for s in cfg['out']['tables']]
cfg['out']['tables_log'] = \
[cfg['out']['table_prefix'] + '_' + s for s in cfg['out']['tables_log']]
tables = dict(zip(df_names, cfg['out']['tables']))
tables_log = dict(zip(df_names, cfg['out']['tables_log']))
# Can not save path to DB (useless?) so set for this max file name length:
set_field_if_no(cfg['out'], 'logfield_fileName_len', 50)
cfg['out']['index_level2_cols'] = cfg['in']['routes_cols'][0]
# ###############################################################
# ## Cumulate all data in cfg['out']['path_temp'] ##################
## Main circle ############################################################
for i1_file, path_gpx in h5_dispenser_and_names_gen(
cfg['in'], cfg['out']):
l.info('{}. {}: '.format(i1_file, path_gpx.name))
# Loading data
dfs = gpxConvert(cfg, path_gpx)
print('write', end=': ')
sys_stdout.flush()
for key, df in dfs.items():
if (not tables.get(key)) or df.empty:
continue
elif key == 'tracks':
# Save last time to can filter next file
cfg['in']['time_last'] = df.index[-1]
sort_time = False if key in {'waypoints', 'routes'} else None
# monkey patching
if 'tracker' in tables[key]:
# Also {} must be in tables[key]. todo: better key+'_fun_tracker' in cfg['out']?
# Trackers processing
trackers_numbers = {
'0-3106432': '1',
'0-2575092': '2',
'0-3124620': '3',
'0-3125300': '4',
'0-3125411': '5',
'0-3126104': '6'
}
tables_pattern = tables[key]
tables_log_pattern = tables_log[key]
df['comment'] = df['comment'].str.split(" @",
n=1,
expand=True)[0]
# split data and save to multipe tables
df_all = df.set_index(['comment', df.index])
for sn, n in trackers_numbers.items(
): # set(df_all.index.get_level_values(0))
try:
df = df_all.loc[sn]
except KeyError:
continue
# redefine saving parameters
cfg['out']['table'] = tables_pattern.format(
trackers_numbers[sn])
cfg['out']['table_log'] = tables_log_pattern.format(
trackers_numbers[sn])
call_with_valid_kwargs(df_filter_and_save_to_h5,
df**cfg,
input=cfg['in'],
sort_time=sort_time)
else:
cfg['out']['table'] = tables[key]
cfg['out']['table_log'] = tables_log[key]
call_with_valid_kwargs(df_filter_and_save_to_h5,
df,
**cfg,
input=cfg['in'],
sort_time=sort_time)
# try:
# if cfg['out']['b_remove_duplicates']:
# for tbls in cfg['out']['tables_have_wrote']:
# for tblName in tbls:
# cfg['out']['db'][tblName].drop_duplicates(keep='last', inplace= True)
# print('Create index', end=', ')
# create_table_index calls create_table which docs sais "cannot index Time64Col() or ComplexCol"
# so load it, index, then save
# level2_index = None
# df = cfg['out']['db'][tblName] # last commented
# df.set_index([navp_all_index, level2_index])
# df.sort_index()
# cfg['out']['db'][tblName].sort_index(inplace=True)
# if df is not None: # resave
# df_log = cfg['out']['db'][tblName]
# cfg['out']['db'].remove(tbls[0])
# cfg['out']['db'][tbls[0]] = df
# cfg['out']['db'][tbls[1]] = df_log
try:
pass
except Exception as e:
print('The end. There are error ', standard_error_info(e))
# import traceback, code
# from sys import exc_info as sys_exc_info
#
# tb = sys_exc_info()[2] # type, value,
# traceback.print_exc()
# last_frame = lambda tb=tb: last_frame(tb.tb_next) if tb.tb_next else tb
# frame = last_frame().tb_frame
# ns = dict(frame.f_globals)
# ns.update(frame.f_locals)
# code.interact(local=ns)
# finally:
# cfg['out']['db'].close()
# failed_storages= h5move_tables(cfg['out'], cfg['out']['tables_have_wrote'])
try:
failed_storages = h5move_tables(cfg['out'],
tbl_names=cfg['out'].get(
'tables_have_wrote', set()))
print('Finishing...' if failed_storages else 'Ok.', end=' ')
# Sort if have any processed data that needs it (not the case for the routes and waypoints), else don't because ``ptprepack`` not closes hdf5 source if it not finds data
if cfg['in'].get('time_last'):
cfg['out']['b_remove_duplicates'] = True
h5index_sort(
cfg['out'],
out_storage_name=f"{cfg['out']['db_path'].stem}-resorted.h5",
in_storages=failed_storages,
tables=cfg['out'].get('tables_have_wrote', set()))
except Ex_nothing_done:
print('ok')
def main(new_arg=None, veusze=None, **kwargs):
"""
Initialise configuration and runs or returns routines
cfg:
['program']['log'],
'out'
'in'
'async'
globals:
load_vsz
l
:param new_arg:
:param veusze: used to reuse veusz embedded object (thus to not leak memory)
:return:
"""
global l, load_vsz
cfg = cfg_from_args(my_argparser(), new_arg, **kwargs)
if not cfg or not cfg['program'].get('return'):
print('Can not initialise')
return cfg
elif cfg['program']['return'] == '<cfg_from_args>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'],
cfg['program']['verbose'])
cfg['program']['log'] = l.root.handlers[
0].baseFilename # sinchronize obtained absolute file name
print('\n' + this_prog_basename(__file__), 'started', end=' ')
__name__ = '__main__' # indicate to other functions that they are called from main
if cfg['out'].get('paths'):
if not cfg['out']['b_images_only']:
raise NotImplementedError(
'Provided out in not "b_images_only" mode!')
cfg['out']['nfiles'] = len(cfg['out']['paths'])
cfg['out']['path'] = cfg['out']['paths'][0]
print(
end=f"\n- {cfg['out']['nfiles']} output files to export images...")
pass
else:
if cfg['out']['b_images_only']:
print(
'in images only mode. Output pattern: ') # todo Export path: '
else:
print('. Output pattern and Data: ')
try:
# Using cfg['out'] to store pattern information
if not Path(cfg['in']['pattern_path']).is_absolute():
cfg['in']['pattern_path'] = Path(cfg['in']['path']).with_name(
str(cfg['in']['pattern_path']))
cfg['out']['path'] = cfg['in']['pattern_path']
cfg['out']['paths'], cfg['out']['nfiles'], cfg['out'][
'path'] = init_file_names(**cfg['out'], b_interact=False)
except Ex_nothing_done as e:
if not cfg['out']['b_images_only']:
l.warning(
f'{e.message} - no pattern. Specify it or use "b_images_only" mode!'
)
return # or raise FileNotFoundError?
if (cfg['out']['b_images_only'] and cfg['out']['paths']):
cfg['in']['paths'] = cfg['out']['paths'] # have all we need to export
else:
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(**cfg['in'],
b_interact=cfg['program']
['b_interact'])
except Ex_nothing_done as e:
print(e.message)
return # or raise FileNotFoundError?
except TypeError: # expected str, bytes or os.PathLike object, not NoneType
# cfg['in']['path'] is None. May be it is not need
cfg['in']['paths'] = [cfg['in']['pattern_path']
] # dummy for compatibility
cfg['in']['nfiles'] = 1
cfg['out']['export_dir'] = dir_from_cfg(cfg['out']['path'].parent,
cfg['out']['export_dir'])
if 'restore_config' in cfg['program']['before_next']:
cfg['in_saved'] = cfg['in'].copy()
# Next is commented because reloading is Ok: not need to Close()
# if cfg['out']['b_images_only'] and not 'Close()' in cfg['program']['before_next']:
# cfg['program']['before_next'].append(
# 'Close()') # usually we need to load new file for export (not only modify previous file)
if cfg['program']['export_timeout_s'] and export_images_timed:
cfg['async'] = {
'loop': asyncio.get_event_loop(),
'export_timeout_s': cfg['program']['export_timeout_s']
}
else:
cfg['async'] = {'loop': None}
load_vsz = load_vsz_closure(cfg['program']['veusz_path'],
cfg['program']['load_timeout_s'],
cfg['program']['b_execute_vsz'])
cfg['load_vsz'] = load_vsz
cfg['co'] = {}
if cfg['in']['table_log'] and cfg['in']['path'].suffix == '.h5' and not (
cfg['out']['b_images_only'] and len(cfg['in']['paths']) > 1):
# load data by ranges from table log rows
cfg['in']['db_path'] = cfg['in']['path']
in_fulls = h5log_names_gen(cfg['in'])
elif cfg['in']['tables']:
# tables instead files
in_fulls = ge_names_from_hdf5_paths(cfg)
else: # switch to use found vsz as source if need only export images (even with database source)
in_fulls = ge_names(cfg)
cor_savings = co_savings(cfg)
cor_savings.send(None)
nfiles = 0
try: # if True:
path_prev = os_getcwd()
os_chdir(cfg['out']['path'].parent)
if cfg['program']['return'] == '<corutines_in_cfg>':
cfg['co']['savings'] = cor_savings
cfg['co']['gen_veusz_and_logs'] = load_to_veusz(in_fulls, cfg)
cfg['co']['send_data'] = co_send_data(load_to_veusz, cfg,
cor_savings)
return cfg # return with link to generator function
elif cfg['in'].get('data_yield_prefix'):
# Cycle with obtaining Veusz data
cfgin_update = None
while True: # for vsz_data, log in cor_send_data.send(cfgin_update):
try:
vsz_data, log = co_send_data.send(cfgin_update)
nfiles += 1
except (GeneratorExit, StopIteration, Ex_nothing_done):
break
if 'f_custom_in_cycle' in cfg['program']:
cfgin_update = cfg['program']['f_custom_in_cycle'](
vsz_data, log)
else:
# Cycle without obtaining Veusz data (or implemented by user's cfg['program']['f_custom_in_cycle'])
for veusze, log in load_to_veusz(in_fulls, cfg, veusze):
file_name_r = Path(log['out_vsz_full']).relative_to(
cfg['out']['path'].parent)
if cfg['program'].get('f_custom_in_cycle'):
cfgin_update = cfg['program']['f_custom_in_cycle'](veusze,
log)
veusze_commands(veusze, cfgin_update, file_name_r)
cor_savings.send((veusze, log))
nfiles += 1
cor_savings.close()
if cfg['program']['return'] != '<embedded_object>':
veusze = None # to note that it is closed in cor_savings.close()
print(f'{nfiles} processed. ok>')
pass
except Exception as e:
l.exception('Not good')
return # or raise FileNotFoundError?
finally:
if cfg['async']['loop']:
cfg['async']['loop'].close()
os_chdir(path_prev)
if veusze and cfg['program']['return'] == '<end>':
veusze.Close()
veusze.WaitForClose()
veusze = None
elif cfg['program']['return'] == '<embedded_object>':
cfg['veusze'] = veusze
return cfg
def main(new_arg=None, veusze=None):
"""
Note: if vsz data source have 'Ag_old_inv' variable then not invert coef. Else invert to use in vsz which not invert coefs
:param new_arg:
:return:
"""
global l
p = veuszPropagate.my_argparser()
p_groups = {
g.title: g
for g in p._action_groups if g.title.split(' ')[-1] != 'arguments'
} # skips special argparse groups
p_groups['in'].add(
'--channels_list',
help=
'channels needed zero calibration: "magnetometer" or "M" for magnetometer and any else for accelerometer, use "M, A" for both, empty to skip '
)
p_groups['in'].add(
'--widget',
help=
'path to Veusz widget property which contains coefficients. For example "/fitV(force)/grid1/graph/fit1/values"'
)
p_groups['in'].add(
'--data_for_coef',
default='max_incl_of_fit_t',
help=
'Veusz data to use as coef. If used with widget then this data is appended to data from widget'
)
p_groups['out'].add('--out.path', help='path to db where write coef')
p_groups['out'].add(
'--re_tbl_from_vsz_name',
help=
'regex to extract hdf5 table name from to Veusz file name (last used "\D*\d*")'
# ? why not simly specify table name?
)
# todo: "b_update_existed" arg will be used here for exported images. Check whether False works or prevent open vsz
cfg = cfg_from_args(p, new_arg)
if not Path(cfg['program']['log']).is_absolute():
cfg['program']['log'] = str(
Path(__file__).parent.joinpath(
cfg['program']['log'])) # l.root.handlers[0].baseFilename
if not cfg:
return
if new_arg == '<return_cfg>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'],
cfg['program']['verbose'])
veuszPropagate.l = l
print('\n' + this_prog_basename(__file__), 'started', end=' ')
if cfg['out']['b_images_only']:
print('in images only mode.')
try:
print('Output pattern ')
# Using cfg['out'] to store pattern information
if not Path(cfg['in']['pattern_path']).is_absolute():
cfg['in']['pattern_path'] = str(cfg['in']['path'].parent.joinpath(
cfg['in']['pattern_path']))
set_field_if_no(cfg['out'], 'path', cfg['in']['pattern_path'])
cfg['out']['paths'], cfg['out']['nfiles'], cfg['out'][
'path'] = init_file_names(**cfg['out'],
b_interact=cfg['program']['b_interact'])
except Ex_nothing_done as e:
print(e.message, ' - no pattern')
return # or raise FileNotFoundError?
try:
print(end='Data ')
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(
**cfg['in'], b_interact=False) # do not bother user 2nd time
except Ex_nothing_done as e:
print(e.message)
return # or raise FileNotFoundError?
if not cfg['out']['export_dir']:
cfg['out']['export_dir'] = Path(cfg['out']['path']).parent
if cfg['program']['before_next'] and 'restore_config' in cfg['program'][
'before_next']:
cfg['in_saved'] = cfg['in'].copy()
# cfg['loop'] = asyncio.get_event_loop()
# cfg['export_timeout_s'] = 600
cfg['out']['export_dir'] = dir_from_cfg(cfg['out']['path'].parent,
cfg['out']['export_dir'])
veuszPropagate.load_vsz = veuszPropagate.load_vsz_closure(
cfg['program']['veusz_path'],
b_execute_vsz=cfg['program']['b_execute_vsz'])
gen_veusz_and_logs = veuszPropagate.load_to_veusz(
veuszPropagate.ge_names(cfg), cfg, veusze)
names_get = ['Inclination_mean_use1', 'logVext1_m__s'
] # \, 'Inclination_mean_use2', 'logVext2_m__s'
names_get_fits = ['fit'] # , 'fit2'
vsz_data = {n: [] for n in names_get}
for n in names_get_fits:
vsz_data[n] = []
# prepare collecting all coef in text also
names_get_txt_results = ['fit1result'] # , 'fit2result'
txt_results = {n: {} for n in names_get_txt_results}
i_file = 0
for veusze, log in gen_veusz_and_logs:
if not veusze:
continue
i_file += 1
print(i_file)
if cfg['out']['re_tbl_from_vsz_name']:
table = cfg['out']['re_tbl_from_vsz_name'].match(
log['out_name']).group()
else:
table = re.sub(
'^[\d_]*', '',
log['out_name']) # delete all first digits (date part)
for n in names_get:
vsz_data[n].append(veusze.GetData(n)[0])
for n in [cfg['in']['data_for_coef']]:
vsz_data[n] = list(veusze.GetData(n)[0])
# Save velocity coefficients into //{table}//coef//Vabs{i} where i - fit number enumeretad from 0
for i, name_out in enumerate(names_get_fits): # ['fit1', 'fit2']
coef = veusze.Get(
cfg['in']['widget']
) # veusze.Root['fitV(inclination)']['grid1']['graph'][name_out].values.val
if 'a' in coef:
coef_list = [
coef[k] for k in ['d', 'c', 'b', 'a'] if k in coef
]
else:
coef_list = [
coef[k] for k in sorted(coef.keys(), key=digits_first)
]
if cfg['in']['data_for_coef']:
coef_list += vsz_data[cfg['in']['data_for_coef']]
vsz_data[name_out].append(coef_list)
h5copy_coef(None,
cfg['out']['path'],
table,
dict_matrices={
f'//coef//Vabs{i}':
coef_list,
f'//coef//date':
np.float64([
np.NaN,
np.datetime64(datetime.now()).astype(np.int64)
])
})
# h5savecoef(cfg['out']['path'], path=f'//{table}//coef//Vabs{i}', coef=coef_list)
txt_results[names_get_txt_results[i]][table] = str(coef)
# Zeroing matrix - calculated in Veusz by rotation on old0pitch old0roll
Rcor = veusze.GetData(
'Rcor'
)[0] # zeroing angles tuned by "USEcalibr0V_..." in Veusz Custom definitions
if len(cfg['in']['channels']):
l.info(
'Applying zero calibration matrix of peach = {} and roll = {} degrees'
.format(np.rad2deg(veusze.GetData('old0pitch')[0][0]),
np.rad2deg(veusze.GetData('old0roll')[0][0])))
with h5py.File(cfg['out']['path'], 'a') as h5:
for channel in cfg['in']['channels']:
(col_str, coef_str) = channel_cols(channel)
# h5savecoef(cfg['out']['path'], path=f'//{table}//coef//Vabs{i}', coef=coef_list), dict_matrices={'//coef//' + coef_str + '//A': coefs[tbl][channel]['A'], '//coef//' + coef_str + '//C': coefs[tbl][channel]['b']})
# Currently used inclinometers have electronics rotated on 180deg. Before we inserted additional
# rotation operation in Veusz by inverting A_old. Now we want iclude this information in database coef only.
try: # Checking that A_old_inv exist
A_old_inv = veusze.GetData('Ag_old_inv')
is_old_used = True # Rcor is not account for electronic is rotated.
except KeyError:
is_old_used = False # Rcor is account for rotated electronic.
if is_old_used: # The rotation is done in vsz (A_old in vsz is inverted) so need rotate it back to
# use in vsz without such invertion
# Rotate on 180 deg (note: this is not inversion)
A_old_inv = h5[f'//{table}//coef//{coef_str}//A'][...]
A_old = np.dot(A_old_inv,
[[1, 0, 0], [0, -1, 0], [0, 0, -1]
]) # adds 180 deg to roll
else:
A_old = h5[f'//{table}//coef//{coef_str}//A'][...]
# A_old now accounts for rotated electronic
A = np.dot(Rcor, A_old)
h5copy_coef(None,
h5,
table,
dict_matrices={f'//coef//{coef_str}//A': A})
# veusze.Root['fitV(inclination)']['grid1']['graph2'][name_out].function.val
print(vsz_data)
veuszPropagate.export_images(
veusze,
cfg['out'],
f"_{log['out_name']}",
b_skip_if_exists=not cfg['out']['b_update_existed'])
# vsz_data = veusz_data(veusze, cfg['in']['data_yield_prefix'])
# # caller do some processing of data and gives new cfg:
# cfgin_update = yield(vsz_data, log) # to test run veusze.Save('-.vsz')
# cfg['in'].update(cfgin_update) # only update of cfg.in.add_custom_expressions is tested
# if cfg['in']['add_custom']:
# for n, e in zip(cfg['in']['add_custom'], cfg['in']['add_custom_expressions']):
# veusze.AddCustom('definition', n, e, mode='replace')
# #cor_savings.send((veusze, log))
#
#
#
#
# veusze.Save(str(path_vsz_save), mode='hdf5') # veusze.Save(str(path_vsz_save)) saves time with bad resolution
print(f'Ok')
print(txt_results)
for n in names_get:
pd.DataFrame.from_dict(
dict(zip(list(txt_results['fit1result'].keys()),
vsz_data[n]))).to_csv(
Path(cfg['out']['path']).with_name(
f'average_for_fitting-{n}.txt'),
sep='\t',
header=txt_results['fit1result'].keys,
mode='a')
return {**vsz_data, 'veusze': veusze}
def main(new_arg=None, **kwargs):
global l
if __package__ is None:
from sys import path as sys_path
from os import path as os_path
sys_path.append(
os_path.dirname(os_path.dirname(os_path.abspath(__file__))))
from utils2init import prep
cfg = cfg_from_args(my_argparser(), new_arg, **kwargs)
# Input files
default_input_filemask = '*.xml'
inD, namesFE, nFiles, outD, outF, outE, bWrite2dir, msgFile = prep(
{
'path': cfg['in']['path'],
'out_path': cfg['out']['path']
}, default_input_filemask)
l = init_logging(logging, None, cfg['program']['log'],
cfg['program']['verbose'])
l.warning('\n' + this_prog_basename(__file__) + ' started. ')
l.warning(msgFile)
# set_field_if_no(cfg['out'], 'dt_between_track_segments', 99999)
gpx = parse_smses(cfg)
try:
f = open(cfg['in']['path'].with_suffix('.gpx'), 'w')
bMissedCoordTo0 = 'b_missed_coord_to_zeros' in cfg['process'] and cfg[
'process']['b_missed_coord_to_zeros']
if bMissedCoordTo0:
for p in gpx.walk(only_points=True):
if p.latitude is None or p.longitude is None:
p.latitude = '0' # float('NaN') #0
p.longitude = '0' # float('NaN') #0
# if p_prev==p:
# p.delete
# p_prev= p
# gpx.add_missing_data() #remove_empty()
f.write(gpx.to_xml())
print('ok')
except Ex_nothing_done as e:
print(e.message)
except Exception as e:
msg_option = f'The end. There are error {standard_error_info(e)}'
print(msg_option)
try:
err_msg = e.msg
l.error(' '.join([err_msg, msg_option]))
except AttributeError:
l.error(msg_option)
finally:
f.close()
try:
# if not bWrite2dir:
# fp_out.close()
# l.handlers[0].flush()
logging.shutdown()
except:
pass
def main(new_arg=None):
cfg = cfg_from_args(my_argparser(), new_arg)
if not cfg:
return
if new_arg == '<return_cfg>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose'])
if not cfg['out']['path'].is_absolute():
# set path relative to cfg['in']['db_path']
cfg['out']['path'] = cfg['in']['db_path'].with_name(str(cfg['out']['path']))
l.warning('\n {}({}) is gonna save gpx to ..{} dir. '.format(
this_prog_basename(__file__), cfg['in']['db_path'], cfg['out']['path'].parent))
if cfg['out']['select_from_tablelog_ranges'] is None:
gpx_symbols = None
else:
gpx_symbols = init_gpx_symbols_fun(cfg['out'])
global gpx_names_funs # Shortcat for cfg['out']['gpx_names_funs']
# Load data #################################################################
qstr_trange_pattern = "index>=Timestamp('{}') & index<=Timestamp('{}')"
with pd.HDFStore(cfg['in']['db_path'], mode='r') as store:
# Find tables by pattern
if '*' in cfg['in']['tables'][0]:
# if 'table_prefix' in cfg['in']
pattern_tables = cfg['in']['tables'][0]
cfg['in']['tables'] = h5find_tables(store, pattern_tables)
len_tables = len(cfg['in']['tables'])
msg = 'Found {} tables with pattern {}'.format(len_tables, pattern_tables)
if len_tables:
l.info(msg)
else:
raise Ex_nothing_done(msg + '!')
gpx_names_funs = []
for itbl in range(len(cfg['in']['tables'])): # same fo each table
gpx_names_funs.append(cfg['out']['gpx_names_funs'][0])
else: # fixed number of tables
# initialise with defaults if need:
gpx_names_funs = cfg['out']['gpx_names_funs']
for itbl in range(len(gpx_names_funs), len(cfg['in']['tables'])):
gpx_names_funs.append('i+1')
dfs_rnav = []
tbl_names_all_shortened = []
for itbl, tblD in enumerate(cfg['in']['tables']):
print(itbl, '. ', tblD, end=': ', sep='')
if cfg['in']['tables_log'][0]:
tblL = tblD + '/' + cfg['in']['tables_log'][0]
try:
dfL = store[tblL]
except KeyError as e:
l.warning(' '.join([s for s in e.args if isinstance(s, str)]))
continue
else: # only for tables without log (usually no such tables)
l.warning('configuration specifies to get data without use of "log..." tables')
st_en = store[tblD].index[[0, -1]]
if cfg['process']['period_files']:
t_intervals_start = pd.date_range(
start=st_en[0].normalize(),
end=max(st_en[-1], st_en[-1].normalize() + pd_period_to_timedelta(
cfg['process']['period_files'])),
freq=cfg['process']['period_files'])[1:] # makes last t_interval_start >= all_data[-1]
dfL = pd.DataFrame.from_records({'DateEnd': t_intervals_start, 'fileName': tblD},
index=st_en[:1].append(t_intervals_start[:-1]))
else:
dfL = pd.DataFrame.from_records({'DateEnd': st_en[-1], 'fileName': tblD}, index=st_en[:1])
gpx_names_fun_str = "lambda i, row, t=0: '{}'.format({})".format(
cfg['out']['gpx_names_fun_format'],
gpx_names_funs[itbl])
gpx_names_fun = eval(compile(gpx_names_fun_str, '', 'eval'))
if cfg['out']['select_from_tablelog_ranges'] is None:
# Use all data for ranges specified in log rows and saves tracks (not points)
for irow, r in enumerate(dfL.itertuples()): # iterrows()
qstr = qstr_trange_pattern.format(r.Index, r.DateEnd)
print(qstr, end='... ')
try:
dfD = store.select(cfg['in']['table_nav'
] if cfg['in']['table_nav'] else tblD, qstr,
columns=['Lat', 'Lon', 'DepEcho'])
except Exception as e:
l.exception('Error when query: {}. '.format(qstr))
# '\n==> '.join([s for s in e.args if isinstance(s, str)])))
continue
# Keep data with period = 1s only
dfD = dfD[~dfD.index.round(pd.Timedelta(seconds=1)).duplicated()]
# dfD.drop_duplicates(['Lat', 'Lon', 'index'])'
bGood = filterGlobal_minmax(dfD, dfD.index, cfg['filter'])
dfD = dfD[bGood]
# Add UTC time and table name to output file name
# Local time and table name to gpx object name
str_time_long = '{:%y%m%d_%H%M}'.format(r.Index)
r = r._replace(Index=timzone_view(r.Index, cfg['out']['dt_from_utc_in_comments']))
tblD_safe = file_from_tblname(tblD, cfg['in']['tables_log'][0])
try:
gpx_names_fun_result = gpx_names_fun(tblD_safe, r) # '{:%y%m%d}'.format(timeLocal)
except TypeError as e:
raise TypeError('Can not evalute gpx_names_fun "{}"'.format(gpx_names_fun_str)).with_traceback(
e.__traceback__)
save_to_gpx(
dfD, cfg['out']['path'].with_name(f'{str_time_long}{tblD_safe}'),
gpx_obj_namef=gpx_names_fun_result, cfg_proc=cfg['process'])
if len(cfg['in']['tables']) > 1:
nav2add_cur = dfD if irow == 0 else nav2add_cur.append(dfD)
if len(cfg['in']['tables']) > 1:
nav2add_cur = dfD.assign(itbl=itbl)
else:
# Use only 1 data point per log row
if cfg['out']['select_from_tablelog_ranges'] != 0:
print('selecting from {} row index of log table'.format(
cfg['out']['select_from_tablelog_ranges']))
try:
dfL.index = dfL.index.tz_convert('UTC')
except TypeError as e:
print((e.msg if hasattr(e, 'msg') else str(e)) + '!\n- continue presume on UTC log index...')
print(end='all log data ')
time_points = (dfL.index if cfg['out']['select_from_tablelog_ranges'] == 0 else
dfL['DateEnd'] if cfg['out']['select_from_tablelog_ranges'] == -1 else
None)
if time_points is None:
raise (ValueError("cfg['out']['select_from_tablelog_ranges'] must be 0 or -1"))
cols_nav = ['Lat', 'Lon', 'DepEcho']
nav2add = h5select(store, cfg['in']['table_nav'], cols_nav, time_points=time_points,
dt_check_tolerance=cfg['process']['dt_search_nav_tolerance'],
query_range_lims=(time_points[0], dfL['DateEnd'][-1])
)[0]
cols_nav = nav2add.columns # not all columns may be loaded
# Try get non NaN from dfL if it has needed columns (we used to write there edges' data with _st/_en suffixes)
isna = nav2add.isna()
dfL_col_suffix = 'st' if cfg['out']['select_from_tablelog_ranges'] == 0 else 'en'
for col in cols_nav:
col_dat = f'{col}_{dfL_col_suffix}'
if isna[col].any() and col_dat in dfL.columns:
b_use = isna[col].values & dfL[col_dat].notna().values
nav2add.loc[b_use, col] = dfL.loc[b_use, col_dat].values
nav2add.index = timzone_view(nav2add.index, dt_from_utc=cfg['out']['dt_from_utc_in_comments'])
# tz_local= tzoffset(None, cfg['out']['dt_from_utc_in_comments'].total_seconds())
# if nav2add.index.tz is None:
# # think if time zone of tz-naive Timestamp is naive then it is UTC
# nav2add.index = nav2add.index.tz_localize('UTC')
# nav2add.tz_convert(tz_local, copy= False)
# Save to gpx waypoints
nav2add_cur = nav2add.assign(itbl=itbl)
# if 'gpx_names_funs' in cfg['out'] and \
# len(cfg['out']['gpx_names_funs'])>itbl:
#
# gpx_names = eval(compile('lambda i: str({})'.format(
# cfg['out']['gpx_names_funs'][itbl]), [], 'eval'))
#
save_to_gpx(nav2add_cur,
cfg['out']['path'] / f"stations_{file_from_tblname(tblD, cfg['in']['tables_log'][0])}",
gpx_obj_namef=gpx_names_fun, waypoint_symbf=gpx_symbols,
cfg_proc=cfg['process']
)
# save_to_csv(nav2add, dfL.index, cfg['out']['path'].with_name(f'nav{tblD}.txt'))
if False: # Show table info
store.get_storer(tblD).table
nodes = sorted(store.root.__members__) # , key=number_key
print(nodes)
# store.get_node('CTD_Idronaut(Redas)').logFiles # next level nodes
# prepare saving of combined gpx
if tbl_names_all_shortened:
i_new = 0
for c_prev, c_new in zip(tbl_names_all_shortened[-1], tblD):
if c_new == c_prev:
i_new += 1
else:
break
tbl_names_all_shortened.append(tblD[i_new:])
else:
tbl_names_all_shortened.append(tblD)
dfs_rnav.append(nav2add_cur)
if len(cfg['in']['tables']) > 1 and cfg['out']['gpx_names_funs_cobined']:
print('combined: ', end='') # Save combined data to gpx
df_rnav_combined = pd.concat(dfs_rnav)
df_rnav_combined.sort_index(inplace=True)
# Save to gpx waypoints
if 'gpx_names_funs' in cfg['out']['gpx_names_funs_cobined']:
gpx_names_funs = [ # row not used, it is here only for compability with tracks
eval(compile("lambda i: " + f, '', 'eval')) for f in gpx_names_funs]
gpx_names_fun = eval(compile(
"lambda i,row,t: '{gpx_names_fun_format}'.format({gpx_names_funs_cobined})".format_map(
cfg['out']), '', 'eval'))
# gpx_symbols = lambda row: cfg['out']['gpx_symbols'][sym_index_fun(row)]
# gpx_names = eval(compile("lambda i,row: '{gpx_names_fun_format}'.format({gpx_names_funs_cobined})".format_map(cfg['out']), '', 'eval'))
# gpx_names = lambda i: str(i + 1)
save_to_gpx(
df_rnav_combined,
cfg['out']['path'].with_name(
'all_' + file_from_tblname(','.join(tbl_names_all_shortened), cfg['in']['tables_log'][0])),
gpx_obj_namef=gpx_names_fun, waypoint_symbf=gpx_symbols, cfg_proc=cfg['process'])
print('Ok')
def main(new_arg=None, **kwargs):
"""
:param new_arg: list of strings, command line arguments
:kwargs: dicts for each section: to overwrite values in them (overwrites even high priority values, other values remains)
Note: if new_arg=='<cfg_from_args>' returns cfg but it will be None if argument
argv[1:] == '-h' or '-v' passed to this code
argv[1] is cfgFile. It was used with cfg files:
'csv2h5_nav_supervisor.ini'
'csv2h5_IdrRedas.ini'
'csv2h5_Idronaut.ini'
:return:
"""
global l
cfg = cfg_from_args(my_argparser(), new_arg, **kwargs)
if not cfg or not cfg['program'].get('return'):
print('Can not initialise')
return cfg
elif cfg['program']['return'] == '<cfg_from_args>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'],
cfg['program']['verbose'])
print('\n' + this_prog_basename(__file__), end=' started. ')
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(**cfg['in'],
b_interact=cfg['program']['b_interact'])
except Ex_nothing_done as e:
print(e.message)
return ()
bOld_FF00FF = False
# if 'TermGrunt' in sys.argv[1] FF00FF' in str(cfg['in']['path']): # 'TermGrunt.h5' ? args.path.endswith ('bin'):
# bOld_FF00FF = True
# cfg['in'].update({
# 'header': 'TERM',
# 'dt_from_utc': timedelta(hours=-1),
# 'fs': 1, 'b_time_fromtimestamp': True,
# 'b_time_fromtimestamp_source': False})
# else: # 'Katran.h5'
# cfg['in'].update({
# 'delimiter_hex': '000000E6',
# 'header': 'P, Temp, Cond',
# 'dt_from_utc': timedelta(hours=0),
# 'fs': 10, 'b_time_fromtimestamp': False,
# 'b_time_fromtimestamp_source': False})
set_field_if_no(
cfg['in'], 'dtype', 'uint{:d}'.format(2**(3 + np.searchsorted(
2**np.array([3, 4, 5, 6, 7]) > np.array(
8 * (cfg['in']['data_word_len'] - 1)), 1))))
# Prepare cpecific format loading and writing
set_field_if_no(cfg['in'], 'coltime', [])
cfg['in'] = init_input_cols(cfg['in'])
cfg['out']['names'] = np.array(cfg['in']['dtype'].names)[ \
cfg['in']['cols_loaded_save_b']]
cfg['out']['formats'] = [
cfg['in']['dtype'].fields[n][0] for n in cfg['out']['names']
]
cfg['out']['dtype'] = np.dtype({
'formats': cfg['out']['formats'],
'names': cfg['out']['names']
})
h5init(cfg['in'], cfg['out'])
# cfg['Period'] = 1.0 / cfg['in']['fs'] # instead Second can use Milli / Micro / Nano:
# cfg['pdPeriod'] = pd.to_timedelta(cfg['Period'], 's')
# #pd.datetools.Second(cfg['Period'])\
# if 1 % cfg['in']['fs'] == 0 else\
# pd.datetools.Nano(cfg['Period'] * 1e9)
# log table of loaded files. columns: Start time, file name, and its index in array off all loaded data:
log_item = cfg['out']['log'] = {
} # fields will have: 'fileName': None, 'fileChangeTime': None, 'rows': 0
strLog = ''
# from collections import namedtuple
# type_log_files = namedtuple('type_log_files', ['label','iStart'])
# log.sort(axis=0, order='log_item['Date0']')#sort files by time
dfLogOld, cfg['out']['db'], cfg['out'][
'b_skip_if_up_to_date'] = h5temp_open(**cfg['out'])
if 'log' in cfg['program'].keys():
f = open(PurePath(sys_argv[0]).parent / cfg['program']['log'],
'a',
encoding='cp1251')
f.writelines(
datetime.now().strftime('\n\n%d.%m.%Y %H:%M:%S> processed ' +
str(cfg['in']['nfiles']) + ' file' +
's:' if cfg['in']['nfiles'] > 1 else ':'))
b_remove_duplicates = False # normally no duplicates but will if detect
# Config specially for readBinFramed
set_field_if_no(cfg['in'], 'b_byte_order_is_big_endian', True)
set_field_if_no(cfg['in'], 'b_baklan', False)
set_field_if_no(cfg['in'], 'b_time_fromtimestamp_source', False)
cfg['out']['fs'] = cfg['in']['fs']
if True:
## Main circle ############################################################
for i1_file, path_in in h5_dispenser_and_names_gen(
cfg['in'], cfg['out']):
l.info('{}. {}: '.format(i1_file, path_in.name))
# Loading data
if bOld_FF00FF:
V = readFF00FF(path_in, cfg)
iFrame = np.arange(len(V))
else:
V, iFrame = readBinFramed(path_in, cfg['in'])
if ('b_time_fromtimestamp' in cfg['in'] and cfg['in']['b_time_fromtimestamp']) or \
('b_time_fromtimestamp_source' in cfg['in'] and cfg['in']['b_time_fromtimestamp_source']):
path_in_rec = os_path.join(
'd:\\workData\\_source\\BalticSea\\151021_T1Grunt_Pregol\\_source\\not_corrected',
os_path.basename(path_in)[:-3] + 'txt'
) if cfg['in']['b_time_fromtimestamp_source'] else path_in
log_item['Date0'] = datetime.fromtimestamp(
os_path.getmtime(path_in_rec)) # getctime is bad
log_item['Date0'] -= iFrame[-1] * timedelta(
seconds=1 / cfg['in']['fs']
) # use for computer filestamp at end of recording
else:
log_item['Date0'] = datetime.strptime(
path_in.stem, cfg['in']['filename2timestart_format'])
log_item['Date0'] += cfg['in']['dt_from_utc']
tim = log_item['Date0'] + iFrame * timedelta(
seconds=1 / cfg['in']['fs']
) # tim = pd.date_range(log_item['Date0'], periods=np.size(V, 0), freq=cfg['pdPeriod'])
df = pd.DataFrame(
V.view(dtype=cfg['out']['dtype']), # np.uint16
columns=cfg['out']['names'],
index=tim)
# pd.DataFrame(V, columns=cfg['out']['names'], dtype=cfg['out']['formats'], index=tim)
if df.empty: # log['rows']==0
print('No data => skip file')
continue
df, tim = set_filterGlobal_minmax(df,
cfg_filter=cfg['filter'],
log=log_item,
dict_to_save_last_time=cfg['in'])
if log_item['rows_filtered']:
print('filtered out {}, remains {}'.format(
log_item['rows_filtered'], log_item['rows']))
if not log_item['rows']:
l.warning('no data! => skip file')
continue
elif log_item['rows']:
print(
'.', end=''
) # , divisions=d.divisions), divisions=pd.date_range(tim[0], tim[-1], freq='1D')
else:
l.warning('no data! => skip file')
continue
# Append to Store
h5_append(cfg['out'], df.astype('int32'), log_item)
if 'txt' in cfg['program'].keys(): # can be saved as text too
np.savetxt(cfg['program']['txt'],
V,
delimiter='\t',
newline='\n',
header=cfg['in']['header'] + log_item['fileName'],
fmt='%d',
comments='')
try:
if b_remove_duplicates:
for tblName in (cfg['out']['table'] +
cfg['out']['tableLog_names']):
cfg['out']['db'][tblName].drop_duplicates(
keep='last', inplace=True) # subset='fileName',?
if len(strLog):
print('Create index', end=', ')
for tblName in (cfg['out']['table'] +
cfg['out']['tableLog_names']):
cfg['out']['db'].create_table_index(tblName,
columns=['index'],
kind='full')
else:
print('done nothing')
except Exception as e:
l.exception('The end. There are error ')
import traceback, code
from sys import exc_info as sys_exc_info
tb = sys_exc_info()[2] # type, value,
traceback.print_exc()
last_frame = lambda tb=tb: last_frame(tb.tb_next) if tb.tb_next else tb
frame = last_frame().tb_frame
ns = dict(frame.f_globals)
ns.update(frame.f_locals)
code.interact(local=ns)
# sort index if have any processed data (needed because ``ptprepack`` not closses hdf5 source if it not finds data)
if cfg['in'].get('time_last'):
failed_storages = h5move_tables(cfg['out'])
print('Ok.', end=' ')
h5index_sort(
cfg['out'],
out_storage_name=f"{cfg['out']['db_path'].stem}-resorted.h5",
in_storages=failed_storages)
def main():
print('\n' + this_prog_basename(__file__), end=' started. ')
import_pattern = """
ImportFileHDF5(u'../181005_ABP44.h5', [u'/181017inclinometers/incl03/coef', u'/181017inclinometers/incl03/table'], linked=True, namemap={u'/181017inclinometers/incl03/coef/G/A': u'Ag_old_inv', u'/181017inclinometers/incl03/coef/G/C': u'Cg', u'/181017inclinometers/incl03/coef/H/A': u'Ah_old_inv', u'/181017inclinometers/incl03/coef/H/C': u'Ch', u'/181017inclinometers/incl03/table/Ax': u'countsAx', u'/181017inclinometers/incl03/table/Ay': u'countsAy', u'/181017inclinometers/incl03/table/Az': u'countsAz', u'/181017inclinometers/incl03/table/Mx': u'countsMx', u'/181017inclinometers/incl03/table/My': u'countsMy', u'/181017inclinometers/incl03/table/Mz': u'countsMz', u'/181017inclinometers/incl03/table/Temp': u'sT'}, renames={u'Vabs0': u'kVabs'})
""".replace('{', '{{{{').replace('}', '}}}}').replace('incl03', '{probe}')
cfg = veuszPropagate.main([
Path(veuszPropagate.__file__).parent.with_name('veuszPropagate.ini'),
'--data_yield_prefix',
'-',
'--path',
str(path_db
), # use for custom loading from db and some source is required
'--tables_list',
'incl{}',
'--pattern_path',
str(path_db.parent.joinpath(
fr'inclinometer\*.vsz')), # {str_date}incl03
'--before_next',
'restore_config',
# '--add_custom_list',
# 'USEtime',
# '--add_custom_expressions_list',
# """
# "[['2018-10-03T17:23:00', '2018-10-03T18:25:00']]"
# """,
'--b_update_existed',
'True',
'--export_pages_int_list',
'0', # ''4 ', #''
'--b_interact',
'0',
'--b_images_only',
'True'
])
if not cfg:
return 1
cor_savings = veuszPropagate.co_savings(cfg)
cor_savings.send(None)
# Custom loading from db
cfg['in']['db_parent_path'] = f'{str_date}inclinometers'
if not 'out' in cfg:
cfg['out'] = {}
cfg['out']['f_file_name'] = lambda tbl: f'{str_date}{tbl}'
def ge_names(cfg, f_file_name=lambda x: x):
"""
Replasing for veuszPropagate.ge_names() to use tables instead files
:param cfg:
:return:
"""
with pd.HDFStore(cfg['in']['path'], mode='r') as store:
if len(cfg['in']['tables']) == 1:
cfg['in']['tables'] = h5find_tables(
store,
cfg['in']['tables'][0],
parent_name=cfg['in']['db_parent_path'])
for tbl in cfg['in']['tables']:
# if int(tbl[-2:]) in {5,9,10,11,14,20}:
yield f_file_name(tbl)
# def f_file_name(file_name):
# p = Path(file_name)
# return str(p.with_name(p.stem + name_add_time + p.suffix))
gen_veusz_and_logs = veuszPropagate.load_to_veusz(
ge_names(cfg, cfg['out']['f_file_name']), cfg)
cor_send_data = veuszPropagate.co_send_data(gen_veusz_and_logs, cfg,
cor_savings)
cfgin_update = None
while True: # for vsz_data, log in cor_send_data.send(cfgin_update):
try:
vsz_data, log = cor_send_data.send(cfgin_update)
# will delete cfg['in']['tables']
except (GeneratorExit, StopIteration, Ex_nothing_done):
print('ok>')
break
# i_ = log['fileName'].rfind('incl')
probe = log['fileName'].replace(
str_date, ''
) # re.sub('^[\d_]*', '', in_file.stem), # use last digits (date part) log['fileName'][:(i_)]
if not cfg['out']['b_images_only']:
cfgin_update = {
'add_custom': ['USEtime'],
# 'add_custom_expressions': [expr],
'eval': [import_pattern.format(probe=probe)]
}
def main():
print('\n' + this_prog_basename(__file__), end=' started. ')
# try:
# cfg['in']= init_file_names(cfg['in'])
# except Ex_nothing_done as e:
# print(e.message)
# return()
# gen_names = ge_names(cfg)
gen_data = veuszPropagate([os_path.join(os_path.dirname(
file_veuszPropagate), 'veuszPropagate.ini'),
'--data_yield_prefix', 'Hxyz', '--path', cfg['in']['path'], '--pattern_path',
r'd:\workData\BalticSea\171003_ANS36\inclinometr\171015_intercal_on_board\~pattern~.vsz',
'--eval_list',
"""
'ImportFileCSV(u"{nameRFE}", blanksaredata=True, encoding="ascii", headermode="1st", linked=True, dsprefix=u"counts", rowsignore=2, skipwhitespace=True)',
"TagDatasets(u'source', [u'countsAx', u'countsAy', u'countsAz', u'countsBattery', u'countsDay', u'countsHour', u'countsMinute', u'countsMonth', u'countsMx', u'countsMy', u'countsMz', u'countsSecond', u'countsTemp', u'countsYear'])"
""",
# '--import_method', 'ImportFileCSV',
'--add_custom_list', 'Ah_old, Ch',
'--add_custom_expressions_list',
"""
'float64([[ 1, 0, 0],\
[0, 1, 0],\
[0, 0, 1]])\
',
'float64([[0, 0, 0]])'
""", '--before_next', 'restore_config',
'--export_pages_int_list', '1'])
cfgin_update = None
while True:
try:
d = gen_data.send(cfgin_update)
except (GeneratorExit, StopIteration):
print('ok>')
break
except Exception as e:
print('There are error ', standard_error_info(e))
continue
Hxyz = d['Hxyz']
# Hxyz = np.column_stack((a['Mx'], a['My'], a['Mz']))[slice(*iUseTime.flat)].T
if len(Hxyz) < 3 or not np.prod(Hxyz.shape): # 3 is ok but may be empty
print('\nNo data from Veusz!\n')
bBad = True
else:
Ah, b = calibrate(Hxyz)
if Ah[0, 0] > 10:
print('\nBad calibration!\n')
bBad = True
else:
bBad = False
if bBad:
print('use 1st coef!')
b = np.float64([[46, -166, -599]])
Ah = np.float64([[0.0054, -0.0001, -0.0001],
[-0.0001, 0.0069, -0.0001],
[-0.0001, -0.0001, 0.0089]])
"""
AddCustom('definition', u'Ch', u'float64([[60,-160,-650]])')
AddCustom('definition', u'Ah_old', u'float64([[50,0,0],\n[0,65,0],\n[0,0,90]])*1e-4')
"""
# calibrate_plot(Hxyz, Ah, b)
A_str, b_str = coef2str(Ah, b)
# b_str= 'float64([{}])'.format(b_str)
if not bBad:
print('calibration coefficients calculated:',
'\nA = \n', A_str, '\nb = \n', b_str)
cfgin_update = {'add_custom_expressions': [A_str, b_str]}
def main(config: ConfigType) -> None:
"""
----------------------------
Calculates coefficients from
data of Pandas HDF5 store*.h5
and saves them back
----------------------------
1. Obtains command line arguments (for description see my_argparser()) that can be passed from new_arg and ini.file
also.
2. Loads device data of calibration in laboratory from hdf5 database (cfg['in']['db_path'])
2. Calibrates configured by cfg['in']['channels'] channels ('accelerometer' and/or 'magnetometer'): soft iron
3. Wrong implementation - not use cfg['in']['timerange_nord']! todo: Rotate compass using cfg['in']['timerange_nord']
:param config: returns cfg if new_arg=='<cfg_from_args>' but it will be None if argument
argv[1:] == '-h' or '-v' passed to this code
argv[1] is cfgFile. It was used with cfg files:
"""
global cfg, l
cfg = main_init(config, cs_store_name, __file__=None)
cfg = main_init_input_file(cfg, cs_store_name)
# input data tables may be defined by 'probes_prefix' and 'probes' fields of cfg['in']
if cfg['in']['probes'] or not len(cfg['in']['tables']):
if cfg['in']['probes']:
cfg['in']['tables'] = [
f"{cfg['in']['probes_prefix']}{probe:0>2}"
for probe in cfg['in']['probes']
]
elif cfg['in']['probes_prefix']:
cfg['in']['tables'] = [f"{cfg['in']['probes_prefix']}.*"]
# else: # default config
# cfg['in']['tables'] = ['.*']
#h5init(cfg['in'], cfg['out'])
#cfg['out']['dt_from_utc'] = 0
# cfg = cfg_from_args(my_argparser(), new_arg)
lf.info("{:s}({:s}) for channels: {} started. ",
this_prog_basename(__file__), ', '.join(cfg['in']['tables']),
cfg['in']['channels'])
fig = None
fig_filt = None
fig_save_dir_path = cfg['in']['db_path'].parent
with pd.HDFStore(cfg['in']['db_path'], mode='r') as store:
if len(cfg['in']['tables']) == 1:
cfg['in']['tables'] = h5find_tables(store, cfg['in']['tables'][0])
coefs = {}
for itbl, tbl in enumerate(cfg['in']['tables'], start=1):
probe_number = int(re.findall('\d+', tbl)[0])
lf.info(f'{itbl}. {tbl}: ')
if isinstance(cfg['in']['timerange'],
Mapping): # individual interval for each table
if probe_number in cfg['in']['timerange']:
timerange = cfg['in']['timerange'][probe_number]
else:
timerange = None
else:
timerange = cfg['in'][
'timerange'] # same interval for each table
a = load_hdf5_data(store, table=tbl, t_intervals=timerange)
# iUseTime = np.searchsorted(stime, [np.array(s, 'datetime64[s]') for s in np.array(strTimeUse)])
# Calibrate channels of 'accelerometer' or/and 'magnetometer'
coefs[tbl] = {}
for channel in cfg['in']['channels']:
print(f' channel "{channel}"', end=' ')
(col_str, coef_str) = channel_cols(channel)
# filtering # col_str == 'A'?
if True:
b_ok = np.zeros(a.shape[0], bool)
for component in ['x', 'y', 'z']:
b_ok |= is_works(
a[col_str + component],
noise=cfg['filter']['no_works_noise'][channel])
lf.info('Filtered not working area: {:2.1f}%',
(b_ok.size - b_ok.sum()) * 100 / b_ok.size)
# vec3d = np.column_stack(
# (a[col_str + 'x'], a[col_str + 'y'], a[col_str + 'z']))[:, b_ok].T # [slice(*iUseTime.flat)]
vec3d = a.loc[
b_ok, [col_str + 'x', col_str + 'y', col_str +
'z']].to_numpy(float).T
index = a.index[b_ok]
vec3d, b_ok, fig_filt = filter_channes(
vec3d,
index,
fig_filt,
fig_save_prefix=
f"{fig_save_dir_path / tbl}-'{channel}'",
blocks=cfg['filter']['blocks'],
offsets=cfg['filter']['offsets'],
std_smooth_sigma=cfg['filter']['std_smooth_sigma'])
A, b = calibrate(vec3d)
window_title = f"{tbl} '{channel}' channel ellipse"
fig = calibrate_plot(vec3d,
A,
b,
fig,
window_title=window_title)
fig.savefig(fig_save_dir_path / (window_title + '.png'),
dpi=300,
bbox_inches="tight")
A_str, b_str = coef2str(A, b)
lf.info(
'Calibration coefficients calculated: \nA = \n{:s}\nb = \n{:s}',
A_str, b_str)
coefs[tbl][channel] = {'A': A, 'b': b}
# Zeroing Nord direction
timerange_nord = cfg['in']['timerange_nord']
if isinstance(timerange_nord, Mapping):
timerange_nord = timerange_nord.get(probe_number)
if timerange_nord:
coefs[tbl]['M']['azimuth_shift_deg'] = zeroing_azimuth(
store, tbl, timerange_nord, calc_vel_flat_coef(coefs[tbl]),
cfg['in'])
else:
lf.info('not zeroing North')
# Write coefs to each of output tables named same as input
for cfg_output in (['in', 'out'] if cfg['out'].get('db_path') else ['in']):
lf.info('Writing to {}', cfg[cfg_output]['db_path'])
for itbl, tbl in enumerate(cfg['in']['tables'], start=1):
# i_search = re.search('\d*$', tbl)
# for channel in cfg['in']['channels']:
# (col_str, coef_str) = channel_cols(channel)
# dict_matrices = {f'//coef//{coef_str}//A': coefs[tbl][channel]['A'],
# f'//coef//{coef_str}//C': coefs[tbl][channel]['b'],
# }
# if channel == 'M':
# if coefs[tbl]['M'].get('azimuth_shift_deg'):
# dict_matrices[f'//coef//{coef_str}//azimuth_shift_deg'] = coefs[tbl]['M']['azimuth_shift_deg']
# # Coping probe number to coefficient to can manually check when copy manually
# if i_search:
# try:
# dict_matrices['//coef//i'] = int(i_search.group(0))
# except Exception as e:
# pass
dict_matrices = dict_matrices_for_h5(coefs[tbl], tbl,
cfg['in']['channels'])
h5copy_coef(None,
cfg[cfg_output]['db_path'],
tbl,
dict_matrices=dict_matrices)
print('Ok>', end=' ')
def main():
print('\n' + this_prog_basename(__file__), end=' started. ')
# try:
# cfg['in']= init_file_names(cfg['in'])
# except Ex_nothing_done as e:
# print(e.message)
# return()
# gen_names = ge_names(cfg)
cfg = veuszPropagate([
os_path.join(os_path.dirname(file_veuszPropagate),
'veuszPropagate_incl.ini'),
# '--path', r'd:\workData\BalticSea\171003_ANS36\inclinometr\171017\171017#??.TXT', # windows
'--path',
r'/mnt/D/workData/BalticSea/171003Strahov/inclinometr/171017/171017#??.TXT',
# in Linux match the case is important
'--pattern_path',
'171017#01.vsz',
'--log',
os_path.join(os_path.dirname(file_veuszPropagate),
'logs/viewsPropagate.log'),
'--data_yield_prefix',
'Hxyz',
'--eval_list',
"""
"ImportFileCSV(u'{nameRFE}', blanksaredata=True, encoding='ascii', headermode='1st', linked=True, dsprefix='counts', rowsignore=2, skipwhitespace=True)",
"TagDatasets(u'source', [u'countsAx', u'countsAy', u'countsAz', u'countsBattery', u'countsDay', u'countsHour', u'countsMinute', u'countsMonth', u'countsMx', u'countsMy', u'countsMz', u'countsSecond', u'countsTemp', u'countsYear'])"
""",
# '--import_method', 'ImportFileCSV',
'--add_custom_list',
'Ag_old', # , Ch',
'--add_custom_expressions_list',
# """
# 'float64([[1,0,0],[0,1,0],[0,0,1]])/16384.0'
# """
None,
'--before_next_list',
'restore_config', # 'Close(), '
'--export_pages_int_list',
'1,3,4,5', #
'--veusz_path',
'/usr/lib64/python3.6/site-packages/veusz-2.1.1-py3.6-linux-x86_64.egg/veusz',
# '/home/korzh/Python/other_sources/veusz/veusz',
'-V',
'DEBUG'
])
# os_path.dirname( #
if not cfg:
exit(0)
file_cal_pattern = cfg['in']['path'].with_name(
'171121zeroing/INKL_{:03}_data.txt')
iFile = cfg['in']['start_file'] # inclinometers are numbered from 1
cfgin_update = None
while True:
iFile += 1
try:
d, log = cfg['co_send_data'].send(cfgin_update)
except (GeneratorExit, StopIteration):
print('ok>')
break
except Exception as e:
print('There are error ', standard_error_info(e))
continue
i = int(log['fileName'].split('#')[1])
Hxyz = d['Hxyz']
# Hxyz = np.column_stack((a['Mx'], a['My'], a['Mz']))[slice(*iUseTime.flat)].T
if len(Hxyz) < 3 or not np.prod(
Hxyz.shape): # 3 is ok but may be empty
print('\nNo data from Veusz!\n')
bBad = True
else:
file_data = file_cal_pattern.format(i)
with open(file_data) as f:
Ag_str = f.read()
Ag_str = re.sub(r'((?<=\d)([ ]+))|(?=\n)', r',\1', Ag_str)
Ag = np.float64(eval(Ag_str))
Ag_str = 'float64({})'.format(Ag_str)
if Ag[0, 0] > 10:
print('\nBad calibration!\n')
bBad = True
else:
bBad = False
if bBad:
print('using default coef!')
Ag_str = 'float64([[1,0,0],[0,1,0],[0,0,1]])/16384.0'
"""
AddCustom('definition', u'Ch', u'float64([[60,-160,-650]])')
AddCustom('definition', u'Ah_old', u'float64([[50,0,0],\n[0,65,0],\n[0,0,90]])*1e-4')
"""
# calibrate_plot(Hxyz, Ah, b)
# A_str, b_str = coef2str(Ah, b)
# b_str= 'float64([{}])'.format(b_str)
if not bBad:
print(
'calibration coefficient loaded ({}): '.format(
os_path.basename(file_data)), 'A = \n', Ag_str)
cfgin_update = {'add_custom_expressions': [Ag_str]}
