def main(new_arg): cfg = cfg_from_args(my_argparser(), new_arg) if not cfg: return if cfg['program']['return'] == '<cfg_from_args>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) print('\n' + this_prog_basename(__file__), end=' started. ') try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names(**cfg['in'], b_interact=cfg['program']['b_interact']) except Ex_nothing_done as e: print(e.message) return () # cfg = {'in': {}} # cfg['in']['path'] = \ # r'd:\workData\BalticSea\181005_ABP44\navigation\2018-10-06tracks_copy.gpx' # r'd:\WorkData\_experiment\_2017\tracker\170502.gpx' # r'd:\workData\_experiment\2016\GPS_tracker\sms_backup\sms-20160225135922.gpx' for ifile, nameFull in enumerate(cfg['in']['paths'], start=1): print('{}. {}'.format(ifile, nameFull), end=', ') gpx2csv(nameFull)
def main_cfg(cfg: DictConfig): # hydra required arg, not use when call """ ---------------------------- Add data from CSV-like files to Pandas HDF5 store*.h5 ---------------------------- """ #print(OmegaConf.to_yaml(cfg)) global lf # cfg = cfg_from_args(argparser_files(), **kwargs) if not cfg.program.return_: print('Can not initialise') return cfg elif cfg.program.return_ == '<cfg_from_args>': # to help testing return cfg lf = LoggingStyleAdapter(init_logging(logging, None, cfg.program.log, cfg.rogram.verbose)) print('\n' + this_prog_basename(__file__), end=' started. ') try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in']['path'] = init_file_names( **cfg['in'], b_interact=cfg['program']['b_interact']) except Ex_nothing_done as e: print(e.message) return () return cfg
def main_init(cfg: DictConfig) -> DictConfig: """ Common startup initializer :param cfg: :return: """ #global lf # if cfg.search_path is not None: # override_path = hydra.utils.to_absolute_path(cfg.search_path) # override_conf = OmegaConf.load(override_path) # cfg = OmegaConf.merge(cfg, override_conf) print("Working directory : {}".format(os.getcwd())) print(OmegaConf.to_yaml(cfg)) # cfg = cfg_from_args(argparser_files(), **kwargs) if not cfg.program.return_: print('Can not initialise') return cfg elif cfg.program.return_ == '<cfg_from_args>': # to help testing return cfg hydra.verbose = 1 if cfg.program.verbose == 'DEBUG' else 0 # made compatible to my old cfg print('\n' + this_prog_basename(__file__), end=' started. ') try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names(**cfg.input, b_interact=cfg.program.b_interact) except Ex_nothing_done as e: print(e.message) return () return cfg
def main(new_arg=None): new_arg = [ r'.\h5toGpx_CTDs.ini', '--db_path', r'd:\workData\BalticSea\170614_ANS34\170614Strahov.h5', '--path', r'd:\workData\BalticSea\170614_ANS34\Baklan\2017*p1d5.txt', '--gpx_names_fun_format', '+{:02d}', '--gpx_symbols_list', "'Navaid, Orange'" ] # 'db_path', r'd:\workData\BalticSea\171003_ANS36\171003Strahov.h5' cfg = cfg_from_args(my_argparser(), new_arg) if not cfg: return if new_arg == '<return_cfg>': # to help testing return cfg print('\n' + this_prog_basename(__file__), 'started', end=' ') if not cfg['out']['path'].is_absolute(): cfg['out']['path'] = cfg['in']['db_path'].parent / cfg['out'][ 'path'] # set relative to cfg['in']['db_path'] try: print(end='Data ') cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names(**cfg['in']) # may interact except Ex_nothing_done as e: print(e.message) return # or raise FileNotFoundError? itbl = 0 # compile functions if defined in cfg or assign default gpx_symbols = init_gpx_symbols_fun(cfg['out']) gpx_names_funs = ["i+1"] gpx_names_fun = eval( compile( "lambda i, row: '{}'.format({})".format( cfg['out']['gpx_names_fun_format'], gpx_names_funs[itbl]), [], 'eval')) tim = filename2date([f for f in ge_names(cfg)]) with pd.HDFStore(cfg['in']['db_path'], mode='r') as storeIn: # dfL = storeIn[tblD + '/logFiles'] nav2add = h5select(storeIn, cfg['in']['table_nav'], ['Lat', 'Lon', 'DepEcho'], tim)[0] rnav_df_join = nav2add.assign( itbl=itbl) # copy/append on first/next cycle # Save to gpx waypoints # if 'gpx_names_funs' in cfg['out'] and \ # len(cfg['out']['gpx_names_funs'])>itbl: # # gpx_names = eval(compile('lambda i: str({})'.format( # cfg['out']['gpx_names_funs'][itbl]), [], 'eval')) # save_to_gpx(rnav_df_join[-len(nav2add):], cfg['out']['path'].with_name('fileNames'), gpx_obj_namef=gpx_names_fun, waypoint_symbf=gpx_symbols, cfg_proc=cfg['process'])
def main(): print('\n' + this_prog_basename(__file__), end=' started. ') cfg = veuszPropagate.main([ Path(veuszPropagate.__file__).parent.with_name('veuszPropagate.ini'), '--data_yield_prefix', '-', '--path', str(path_db ), # use for custom loading from db and some source is required '--pattern_path', str( path_db.parent.joinpath( r'CTD_S&S48M#1253\profiles\181005_1810-1813.vsz')), # '--before_next', 'restore_config', # '--add_custom_list', # 'USEtime', # '--add_custom_expressions_list', # """ # "[['2018-10-03T17:23:00', '2018-10-03T18:25:00']]" # """, '--b_update_existed', 'True', # '--export_pages_int_list', '', '--b_interact', '0', # '--b_images_only', 'True' ]) if not cfg: return 1 # Custom loading from db cfg['in']['table_log'] = '/CTD_SST_48M/logRuns' cfg['in']['db_path'] = path_db gen_veusz_and_logs = veuszPropagate.load_to_veusz( h5log_names_gen(cfg['in']), cfg) cor_savings = veuszPropagate.co_savings(cfg) cor_savings.send(None) cor_send_data = veuszPropagate.co_send_data(gen_veusz_and_logs, cfg, cor_savings) cfgin_update = None while True: # for vsz_data, log in cor_send_data.send(cfgin_update): try: vsz_data, log = cor_send_data.send(cfgin_update) # will delete cfg['in']['tables'] except (GeneratorExit, StopIteration, Ex_nothing_done): print('ok>') break custom_expressions_use_time = "[['{Index:%Y-%m-%dT%H:%M:%S}', '{DateEnd:%Y-%m-%dT%H:%M:%S}']]".format_map( cfg['log_row']) if not cfg['out']['b_images_only']: cfgin_update = { 'add_custom': ['USE_timeRange'], 'add_custom_expressions': [custom_expressions_use_time] }
def main_init( cfg, cs_store_name, __file__=None, ): """ - prints parameters - prints message that program (__file__ or cs_store_name) started - converts cfg parameters to types according to its prefixes/suffixes names (see ini2dict()) :param cfg: :param cs_store_name: :param __file__: :return: """ # global lf # if cfg.search_path is not None: # override_path = hydra.utils.to_absolute_path(cfg.search_path) # override_conf = OmegaConf.load(override_path) # cfg = OmegaConf.merge(cfg, override_conf) print("Working directory : {}".format(os.getcwd())) # todo: print only if config changed print(OmegaConf.to_yaml(cfg)) # cfg = cfg_from_args(argparser_files(), **kwargs) if not cfg.program.return_: print('Can not initialise') return cfg elif cfg.program.return_ == '<cfg_from_args>': # to help testing return cfg hydra.verbose = 1 if cfg.program.verbose == 'DEBUG' else 0 # made compatible to my old cfg print('\n' + this_prog_basename(__file__) if __file__ else cs_store_name, end=' started. ') try: cfg_t = ini2dict( cfg) # fields named with type pre/suffixes are converted except MissingMandatoryValue as e: lf.error(standard_error_info(e)) raise Ex_nothing_done() except Exception: lf.exception('startup error') # OmegaConf.update(cfg, "in", cfg.input, merge=False) # error # to allow non primitive types (cfg.out['db']) and special words field names ('in'): # cfg = omegaconf.OmegaConf.to_container(cfg) return cfg_t
def main_init( cfg, cs_store_name, __file__=None, ): """ Common startup initializer: - finds input files - asks user to proceed if need - prints message that program (__file__ or cs_store_name) started :param cfg: :return: Note: if new_arg=='<cfg_from_args>' returns cfg but it will be None if argument argv[1:] == '-h' or '-v' passed to this code argv[1] is cfgFile. It was used with cfg files: 'csv2h5_nav_supervisor.ini' 'csv2h5_IdrRedas.ini' 'csv2h5_Idronaut.ini' """ # global lf # if cfg.search_path is not None: # override_path = hydra.utils.to_absolute_path(cfg.search_path) # override_conf = OmegaConf.load(override_path) # cfg = OmegaConf.merge(cfg, override_conf) print("Working directory : {}".format(os.getcwd())) print(OmegaConf.to_yaml(cfg)) # cfg = cfg_from_args(argparser_files(), **kwargs) if not cfg.program.return_: print('Can not initialise') return cfg elif cfg.program.return_ == '<cfg_from_args>': # to help testing return cfg hydra.verbose = 1 if cfg.program.verbose == 'DEBUG' else 0 # made compatible to my old cfg print('\n' + this_prog_basename(__file__) if __file__ else cs_store_name, end=' started. ') cfg_t = ini2dict(cfg) # fields named with type pre/suffixes are converted # OmegaConf.update(cfg, "in", cfg.input, merge=False) # error # to allow non primitive types (cfg.out['db']) and special words field names ('in'): # cfg = omegaconf.OmegaConf.to_container(cfg) return cfg_t
def main_cfg(cfg: DictConfig): global l # cfg = cfg_from_args(argparser_files(), **kwargs) if not cfg or not cfg['program'].get('return'): print('Can not initialise') return cfg elif cfg['program']['return'] == '<cfg_from_args>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) print('\n' + this_prog_basename(__file__), end=' started. ') try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in']['path'] = init_file_names( **cfg['in'], b_interact=cfg['program']['b_interact']) except Ex_nothing_done as e: print(e.message) return () return cfg
def main(new_arg=None): """ :param new_arg: returns cfg if new_arg=='<cfg_from_args>' but it will be None if argument argv[1:] == '-h' or '-v' passed to this code argv[1] is cfgFile. It was used with cfg files: 'csv2h5_nav_supervisor.ini' 'csv2h5_IdrRedas.ini' 'csv2h5_Idronaut.ini' :return: """ global l cfg = cfg_from_args(my_argparser(), new_arg) if not cfg or not cfg['program'].get('return'): print('Can not initialise') return cfg elif cfg['program']['return'] == '<cfg_from_args>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) print('\n' + this_prog_basename(__file__), end=' started. ') try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names(**{ **cfg['in'], 'path': cfg['in']['db_path'] }, b_interact=cfg['program']['b_interact']) set_field_if_no( cfg['in'], 'tables_log', '{}/logFiles' ) # will be filled by each table from cfg['in']['tables'] cfg['in']['query'] = query_time_range(**cfg['in']) set_field_if_no(cfg['out'], 'db_path', cfg['in']['db_path']) # cfg['out'] = init_file_names(cfg['out'], , path_field='db_path') except Ex_nothing_done as e: print(e.message) return () # args = parser.parse_args() # args.verbose= args.verbose[0] # try: # cfg= ini2dict(args.cfgFile) # cfg['in']['cfgFile']= args.cfgFile # except IOError as e: # print('\n==> '.join([a for a in e.args if isinstance(a,str)])) #e.message # raise(e) # Open text log if 'log' in cfg['program'].keys(): dir_create_if_need(os_path.dirname(cfg['program']['log'])) flog = open(cfg['program']['log'], 'a+', encoding='cp1251') cfg['out']['log'] = OrderedDict({'fileName': None, 'fileChangeTime': None}) # Prepare saving to csv if 'file_names_add_fun' in cfg['out']: file_names_add = eval( compile(cfg['out']['file_names_add_fun'], '', 'eval')) else: file_names_add = lambda i: '.csv' # f'_{i}.csv' # Prepare data for output store and open it if cfg['out']['tables'] == ['None']: # will not write new data table and its log cfg['out']['tables'] = None # cfg['out']['tables_log'] = None # for _runs cfg will be redefined (this only None case that have sense?) h5init(cfg['in'], cfg['out']) # store, dfLogOld = h5temp_open(**cfg['out']) cfg_fileN = os_path.splitext(cfg['in']['cfgFile'])[0] out_tables_log = cfg['out'].get('tables_log') if cfg_fileN.endswith('_runs') or (bool(out_tables_log) and 'logRuns' in out_tables_log[0]): # Will calculate only after filter # todo: calculate derived parameters before were they are bad (or replace all of them if any bad?) func_before_cycle = lambda x: None func_before_filter = lambda df, log_row, cfg: df func_after_filter = lambda df, cfg: log_runs(df, cfg, cfg['out']['log'] ) # this table will be added: cfg['out']['tables_log'] = [cfg['out']['tables'][0] + '/logRuns'] cfg['out'][ 'b_log_ready'] = True # to not apdate time range in h5_append() # Settings to not affect main data table and switch off not compatible options: cfg['out']['tables'] = [] cfg['out'][ 'b_skip_if_up_to_date'] = False # todo: If False check it: need delete all previous result of CTD_calc() or set min_time > its last log time. True not implemented? cfg['program'][ 'b_log_display'] = False # can not display multiple rows log if 'b_save_images' in cfg['extract_runs']: cfg['extract_runs']['path_images'] = cfg['out'][ 'db_path'].with_name('_subproduct') dir_create_if_need(cfg['extract_runs']['path_images']) else: if 'brown' in cfg_fileN.lower(): func_before_cycle = load_coef if 'Lat' in cfg['in']: func_before_filter = lambda *args, **kwargs: add_ctd_params( process_brown(*args, **kwargs), kwargs['cfg']) else: func_before_filter = process_brown else: func_before_cycle = lambda x: None def ctd_coord_and_params(df: pd.DataFrame, log_row, cfg): coord_data_col_ensure(df, log_row) return add_ctd_params(df, cfg) func_before_filter = ctd_coord_and_params func_after_filter = lambda df, cfg: df # nothing after filter func_before_cycle(cfg) # prepare: usually assign data to cfg['for'] if cfg['out'].get('path_csv'): dir_create_if_need(cfg['out']['path_csv']) # Load data Main circle ######################################### # Open input store and cicle through input table log records qstr_trange_pattern = "index>=Timestamp('{}') & index<=Timestamp('{}')" iSt = 1 dfLogOld, cfg['out']['db'], cfg['out'][ 'b_skip_if_up_to_date'] = h5temp_open(**cfg['out']) b_out_db_is_different = cfg['out']['db'] is not None and cfg['out'][ 'db_path_temp'] != cfg['in']['db_path'] # Cycle for each table, for each row in log: # for path_csv in gen_names_and_log(cfg['out'], dfLogOld): with FakeContextIfOpen( lambda f: pd.HDFStore(f, mode='r'), cfg['in']['db_path'], None if b_out_db_is_different else cfg['out']['db'] ) as cfg['in']['db']: # not opens ['in']['db'] if already opened to write for tbl in cfg['in']['tables']: if False: # Show table info nodes = sorted( cfg['out']['db'].root.__members__) # , key=number_key print(nodes) print(tbl, end='. ') df_log = cfg['in']['db'].select(cfg['in']['tables_log'].format(tbl) or tbl, where=cfg['in']['query']) if True: # try: if 'log' in cfg['program'].keys(): nRows = df_log.rows.size flog.writelines(datetime.now().strftime( '\n\n%d.%m.%Y %H:%M:%S> processed ') + f'{nRows} row' + ('s:' if nRows > 1 else ':')) for ifile, r in enumerate(df_log.itertuples(), start=iSt): # name=None print('.', end='') sys_stdout.flush() path_raw = PurePath(r.fileName) cfg['out']['log'].update(fileName=path_raw.name, fileChangeTime=r.fileChangeTime) # save current state cfg['in']['file_stem'] = cfg['out']['log'][ 'fileName'] # for exmple to can extract date in subprogram cfg['in']['fileChangeTime'] = cfg['out']['log'][ 'fileChangeTime'] if cfg['in']['b_skip_if_up_to_date']: have_older_data, have_duplicates = h5del_obsolete( cfg['out'], cfg['out']['log'], dfLogOld) if have_older_data: continue if have_duplicates: cfg['out']['b_remove_duplicates'] = True print('{}. {}'.format(ifile, path_raw.name), end=': ') # Load data qstr = qstr_trange_pattern.format(r.Index, r.DateEnd) df_raw = cfg['in']['db'].select(tbl, qstr) cols = df_raw.columns.tolist() # cfg['in']['lat'] and ['lon'] may be need in add_ctd_params() if Lat not in df_raw if 'Lat_en' in df_log.columns and 'Lat' not in cols: cfg['in']['lat'] = np.nanmean((r.Lat_st, r.Lat_en)) cfg['in']['lon'] = np.nanmean((r.Lon_st, r.Lon_en)) df = func_before_filter(df_raw, log_row=r, cfg=cfg) if df.size: # size is zero means save only log but not data # filter, updates cfg['out']['log']['rows'] df, _ = set_filterGlobal_minmax( df, cfg['filter'], cfg['out']['log']) if 'rows' not in cfg['out']['log']: l.warning('no data!') continue elif isinstance(cfg['out']['log']['rows'], int): print('filtered out {rows_filtered}, remains {rows}'. format_map(cfg['out']['log'])) if cfg['out']['log']['rows']: print('.', end='') else: l.warning('no data!') continue df = func_after_filter(df, cfg=cfg) # Append to Store h5_append(cfg['out'], df, cfg['out']['log'], log_dt_from_utc=cfg['in']['dt_from_utc']) # Copy to csv if cfg['out'].get('path_csv'): fname = '{:%y%m%d_%H%M}-{:%d_%H%M}'.format( r.Index, r.DateEnd) + file_names_add(ifile) if not 'data_columns' in cfg['out']: cfg['out']['data_columns'] = slice(0, -1) # all cols df.to_csv( # [cfg['out']['data_columns']] cfg['out']['path_csv'] / fname, date_format=cfg['out']['text_date_format'], float_format='%5.6g', index_label='Time' ) # to_string, line_terminator='\r\n' # Log to screen (if not prohibited explicitly) if cfg['out']['log'].get('Date0') is not None and ( ('b_log_display' not in cfg['program']) or cfg['program']['b_log_display']): str_log = '{fileName}:\t{Date0:%d.%m.%Y %H:%M:%S}-' \ '{DateEnd:%d. %H:%M:%S%z}\t{rows}rows'.format_map( cfg['out']['log']) # \t{Lat}\t{Lon}\t{strOldVal}->\t{mag} l.info(str_log) else: str_log = str(cfg['out']['log'].get('rows', '0')) # Log to logfile if 'log' in cfg['program'].keys(): flog.writelines('\n' + str_log) if b_out_db_is_different: try: if cfg['out']['tables'] is not None: print('') if cfg['out']['b_remove_duplicates']: h5remove_duplicates(cfg['out'], cfg_table_keys=('tables', 'tables_log')) # Create full indexes. Must be done because of using ptprepack in h5move_tables() below l.debug('Create index') for tblName in (cfg['out']['tables'] + cfg['out']['tables_log']): try: cfg['out']['db'].create_table_index(tblName, columns=['index'], kind='full') except Exception as e: l.warning( ': table {}. Index not created - error'.format( tblName), '\n==> '.join( [s for s in e.args if isinstance(s, str)])) except Exception as e: l.exception('The end. There are error ') import traceback, code from sys import exc_info as sys_exc_info tb = sys_exc_info()[2] # type, value, traceback.print_exc() last_frame = lambda tb=tb: last_frame(tb.tb_next ) if tb.tb_next else tb frame = last_frame().tb_frame ns = dict(frame.f_globals) ns.update(frame.f_locals) code.interact(local=ns) finally: cfg['out']['db'].close() if cfg['program']['log']: flog.close() if cfg['out']['db'].is_open: print('Wait store is closing...') sleep(2) failed_storages = h5move_tables(cfg['out']) print('Finishing...' if failed_storages else 'Ok.', end=' ') h5index_sort( cfg['out'], out_storage_name=f"{cfg['out']['db_path'].stem}-resorted.h5", in_storages=failed_storages)
def main(): print('\n' + this_prog_basename(__file__), end=' started. ') import_pattern = """ ImportFileHDF5(u'180905_clockCheck.h5', [ u'/{probe}/coef/G/A', u'/{probe}/coef/G/C', u'/{probe}/coef/H/A', u'/{probe}/coef/H/C', u'/{probe}/table'], linked=True, namemap={{u'/{probe}/coef/G/A': u'Ag_old', u'/{probe}/coef/G/C': u'Cg', u'/{probe}/coef/H/A': u'Ah_old', u'/{probe}/coef/H/C': u'Ch', u'/{probe}/table/Ax': u'countsAx', u'/{probe}/table/Ay': u'countsAy', u'/{probe}/table/Az': u'countsAz', u'/{probe}/table/Mx': u'countsMx', u'/{probe}/table/My': u'countsMy', u'/{probe}/table/Mz': u'countsMz'}}, renames={{u'Temp': u'sT'}}) """.replace('{{', '{{{{').replace('}}', '}}}}') cfg = veuszPropagate.main([ Path(veuszPropagate.__file__).parent.with_name('veuszPropagate.ini'), '--data_yield_prefix', '-', '--path', str(path_cruise.joinpath(r'incl*.txt')), '--pattern_path', str(path_cruise.with_name('180905_1320incl01.vsz')), # '--import_method', 'ImportFileCSV', '--before_next', 'restore_config', # '--add_custom_list', # 'USEtime', # '--add_custom_expressions_list', # """ # "[['2018-09-05T13:19:55', '2018-09-05T13:20:25']]" # """, '--export_pages_int_list', '6,7', '--b_interact', '0', '--b_images_only', 'True' ]) if not cfg: return 1 # veuszPropagate.load_vsz = cfg['load_vsz'] log_times = pd.to_datetime( [ '05.09.2018 13:20', # 05.09.2018 13:20-00 первое качание '07.09.2018 15:48', # 07.09.2018 15:48-00 второе качание '11.09.2018 12:59' # 11.09.2018 12:59-00 третье качание ], dayfirst=True) t_log_list = pd.to_datetime(log_times) dt_to_st = pd.Timedelta(seconds=-5) dt_to_en = pd.Timedelta(seconds=30) custom_expressions_use_time = [ "[['{:%Y-%m-%dT%H:%M:%S}', '{:%Y-%m-%dT%H:%M:%S}']]".format( t_log + dt_to_st, t_log + dt_to_en) for t_log in t_log_list ] # u"[['2018-09-05T13:19:55', '2018-09-05T13:20:25']]", # u"[['2018-09-07T15:47:55', '2018-09-07T15:48:25']]", # u"[['2018-09-11T13:58:55', '2018-09-11T13:59:55']]", cor_savings = veuszPropagate.co_savings(cfg) cor_savings.send(None) for t_log, expr in zip(t_log_list, custom_expressions_use_time): name_add_time = '_{:%y%m%d_%H%M}-{:%H%M}'.format( t_log + dt_to_st, t_log + dt_to_en) print('Processing group {}...'.format(name_add_time)) def f_file_name(file_name): p = Path(file_name) return str(p.with_name(p.stem + name_add_time + p.suffix)) gen_veusz_and_logs = veuszPropagate.load_to_veusz( veuszPropagate.ge_names(cfg, f_file_name), cfg) cor_send_data = veuszPropagate.co_send_data(gen_veusz_and_logs, cfg, cor_savings) cfgin_update = None while True: # for vsz_data, log in cor_send_data.send(cfgin_update): try: vsz_data, log = cor_send_data.send(cfgin_update) except (GeneratorExit, StopIteration, Ex_nothing_done): print('ok>') break # except Exception as e: # print('There are error: ', standard_error_info(e)) # break # continue probe = log['fileName'][:log['fileName'].find('_')] if not cfg['out']['b_images_only']: cfgin_update = { 'add_custom': ['USEtime'], 'add_custom_expressions': [expr], 'eval': [import_pattern.format(probe=probe)] }
def main(new_arg=None): cfg = cfg_from_args(my_argparser(), new_arg) if not cfg or not cfg['program'].get('return'): print('Can not initialise') return cfg elif cfg['program']['return'] == '<cfg_from_args>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) print('\n' + this_prog_basename(__file__), end=' started. ') try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names(**cfg['in'], b_interact=cfg['program']['b_interact'], cfg_search_parent=cfg['out']) h5init(cfg['in'], cfg['out']) except Ex_nothing_done as e: print(e.message) exit() df_dummy = pd.DataFrame( np.full(1, np.NaN, dtype=np.dtype({ 'formats': ['float64', 'float64'], 'names': cfg['out']['tracks_cols'][1:] })), index=(pd.NaT, )) # used for insert separator lines if 'routes_cols' not in cfg['in']: cfg['in']['routes_cols'] = cfg['in']['waypoints_cols'] if 'routes_cols' not in cfg['out']: cfg['out']['routes_cols'] = cfg['out'][ 'waypoints_cols'] # cfg['in']['routes_cols'] # # Writing if True: # try: l.warning('processing ' + str(cfg['in']['nfiles']) + ' file' + 's:' if cfg['in']['nfiles'] > 1 else ':') cfg['out']['log'] = {} set_field_if_no(cfg['out'], 'table_prefix', PurePath(cfg['in']['path']).stem) cfg['out']['table_prefix'] = cfg['out']['table_prefix'].replace( '-', '') if len([t for t in cfg['out']['tables'] if len(t)]) > 1: cfg['out']['tables'] = \ [cfg['out']['table_prefix'] + '_' + s for s in cfg['out']['tables']] cfg['out']['tables_log'] = \ [cfg['out']['table_prefix'] + '_' + s for s in cfg['out']['tables_log']] tables = dict(zip(df_names, cfg['out']['tables'])) tables_log = dict(zip(df_names, cfg['out']['tables_log'])) # Can not save path to DB (useless?) so set for this max file name length: set_field_if_no(cfg['out'], 'logfield_fileName_len', 50) cfg['out']['index_level2_cols'] = cfg['in']['routes_cols'][0] # ############################################################### # ## Cumulate all data in cfg['out']['path_temp'] ################## ## Main circle ############################################################ for i1_file, path_gpx in h5_dispenser_and_names_gen( cfg['in'], cfg['out']): l.info('{}. {}: '.format(i1_file, path_gpx.name)) # Loading data dfs = gpxConvert(cfg, path_gpx) print('write', end=': ') sys_stdout.flush() for key, df in dfs.items(): if (not tables.get(key)) or df.empty: continue elif key == 'tracks': # Save last time to can filter next file cfg['in']['time_last'] = df.index[-1] sort_time = False if key in {'waypoints', 'routes'} else None # monkey patching if 'tracker' in tables[key]: # Also {} must be in tables[key]. todo: better key+'_fun_tracker' in cfg['out']? # Trackers processing trackers_numbers = { '0-3106432': '1', '0-2575092': '2', '0-3124620': '3', '0-3125300': '4', '0-3125411': '5', '0-3126104': '6' } tables_pattern = tables[key] tables_log_pattern = tables_log[key] df['comment'] = df['comment'].str.split(" @", n=1, expand=True)[0] # split data and save to multipe tables df_all = df.set_index(['comment', df.index]) for sn, n in trackers_numbers.items( ): # set(df_all.index.get_level_values(0)) try: df = df_all.loc[sn] except KeyError: continue # redefine saving parameters cfg['out']['table'] = tables_pattern.format( trackers_numbers[sn]) cfg['out']['table_log'] = tables_log_pattern.format( trackers_numbers[sn]) call_with_valid_kwargs(df_filter_and_save_to_h5, df**cfg, input=cfg['in'], sort_time=sort_time) else: cfg['out']['table'] = tables[key] cfg['out']['table_log'] = tables_log[key] call_with_valid_kwargs(df_filter_and_save_to_h5, df, **cfg, input=cfg['in'], sort_time=sort_time) # try: # if cfg['out']['b_remove_duplicates']: # for tbls in cfg['out']['tables_have_wrote']: # for tblName in tbls: # cfg['out']['db'][tblName].drop_duplicates(keep='last', inplace= True) # print('Create index', end=', ') # create_table_index calls create_table which docs sais "cannot index Time64Col() or ComplexCol" # so load it, index, then save # level2_index = None # df = cfg['out']['db'][tblName] # last commented # df.set_index([navp_all_index, level2_index]) # df.sort_index() # cfg['out']['db'][tblName].sort_index(inplace=True) # if df is not None: # resave # df_log = cfg['out']['db'][tblName] # cfg['out']['db'].remove(tbls[0]) # cfg['out']['db'][tbls[0]] = df # cfg['out']['db'][tbls[1]] = df_log try: pass except Exception as e: print('The end. There are error ', standard_error_info(e)) # import traceback, code # from sys import exc_info as sys_exc_info # # tb = sys_exc_info()[2] # type, value, # traceback.print_exc() # last_frame = lambda tb=tb: last_frame(tb.tb_next) if tb.tb_next else tb # frame = last_frame().tb_frame # ns = dict(frame.f_globals) # ns.update(frame.f_locals) # code.interact(local=ns) # finally: # cfg['out']['db'].close() # failed_storages= h5move_tables(cfg['out'], cfg['out']['tables_have_wrote']) try: failed_storages = h5move_tables(cfg['out'], tbl_names=cfg['out'].get( 'tables_have_wrote', set())) print('Finishing...' if failed_storages else 'Ok.', end=' ') # Sort if have any processed data that needs it (not the case for the routes and waypoints), else don't because ``ptprepack`` not closes hdf5 source if it not finds data if cfg['in'].get('time_last'): cfg['out']['b_remove_duplicates'] = True h5index_sort( cfg['out'], out_storage_name=f"{cfg['out']['db_path'].stem}-resorted.h5", in_storages=failed_storages, tables=cfg['out'].get('tables_have_wrote', set())) except Ex_nothing_done: print('ok')
def main(new_arg=None, veusze=None, **kwargs): """ Initialise configuration and runs or returns routines cfg: ['program']['log'], 'out' 'in' 'async' globals: load_vsz l :param new_arg: :param veusze: used to reuse veusz embedded object (thus to not leak memory) :return: """ global l, load_vsz cfg = cfg_from_args(my_argparser(), new_arg, **kwargs) if not cfg or not cfg['program'].get('return'): print('Can not initialise') return cfg elif cfg['program']['return'] == '<cfg_from_args>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) cfg['program']['log'] = l.root.handlers[ 0].baseFilename # sinchronize obtained absolute file name print('\n' + this_prog_basename(__file__), 'started', end=' ') __name__ = '__main__' # indicate to other functions that they are called from main if cfg['out'].get('paths'): if not cfg['out']['b_images_only']: raise NotImplementedError( 'Provided out in not "b_images_only" mode!') cfg['out']['nfiles'] = len(cfg['out']['paths']) cfg['out']['path'] = cfg['out']['paths'][0] print( end=f"\n- {cfg['out']['nfiles']} output files to export images...") pass else: if cfg['out']['b_images_only']: print( 'in images only mode. Output pattern: ') # todo Export path: ' else: print('. Output pattern and Data: ') try: # Using cfg['out'] to store pattern information if not Path(cfg['in']['pattern_path']).is_absolute(): cfg['in']['pattern_path'] = Path(cfg['in']['path']).with_name( str(cfg['in']['pattern_path'])) cfg['out']['path'] = cfg['in']['pattern_path'] cfg['out']['paths'], cfg['out']['nfiles'], cfg['out'][ 'path'] = init_file_names(**cfg['out'], b_interact=False) except Ex_nothing_done as e: if not cfg['out']['b_images_only']: l.warning( f'{e.message} - no pattern. Specify it or use "b_images_only" mode!' ) return # or raise FileNotFoundError? if (cfg['out']['b_images_only'] and cfg['out']['paths']): cfg['in']['paths'] = cfg['out']['paths'] # have all we need to export else: try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names(**cfg['in'], b_interact=cfg['program'] ['b_interact']) except Ex_nothing_done as e: print(e.message) return # or raise FileNotFoundError? except TypeError: # expected str, bytes or os.PathLike object, not NoneType # cfg['in']['path'] is None. May be it is not need cfg['in']['paths'] = [cfg['in']['pattern_path'] ] # dummy for compatibility cfg['in']['nfiles'] = 1 cfg['out']['export_dir'] = dir_from_cfg(cfg['out']['path'].parent, cfg['out']['export_dir']) if 'restore_config' in cfg['program']['before_next']: cfg['in_saved'] = cfg['in'].copy() # Next is commented because reloading is Ok: not need to Close() # if cfg['out']['b_images_only'] and not 'Close()' in cfg['program']['before_next']: # cfg['program']['before_next'].append( # 'Close()') # usually we need to load new file for export (not only modify previous file) if cfg['program']['export_timeout_s'] and export_images_timed: cfg['async'] = { 'loop': asyncio.get_event_loop(), 'export_timeout_s': cfg['program']['export_timeout_s'] } else: cfg['async'] = {'loop': None} load_vsz = load_vsz_closure(cfg['program']['veusz_path'], cfg['program']['load_timeout_s'], cfg['program']['b_execute_vsz']) cfg['load_vsz'] = load_vsz cfg['co'] = {} if cfg['in']['table_log'] and cfg['in']['path'].suffix == '.h5' and not ( cfg['out']['b_images_only'] and len(cfg['in']['paths']) > 1): # load data by ranges from table log rows cfg['in']['db_path'] = cfg['in']['path'] in_fulls = h5log_names_gen(cfg['in']) elif cfg['in']['tables']: # tables instead files in_fulls = ge_names_from_hdf5_paths(cfg) else: # switch to use found vsz as source if need only export images (even with database source) in_fulls = ge_names(cfg) cor_savings = co_savings(cfg) cor_savings.send(None) nfiles = 0 try: # if True: path_prev = os_getcwd() os_chdir(cfg['out']['path'].parent) if cfg['program']['return'] == '<corutines_in_cfg>': cfg['co']['savings'] = cor_savings cfg['co']['gen_veusz_and_logs'] = load_to_veusz(in_fulls, cfg) cfg['co']['send_data'] = co_send_data(load_to_veusz, cfg, cor_savings) return cfg # return with link to generator function elif cfg['in'].get('data_yield_prefix'): # Cycle with obtaining Veusz data cfgin_update = None while True: # for vsz_data, log in cor_send_data.send(cfgin_update): try: vsz_data, log = co_send_data.send(cfgin_update) nfiles += 1 except (GeneratorExit, StopIteration, Ex_nothing_done): break if 'f_custom_in_cycle' in cfg['program']: cfgin_update = cfg['program']['f_custom_in_cycle']( vsz_data, log) else: # Cycle without obtaining Veusz data (or implemented by user's cfg['program']['f_custom_in_cycle']) for veusze, log in load_to_veusz(in_fulls, cfg, veusze): file_name_r = Path(log['out_vsz_full']).relative_to( cfg['out']['path'].parent) if cfg['program'].get('f_custom_in_cycle'): cfgin_update = cfg['program']['f_custom_in_cycle'](veusze, log) veusze_commands(veusze, cfgin_update, file_name_r) cor_savings.send((veusze, log)) nfiles += 1 cor_savings.close() if cfg['program']['return'] != '<embedded_object>': veusze = None # to note that it is closed in cor_savings.close() print(f'{nfiles} processed. ok>') pass except Exception as e: l.exception('Not good') return # or raise FileNotFoundError? finally: if cfg['async']['loop']: cfg['async']['loop'].close() os_chdir(path_prev) if veusze and cfg['program']['return'] == '<end>': veusze.Close() veusze.WaitForClose() veusze = None elif cfg['program']['return'] == '<embedded_object>': cfg['veusze'] = veusze return cfg
def main(new_arg=None, veusze=None): """ Note: if vsz data source have 'Ag_old_inv' variable then not invert coef. Else invert to use in vsz which not invert coefs :param new_arg: :return: """ global l p = veuszPropagate.my_argparser() p_groups = { g.title: g for g in p._action_groups if g.title.split(' ')[-1] != 'arguments' } # skips special argparse groups p_groups['in'].add( '--channels_list', help= 'channels needed zero calibration: "magnetometer" or "M" for magnetometer and any else for accelerometer, use "M, A" for both, empty to skip ' ) p_groups['in'].add( '--widget', help= 'path to Veusz widget property which contains coefficients. For example "/fitV(force)/grid1/graph/fit1/values"' ) p_groups['in'].add( '--data_for_coef', default='max_incl_of_fit_t', help= 'Veusz data to use as coef. If used with widget then this data is appended to data from widget' ) p_groups['out'].add('--out.path', help='path to db where write coef') p_groups['out'].add( '--re_tbl_from_vsz_name', help= 'regex to extract hdf5 table name from to Veusz file name (last used "\D*\d*")' # ? why not simly specify table name? ) # todo: "b_update_existed" arg will be used here for exported images. Check whether False works or prevent open vsz cfg = cfg_from_args(p, new_arg) if not Path(cfg['program']['log']).is_absolute(): cfg['program']['log'] = str( Path(__file__).parent.joinpath( cfg['program']['log'])) # l.root.handlers[0].baseFilename if not cfg: return if new_arg == '<return_cfg>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) veuszPropagate.l = l print('\n' + this_prog_basename(__file__), 'started', end=' ') if cfg['out']['b_images_only']: print('in images only mode.') try: print('Output pattern ') # Using cfg['out'] to store pattern information if not Path(cfg['in']['pattern_path']).is_absolute(): cfg['in']['pattern_path'] = str(cfg['in']['path'].parent.joinpath( cfg['in']['pattern_path'])) set_field_if_no(cfg['out'], 'path', cfg['in']['pattern_path']) cfg['out']['paths'], cfg['out']['nfiles'], cfg['out'][ 'path'] = init_file_names(**cfg['out'], b_interact=cfg['program']['b_interact']) except Ex_nothing_done as e: print(e.message, ' - no pattern') return # or raise FileNotFoundError? try: print(end='Data ') cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names( **cfg['in'], b_interact=False) # do not bother user 2nd time except Ex_nothing_done as e: print(e.message) return # or raise FileNotFoundError? if not cfg['out']['export_dir']: cfg['out']['export_dir'] = Path(cfg['out']['path']).parent if cfg['program']['before_next'] and 'restore_config' in cfg['program'][ 'before_next']: cfg['in_saved'] = cfg['in'].copy() # cfg['loop'] = asyncio.get_event_loop() # cfg['export_timeout_s'] = 600 cfg['out']['export_dir'] = dir_from_cfg(cfg['out']['path'].parent, cfg['out']['export_dir']) veuszPropagate.load_vsz = veuszPropagate.load_vsz_closure( cfg['program']['veusz_path'], b_execute_vsz=cfg['program']['b_execute_vsz']) gen_veusz_and_logs = veuszPropagate.load_to_veusz( veuszPropagate.ge_names(cfg), cfg, veusze) names_get = ['Inclination_mean_use1', 'logVext1_m__s' ] # \, 'Inclination_mean_use2', 'logVext2_m__s' names_get_fits = ['fit'] # , 'fit2' vsz_data = {n: [] for n in names_get} for n in names_get_fits: vsz_data[n] = [] # prepare collecting all coef in text also names_get_txt_results = ['fit1result'] # , 'fit2result' txt_results = {n: {} for n in names_get_txt_results} i_file = 0 for veusze, log in gen_veusz_and_logs: if not veusze: continue i_file += 1 print(i_file) if cfg['out']['re_tbl_from_vsz_name']: table = cfg['out']['re_tbl_from_vsz_name'].match( log['out_name']).group() else: table = re.sub( '^[\d_]*', '', log['out_name']) # delete all first digits (date part) for n in names_get: vsz_data[n].append(veusze.GetData(n)[0]) for n in [cfg['in']['data_for_coef']]: vsz_data[n] = list(veusze.GetData(n)[0]) # Save velocity coefficients into //{table}//coef//Vabs{i} where i - fit number enumeretad from 0 for i, name_out in enumerate(names_get_fits): # ['fit1', 'fit2'] coef = veusze.Get( cfg['in']['widget'] ) # veusze.Root['fitV(inclination)']['grid1']['graph'][name_out].values.val if 'a' in coef: coef_list = [ coef[k] for k in ['d', 'c', 'b', 'a'] if k in coef ] else: coef_list = [ coef[k] for k in sorted(coef.keys(), key=digits_first) ] if cfg['in']['data_for_coef']: coef_list += vsz_data[cfg['in']['data_for_coef']] vsz_data[name_out].append(coef_list) h5copy_coef(None, cfg['out']['path'], table, dict_matrices={ f'//coef//Vabs{i}': coef_list, f'//coef//date': np.float64([ np.NaN, np.datetime64(datetime.now()).astype(np.int64) ]) }) # h5savecoef(cfg['out']['path'], path=f'//{table}//coef//Vabs{i}', coef=coef_list) txt_results[names_get_txt_results[i]][table] = str(coef) # Zeroing matrix - calculated in Veusz by rotation on old0pitch old0roll Rcor = veusze.GetData( 'Rcor' )[0] # zeroing angles tuned by "USEcalibr0V_..." in Veusz Custom definitions if len(cfg['in']['channels']): l.info( 'Applying zero calibration matrix of peach = {} and roll = {} degrees' .format(np.rad2deg(veusze.GetData('old0pitch')[0][0]), np.rad2deg(veusze.GetData('old0roll')[0][0]))) with h5py.File(cfg['out']['path'], 'a') as h5: for channel in cfg['in']['channels']: (col_str, coef_str) = channel_cols(channel) # h5savecoef(cfg['out']['path'], path=f'//{table}//coef//Vabs{i}', coef=coef_list), dict_matrices={'//coef//' + coef_str + '//A': coefs[tbl][channel]['A'], '//coef//' + coef_str + '//C': coefs[tbl][channel]['b']}) # Currently used inclinometers have electronics rotated on 180deg. Before we inserted additional # rotation operation in Veusz by inverting A_old. Now we want iclude this information in database coef only. try: # Checking that A_old_inv exist A_old_inv = veusze.GetData('Ag_old_inv') is_old_used = True # Rcor is not account for electronic is rotated. except KeyError: is_old_used = False # Rcor is account for rotated electronic. if is_old_used: # The rotation is done in vsz (A_old in vsz is inverted) so need rotate it back to # use in vsz without such invertion # Rotate on 180 deg (note: this is not inversion) A_old_inv = h5[f'//{table}//coef//{coef_str}//A'][...] A_old = np.dot(A_old_inv, [[1, 0, 0], [0, -1, 0], [0, 0, -1] ]) # adds 180 deg to roll else: A_old = h5[f'//{table}//coef//{coef_str}//A'][...] # A_old now accounts for rotated electronic A = np.dot(Rcor, A_old) h5copy_coef(None, h5, table, dict_matrices={f'//coef//{coef_str}//A': A}) # veusze.Root['fitV(inclination)']['grid1']['graph2'][name_out].function.val print(vsz_data) veuszPropagate.export_images( veusze, cfg['out'], f"_{log['out_name']}", b_skip_if_exists=not cfg['out']['b_update_existed']) # vsz_data = veusz_data(veusze, cfg['in']['data_yield_prefix']) # # caller do some processing of data and gives new cfg: # cfgin_update = yield(vsz_data, log) # to test run veusze.Save('-.vsz') # cfg['in'].update(cfgin_update) # only update of cfg.in.add_custom_expressions is tested # if cfg['in']['add_custom']: # for n, e in zip(cfg['in']['add_custom'], cfg['in']['add_custom_expressions']): # veusze.AddCustom('definition', n, e, mode='replace') # #cor_savings.send((veusze, log)) # # # # # veusze.Save(str(path_vsz_save), mode='hdf5') # veusze.Save(str(path_vsz_save)) saves time with bad resolution print(f'Ok') print(txt_results) for n in names_get: pd.DataFrame.from_dict( dict(zip(list(txt_results['fit1result'].keys()), vsz_data[n]))).to_csv( Path(cfg['out']['path']).with_name( f'average_for_fitting-{n}.txt'), sep='\t', header=txt_results['fit1result'].keys, mode='a') return {**vsz_data, 'veusze': veusze}
def main(new_arg=None, **kwargs): global l if __package__ is None: from sys import path as sys_path from os import path as os_path sys_path.append( os_path.dirname(os_path.dirname(os_path.abspath(__file__)))) from utils2init import prep cfg = cfg_from_args(my_argparser(), new_arg, **kwargs) # Input files default_input_filemask = '*.xml' inD, namesFE, nFiles, outD, outF, outE, bWrite2dir, msgFile = prep( { 'path': cfg['in']['path'], 'out_path': cfg['out']['path'] }, default_input_filemask) l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) l.warning('\n' + this_prog_basename(__file__) + ' started. ') l.warning(msgFile) # set_field_if_no(cfg['out'], 'dt_between_track_segments', 99999) gpx = parse_smses(cfg) try: f = open(cfg['in']['path'].with_suffix('.gpx'), 'w') bMissedCoordTo0 = 'b_missed_coord_to_zeros' in cfg['process'] and cfg[ 'process']['b_missed_coord_to_zeros'] if bMissedCoordTo0: for p in gpx.walk(only_points=True): if p.latitude is None or p.longitude is None: p.latitude = '0' # float('NaN') #0 p.longitude = '0' # float('NaN') #0 # if p_prev==p: # p.delete # p_prev= p # gpx.add_missing_data() #remove_empty() f.write(gpx.to_xml()) print('ok') except Ex_nothing_done as e: print(e.message) except Exception as e: msg_option = f'The end. There are error {standard_error_info(e)}' print(msg_option) try: err_msg = e.msg l.error(' '.join([err_msg, msg_option])) except AttributeError: l.error(msg_option) finally: f.close() try: # if not bWrite2dir: # fp_out.close() # l.handlers[0].flush() logging.shutdown() except: pass
def main(new_arg=None): cfg = cfg_from_args(my_argparser(), new_arg) if not cfg: return if new_arg == '<return_cfg>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) if not cfg['out']['path'].is_absolute(): # set path relative to cfg['in']['db_path'] cfg['out']['path'] = cfg['in']['db_path'].with_name(str(cfg['out']['path'])) l.warning('\n {}({}) is gonna save gpx to ..{} dir. '.format( this_prog_basename(__file__), cfg['in']['db_path'], cfg['out']['path'].parent)) if cfg['out']['select_from_tablelog_ranges'] is None: gpx_symbols = None else: gpx_symbols = init_gpx_symbols_fun(cfg['out']) global gpx_names_funs # Shortcat for cfg['out']['gpx_names_funs'] # Load data ################################################################# qstr_trange_pattern = "index>=Timestamp('{}') & index<=Timestamp('{}')" with pd.HDFStore(cfg['in']['db_path'], mode='r') as store: # Find tables by pattern if '*' in cfg['in']['tables'][0]: # if 'table_prefix' in cfg['in'] pattern_tables = cfg['in']['tables'][0] cfg['in']['tables'] = h5find_tables(store, pattern_tables) len_tables = len(cfg['in']['tables']) msg = 'Found {} tables with pattern {}'.format(len_tables, pattern_tables) if len_tables: l.info(msg) else: raise Ex_nothing_done(msg + '!') gpx_names_funs = [] for itbl in range(len(cfg['in']['tables'])): # same fo each table gpx_names_funs.append(cfg['out']['gpx_names_funs'][0]) else: # fixed number of tables # initialise with defaults if need: gpx_names_funs = cfg['out']['gpx_names_funs'] for itbl in range(len(gpx_names_funs), len(cfg['in']['tables'])): gpx_names_funs.append('i+1') dfs_rnav = [] tbl_names_all_shortened = [] for itbl, tblD in enumerate(cfg['in']['tables']): print(itbl, '. ', tblD, end=': ', sep='') if cfg['in']['tables_log'][0]: tblL = tblD + '/' + cfg['in']['tables_log'][0] try: dfL = store[tblL] except KeyError as e: l.warning(' '.join([s for s in e.args if isinstance(s, str)])) continue else: # only for tables without log (usually no such tables) l.warning('configuration specifies to get data without use of "log..." tables') st_en = store[tblD].index[[0, -1]] if cfg['process']['period_files']: t_intervals_start = pd.date_range( start=st_en[0].normalize(), end=max(st_en[-1], st_en[-1].normalize() + pd_period_to_timedelta( cfg['process']['period_files'])), freq=cfg['process']['period_files'])[1:] # makes last t_interval_start >= all_data[-1] dfL = pd.DataFrame.from_records({'DateEnd': t_intervals_start, 'fileName': tblD}, index=st_en[:1].append(t_intervals_start[:-1])) else: dfL = pd.DataFrame.from_records({'DateEnd': st_en[-1], 'fileName': tblD}, index=st_en[:1]) gpx_names_fun_str = "lambda i, row, t=0: '{}'.format({})".format( cfg['out']['gpx_names_fun_format'], gpx_names_funs[itbl]) gpx_names_fun = eval(compile(gpx_names_fun_str, '', 'eval')) if cfg['out']['select_from_tablelog_ranges'] is None: # Use all data for ranges specified in log rows and saves tracks (not points) for irow, r in enumerate(dfL.itertuples()): # iterrows() qstr = qstr_trange_pattern.format(r.Index, r.DateEnd) print(qstr, end='... ') try: dfD = store.select(cfg['in']['table_nav' ] if cfg['in']['table_nav'] else tblD, qstr, columns=['Lat', 'Lon', 'DepEcho']) except Exception as e: l.exception('Error when query: {}. '.format(qstr)) # '\n==> '.join([s for s in e.args if isinstance(s, str)]))) continue # Keep data with period = 1s only dfD = dfD[~dfD.index.round(pd.Timedelta(seconds=1)).duplicated()] # dfD.drop_duplicates(['Lat', 'Lon', 'index'])' bGood = filterGlobal_minmax(dfD, dfD.index, cfg['filter']) dfD = dfD[bGood] # Add UTC time and table name to output file name # Local time and table name to gpx object name str_time_long = '{:%y%m%d_%H%M}'.format(r.Index) r = r._replace(Index=timzone_view(r.Index, cfg['out']['dt_from_utc_in_comments'])) tblD_safe = file_from_tblname(tblD, cfg['in']['tables_log'][0]) try: gpx_names_fun_result = gpx_names_fun(tblD_safe, r) # '{:%y%m%d}'.format(timeLocal) except TypeError as e: raise TypeError('Can not evalute gpx_names_fun "{}"'.format(gpx_names_fun_str)).with_traceback( e.__traceback__) save_to_gpx( dfD, cfg['out']['path'].with_name(f'{str_time_long}{tblD_safe}'), gpx_obj_namef=gpx_names_fun_result, cfg_proc=cfg['process']) if len(cfg['in']['tables']) > 1: nav2add_cur = dfD if irow == 0 else nav2add_cur.append(dfD) if len(cfg['in']['tables']) > 1: nav2add_cur = dfD.assign(itbl=itbl) else: # Use only 1 data point per log row if cfg['out']['select_from_tablelog_ranges'] != 0: print('selecting from {} row index of log table'.format( cfg['out']['select_from_tablelog_ranges'])) try: dfL.index = dfL.index.tz_convert('UTC') except TypeError as e: print((e.msg if hasattr(e, 'msg') else str(e)) + '!\n- continue presume on UTC log index...') print(end='all log data ') time_points = (dfL.index if cfg['out']['select_from_tablelog_ranges'] == 0 else dfL['DateEnd'] if cfg['out']['select_from_tablelog_ranges'] == -1 else None) if time_points is None: raise (ValueError("cfg['out']['select_from_tablelog_ranges'] must be 0 or -1")) cols_nav = ['Lat', 'Lon', 'DepEcho'] nav2add = h5select(store, cfg['in']['table_nav'], cols_nav, time_points=time_points, dt_check_tolerance=cfg['process']['dt_search_nav_tolerance'], query_range_lims=(time_points[0], dfL['DateEnd'][-1]) )[0] cols_nav = nav2add.columns # not all columns may be loaded # Try get non NaN from dfL if it has needed columns (we used to write there edges' data with _st/_en suffixes) isna = nav2add.isna() dfL_col_suffix = 'st' if cfg['out']['select_from_tablelog_ranges'] == 0 else 'en' for col in cols_nav: col_dat = f'{col}_{dfL_col_suffix}' if isna[col].any() and col_dat in dfL.columns: b_use = isna[col].values & dfL[col_dat].notna().values nav2add.loc[b_use, col] = dfL.loc[b_use, col_dat].values nav2add.index = timzone_view(nav2add.index, dt_from_utc=cfg['out']['dt_from_utc_in_comments']) # tz_local= tzoffset(None, cfg['out']['dt_from_utc_in_comments'].total_seconds()) # if nav2add.index.tz is None: # # think if time zone of tz-naive Timestamp is naive then it is UTC # nav2add.index = nav2add.index.tz_localize('UTC') # nav2add.tz_convert(tz_local, copy= False) # Save to gpx waypoints nav2add_cur = nav2add.assign(itbl=itbl) # if 'gpx_names_funs' in cfg['out'] and \ # len(cfg['out']['gpx_names_funs'])>itbl: # # gpx_names = eval(compile('lambda i: str({})'.format( # cfg['out']['gpx_names_funs'][itbl]), [], 'eval')) # save_to_gpx(nav2add_cur, cfg['out']['path'] / f"stations_{file_from_tblname(tblD, cfg['in']['tables_log'][0])}", gpx_obj_namef=gpx_names_fun, waypoint_symbf=gpx_symbols, cfg_proc=cfg['process'] ) # save_to_csv(nav2add, dfL.index, cfg['out']['path'].with_name(f'nav{tblD}.txt')) if False: # Show table info store.get_storer(tblD).table nodes = sorted(store.root.__members__) # , key=number_key print(nodes) # store.get_node('CTD_Idronaut(Redas)').logFiles # next level nodes # prepare saving of combined gpx if tbl_names_all_shortened: i_new = 0 for c_prev, c_new in zip(tbl_names_all_shortened[-1], tblD): if c_new == c_prev: i_new += 1 else: break tbl_names_all_shortened.append(tblD[i_new:]) else: tbl_names_all_shortened.append(tblD) dfs_rnav.append(nav2add_cur) if len(cfg['in']['tables']) > 1 and cfg['out']['gpx_names_funs_cobined']: print('combined: ', end='') # Save combined data to gpx df_rnav_combined = pd.concat(dfs_rnav) df_rnav_combined.sort_index(inplace=True) # Save to gpx waypoints if 'gpx_names_funs' in cfg['out']['gpx_names_funs_cobined']: gpx_names_funs = [ # row not used, it is here only for compability with tracks eval(compile("lambda i: " + f, '', 'eval')) for f in gpx_names_funs] gpx_names_fun = eval(compile( "lambda i,row,t: '{gpx_names_fun_format}'.format({gpx_names_funs_cobined})".format_map( cfg['out']), '', 'eval')) # gpx_symbols = lambda row: cfg['out']['gpx_symbols'][sym_index_fun(row)] # gpx_names = eval(compile("lambda i,row: '{gpx_names_fun_format}'.format({gpx_names_funs_cobined})".format_map(cfg['out']), '', 'eval')) # gpx_names = lambda i: str(i + 1) save_to_gpx( df_rnav_combined, cfg['out']['path'].with_name( 'all_' + file_from_tblname(','.join(tbl_names_all_shortened), cfg['in']['tables_log'][0])), gpx_obj_namef=gpx_names_fun, waypoint_symbf=gpx_symbols, cfg_proc=cfg['process']) print('Ok')
def main(new_arg=None, **kwargs): """ :param new_arg: list of strings, command line arguments :kwargs: dicts for each section: to overwrite values in them (overwrites even high priority values, other values remains) Note: if new_arg=='<cfg_from_args>' returns cfg but it will be None if argument argv[1:] == '-h' or '-v' passed to this code argv[1] is cfgFile. It was used with cfg files: 'csv2h5_nav_supervisor.ini' 'csv2h5_IdrRedas.ini' 'csv2h5_Idronaut.ini' :return: """ global l cfg = cfg_from_args(my_argparser(), new_arg, **kwargs) if not cfg or not cfg['program'].get('return'): print('Can not initialise') return cfg elif cfg['program']['return'] == '<cfg_from_args>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) print('\n' + this_prog_basename(__file__), end=' started. ') try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names(**cfg['in'], b_interact=cfg['program']['b_interact']) except Ex_nothing_done as e: print(e.message) return () bOld_FF00FF = False # if 'TermGrunt' in sys.argv[1] FF00FF' in str(cfg['in']['path']): # 'TermGrunt.h5' ? args.path.endswith ('bin'): # bOld_FF00FF = True # cfg['in'].update({ # 'header': 'TERM', # 'dt_from_utc': timedelta(hours=-1), # 'fs': 1, 'b_time_fromtimestamp': True, # 'b_time_fromtimestamp_source': False}) # else: # 'Katran.h5' # cfg['in'].update({ # 'delimiter_hex': '000000E6', # 'header': 'P, Temp, Cond', # 'dt_from_utc': timedelta(hours=0), # 'fs': 10, 'b_time_fromtimestamp': False, # 'b_time_fromtimestamp_source': False}) set_field_if_no( cfg['in'], 'dtype', 'uint{:d}'.format(2**(3 + np.searchsorted( 2**np.array([3, 4, 5, 6, 7]) > np.array( 8 * (cfg['in']['data_word_len'] - 1)), 1)))) # Prepare cpecific format loading and writing set_field_if_no(cfg['in'], 'coltime', []) cfg['in'] = init_input_cols(cfg['in']) cfg['out']['names'] = np.array(cfg['in']['dtype'].names)[ \ cfg['in']['cols_loaded_save_b']] cfg['out']['formats'] = [ cfg['in']['dtype'].fields[n][0] for n in cfg['out']['names'] ] cfg['out']['dtype'] = np.dtype({ 'formats': cfg['out']['formats'], 'names': cfg['out']['names'] }) h5init(cfg['in'], cfg['out']) # cfg['Period'] = 1.0 / cfg['in']['fs'] # instead Second can use Milli / Micro / Nano: # cfg['pdPeriod'] = pd.to_timedelta(cfg['Period'], 's') # #pd.datetools.Second(cfg['Period'])\ # if 1 % cfg['in']['fs'] == 0 else\ # pd.datetools.Nano(cfg['Period'] * 1e9) # log table of loaded files. columns: Start time, file name, and its index in array off all loaded data: log_item = cfg['out']['log'] = { } # fields will have: 'fileName': None, 'fileChangeTime': None, 'rows': 0 strLog = '' # from collections import namedtuple # type_log_files = namedtuple('type_log_files', ['label','iStart']) # log.sort(axis=0, order='log_item['Date0']')#sort files by time dfLogOld, cfg['out']['db'], cfg['out'][ 'b_skip_if_up_to_date'] = h5temp_open(**cfg['out']) if 'log' in cfg['program'].keys(): f = open(PurePath(sys_argv[0]).parent / cfg['program']['log'], 'a', encoding='cp1251') f.writelines( datetime.now().strftime('\n\n%d.%m.%Y %H:%M:%S> processed ' + str(cfg['in']['nfiles']) + ' file' + 's:' if cfg['in']['nfiles'] > 1 else ':')) b_remove_duplicates = False # normally no duplicates but will if detect # Config specially for readBinFramed set_field_if_no(cfg['in'], 'b_byte_order_is_big_endian', True) set_field_if_no(cfg['in'], 'b_baklan', False) set_field_if_no(cfg['in'], 'b_time_fromtimestamp_source', False) cfg['out']['fs'] = cfg['in']['fs'] if True: ## Main circle ############################################################ for i1_file, path_in in h5_dispenser_and_names_gen( cfg['in'], cfg['out']): l.info('{}. {}: '.format(i1_file, path_in.name)) # Loading data if bOld_FF00FF: V = readFF00FF(path_in, cfg) iFrame = np.arange(len(V)) else: V, iFrame = readBinFramed(path_in, cfg['in']) if ('b_time_fromtimestamp' in cfg['in'] and cfg['in']['b_time_fromtimestamp']) or \ ('b_time_fromtimestamp_source' in cfg['in'] and cfg['in']['b_time_fromtimestamp_source']): path_in_rec = os_path.join( 'd:\\workData\\_source\\BalticSea\\151021_T1Grunt_Pregol\\_source\\not_corrected', os_path.basename(path_in)[:-3] + 'txt' ) if cfg['in']['b_time_fromtimestamp_source'] else path_in log_item['Date0'] = datetime.fromtimestamp( os_path.getmtime(path_in_rec)) # getctime is bad log_item['Date0'] -= iFrame[-1] * timedelta( seconds=1 / cfg['in']['fs'] ) # use for computer filestamp at end of recording else: log_item['Date0'] = datetime.strptime( path_in.stem, cfg['in']['filename2timestart_format']) log_item['Date0'] += cfg['in']['dt_from_utc'] tim = log_item['Date0'] + iFrame * timedelta( seconds=1 / cfg['in']['fs'] ) # tim = pd.date_range(log_item['Date0'], periods=np.size(V, 0), freq=cfg['pdPeriod']) df = pd.DataFrame( V.view(dtype=cfg['out']['dtype']), # np.uint16 columns=cfg['out']['names'], index=tim) # pd.DataFrame(V, columns=cfg['out']['names'], dtype=cfg['out']['formats'], index=tim) if df.empty: # log['rows']==0 print('No data => skip file') continue df, tim = set_filterGlobal_minmax(df, cfg_filter=cfg['filter'], log=log_item, dict_to_save_last_time=cfg['in']) if log_item['rows_filtered']: print('filtered out {}, remains {}'.format( log_item['rows_filtered'], log_item['rows'])) if not log_item['rows']: l.warning('no data! => skip file') continue elif log_item['rows']: print( '.', end='' ) # , divisions=d.divisions), divisions=pd.date_range(tim[0], tim[-1], freq='1D') else: l.warning('no data! => skip file') continue # Append to Store h5_append(cfg['out'], df.astype('int32'), log_item) if 'txt' in cfg['program'].keys(): # can be saved as text too np.savetxt(cfg['program']['txt'], V, delimiter='\t', newline='\n', header=cfg['in']['header'] + log_item['fileName'], fmt='%d', comments='') try: if b_remove_duplicates: for tblName in (cfg['out']['table'] + cfg['out']['tableLog_names']): cfg['out']['db'][tblName].drop_duplicates( keep='last', inplace=True) # subset='fileName',? if len(strLog): print('Create index', end=', ') for tblName in (cfg['out']['table'] + cfg['out']['tableLog_names']): cfg['out']['db'].create_table_index(tblName, columns=['index'], kind='full') else: print('done nothing') except Exception as e: l.exception('The end. There are error ') import traceback, code from sys import exc_info as sys_exc_info tb = sys_exc_info()[2] # type, value, traceback.print_exc() last_frame = lambda tb=tb: last_frame(tb.tb_next) if tb.tb_next else tb frame = last_frame().tb_frame ns = dict(frame.f_globals) ns.update(frame.f_locals) code.interact(local=ns) # sort index if have any processed data (needed because ``ptprepack`` not closses hdf5 source if it not finds data) if cfg['in'].get('time_last'): failed_storages = h5move_tables(cfg['out']) print('Ok.', end=' ') h5index_sort( cfg['out'], out_storage_name=f"{cfg['out']['db_path'].stem}-resorted.h5", in_storages=failed_storages)
def main(): print('\n' + this_prog_basename(__file__), end=' started. ') import_pattern = """ ImportFileHDF5(u'../181005_ABP44.h5', [u'/181017inclinometers/incl03/coef', u'/181017inclinometers/incl03/table'], linked=True, namemap={u'/181017inclinometers/incl03/coef/G/A': u'Ag_old_inv', u'/181017inclinometers/incl03/coef/G/C': u'Cg', u'/181017inclinometers/incl03/coef/H/A': u'Ah_old_inv', u'/181017inclinometers/incl03/coef/H/C': u'Ch', u'/181017inclinometers/incl03/table/Ax': u'countsAx', u'/181017inclinometers/incl03/table/Ay': u'countsAy', u'/181017inclinometers/incl03/table/Az': u'countsAz', u'/181017inclinometers/incl03/table/Mx': u'countsMx', u'/181017inclinometers/incl03/table/My': u'countsMy', u'/181017inclinometers/incl03/table/Mz': u'countsMz', u'/181017inclinometers/incl03/table/Temp': u'sT'}, renames={u'Vabs0': u'kVabs'}) """.replace('{', '{{{{').replace('}', '}}}}').replace('incl03', '{probe}') cfg = veuszPropagate.main([ Path(veuszPropagate.__file__).parent.with_name('veuszPropagate.ini'), '--data_yield_prefix', '-', '--path', str(path_db ), # use for custom loading from db and some source is required '--tables_list', 'incl{}', '--pattern_path', str(path_db.parent.joinpath( fr'inclinometer\*.vsz')), # {str_date}incl03 '--before_next', 'restore_config', # '--add_custom_list', # 'USEtime', # '--add_custom_expressions_list', # """ # "[['2018-10-03T17:23:00', '2018-10-03T18:25:00']]" # """, '--b_update_existed', 'True', '--export_pages_int_list', '0', # ''4 ', #'' '--b_interact', '0', '--b_images_only', 'True' ]) if not cfg: return 1 cor_savings = veuszPropagate.co_savings(cfg) cor_savings.send(None) # Custom loading from db cfg['in']['db_parent_path'] = f'{str_date}inclinometers' if not 'out' in cfg: cfg['out'] = {} cfg['out']['f_file_name'] = lambda tbl: f'{str_date}{tbl}' def ge_names(cfg, f_file_name=lambda x: x): """ Replasing for veuszPropagate.ge_names() to use tables instead files :param cfg: :return: """ with pd.HDFStore(cfg['in']['path'], mode='r') as store: if len(cfg['in']['tables']) == 1: cfg['in']['tables'] = h5find_tables( store, cfg['in']['tables'][0], parent_name=cfg['in']['db_parent_path']) for tbl in cfg['in']['tables']: # if int(tbl[-2:]) in {5,9,10,11,14,20}: yield f_file_name(tbl) # def f_file_name(file_name): # p = Path(file_name) # return str(p.with_name(p.stem + name_add_time + p.suffix)) gen_veusz_and_logs = veuszPropagate.load_to_veusz( ge_names(cfg, cfg['out']['f_file_name']), cfg) cor_send_data = veuszPropagate.co_send_data(gen_veusz_and_logs, cfg, cor_savings) cfgin_update = None while True: # for vsz_data, log in cor_send_data.send(cfgin_update): try: vsz_data, log = cor_send_data.send(cfgin_update) # will delete cfg['in']['tables'] except (GeneratorExit, StopIteration, Ex_nothing_done): print('ok>') break # i_ = log['fileName'].rfind('incl') probe = log['fileName'].replace( str_date, '' ) # re.sub('^[\d_]*', '', in_file.stem), # use last digits (date part) log['fileName'][:(i_)] if not cfg['out']['b_images_only']: cfgin_update = { 'add_custom': ['USEtime'], # 'add_custom_expressions': [expr], 'eval': [import_pattern.format(probe=probe)] }
def main(): print('\n' + this_prog_basename(__file__), end=' started. ') # try: # cfg['in']= init_file_names(cfg['in']) # except Ex_nothing_done as e: # print(e.message) # return() # gen_names = ge_names(cfg) gen_data = veuszPropagate([os_path.join(os_path.dirname( file_veuszPropagate), 'veuszPropagate.ini'), '--data_yield_prefix', 'Hxyz', '--path', cfg['in']['path'], '--pattern_path', r'd:\workData\BalticSea\171003_ANS36\inclinometr\171015_intercal_on_board\~pattern~.vsz', '--eval_list', """ 'ImportFileCSV(u"{nameRFE}", blanksaredata=True, encoding="ascii", headermode="1st", linked=True, dsprefix=u"counts", rowsignore=2, skipwhitespace=True)', "TagDatasets(u'source', [u'countsAx', u'countsAy', u'countsAz', u'countsBattery', u'countsDay', u'countsHour', u'countsMinute', u'countsMonth', u'countsMx', u'countsMy', u'countsMz', u'countsSecond', u'countsTemp', u'countsYear'])" """, # '--import_method', 'ImportFileCSV', '--add_custom_list', 'Ah_old, Ch', '--add_custom_expressions_list', """ 'float64([[ 1, 0, 0],\ [0, 1, 0],\ [0, 0, 1]])\ ', 'float64([[0, 0, 0]])' """, '--before_next', 'restore_config', '--export_pages_int_list', '1']) cfgin_update = None while True: try: d = gen_data.send(cfgin_update) except (GeneratorExit, StopIteration): print('ok>') break except Exception as e: print('There are error ', standard_error_info(e)) continue Hxyz = d['Hxyz'] # Hxyz = np.column_stack((a['Mx'], a['My'], a['Mz']))[slice(*iUseTime.flat)].T if len(Hxyz) < 3 or not np.prod(Hxyz.shape): # 3 is ok but may be empty print('\nNo data from Veusz!\n') bBad = True else: Ah, b = calibrate(Hxyz) if Ah[0, 0] > 10: print('\nBad calibration!\n') bBad = True else: bBad = False if bBad: print('use 1st coef!') b = np.float64([[46, -166, -599]]) Ah = np.float64([[0.0054, -0.0001, -0.0001], [-0.0001, 0.0069, -0.0001], [-0.0001, -0.0001, 0.0089]]) """ AddCustom('definition', u'Ch', u'float64([[60,-160,-650]])') AddCustom('definition', u'Ah_old', u'float64([[50,0,0],\n[0,65,0],\n[0,0,90]])*1e-4') """ # calibrate_plot(Hxyz, Ah, b) A_str, b_str = coef2str(Ah, b) # b_str= 'float64([{}])'.format(b_str) if not bBad: print('calibration coefficients calculated:', '\nA = \n', A_str, '\nb = \n', b_str) cfgin_update = {'add_custom_expressions': [A_str, b_str]}
def main(config: ConfigType) -> None: """ ---------------------------- Calculates coefficients from data of Pandas HDF5 store*.h5 and saves them back ---------------------------- 1. Obtains command line arguments (for description see my_argparser()) that can be passed from new_arg and ini.file also. 2. Loads device data of calibration in laboratory from hdf5 database (cfg['in']['db_path']) 2. Calibrates configured by cfg['in']['channels'] channels ('accelerometer' and/or 'magnetometer'): soft iron 3. Wrong implementation - not use cfg['in']['timerange_nord']! todo: Rotate compass using cfg['in']['timerange_nord'] :param config: returns cfg if new_arg=='<cfg_from_args>' but it will be None if argument argv[1:] == '-h' or '-v' passed to this code argv[1] is cfgFile. It was used with cfg files: """ global cfg, l cfg = main_init(config, cs_store_name, __file__=None) cfg = main_init_input_file(cfg, cs_store_name) # input data tables may be defined by 'probes_prefix' and 'probes' fields of cfg['in'] if cfg['in']['probes'] or not len(cfg['in']['tables']): if cfg['in']['probes']: cfg['in']['tables'] = [ f"{cfg['in']['probes_prefix']}{probe:0>2}" for probe in cfg['in']['probes'] ] elif cfg['in']['probes_prefix']: cfg['in']['tables'] = [f"{cfg['in']['probes_prefix']}.*"] # else: # default config # cfg['in']['tables'] = ['.*'] #h5init(cfg['in'], cfg['out']) #cfg['out']['dt_from_utc'] = 0 # cfg = cfg_from_args(my_argparser(), new_arg) lf.info("{:s}({:s}) for channels: {} started. ", this_prog_basename(__file__), ', '.join(cfg['in']['tables']), cfg['in']['channels']) fig = None fig_filt = None fig_save_dir_path = cfg['in']['db_path'].parent with pd.HDFStore(cfg['in']['db_path'], mode='r') as store: if len(cfg['in']['tables']) == 1: cfg['in']['tables'] = h5find_tables(store, cfg['in']['tables'][0]) coefs = {} for itbl, tbl in enumerate(cfg['in']['tables'], start=1): probe_number = int(re.findall('\d+', tbl)[0]) lf.info(f'{itbl}. {tbl}: ') if isinstance(cfg['in']['timerange'], Mapping): # individual interval for each table if probe_number in cfg['in']['timerange']: timerange = cfg['in']['timerange'][probe_number] else: timerange = None else: timerange = cfg['in'][ 'timerange'] # same interval for each table a = load_hdf5_data(store, table=tbl, t_intervals=timerange) # iUseTime = np.searchsorted(stime, [np.array(s, 'datetime64[s]') for s in np.array(strTimeUse)]) # Calibrate channels of 'accelerometer' or/and 'magnetometer' coefs[tbl] = {} for channel in cfg['in']['channels']: print(f' channel "{channel}"', end=' ') (col_str, coef_str) = channel_cols(channel) # filtering # col_str == 'A'? if True: b_ok = np.zeros(a.shape[0], bool) for component in ['x', 'y', 'z']: b_ok |= is_works( a[col_str + component], noise=cfg['filter']['no_works_noise'][channel]) lf.info('Filtered not working area: {:2.1f}%', (b_ok.size - b_ok.sum()) * 100 / b_ok.size) # vec3d = np.column_stack( # (a[col_str + 'x'], a[col_str + 'y'], a[col_str + 'z']))[:, b_ok].T # [slice(*iUseTime.flat)] vec3d = a.loc[ b_ok, [col_str + 'x', col_str + 'y', col_str + 'z']].to_numpy(float).T index = a.index[b_ok] vec3d, b_ok, fig_filt = filter_channes( vec3d, index, fig_filt, fig_save_prefix= f"{fig_save_dir_path / tbl}-'{channel}'", blocks=cfg['filter']['blocks'], offsets=cfg['filter']['offsets'], std_smooth_sigma=cfg['filter']['std_smooth_sigma']) A, b = calibrate(vec3d) window_title = f"{tbl} '{channel}' channel ellipse" fig = calibrate_plot(vec3d, A, b, fig, window_title=window_title) fig.savefig(fig_save_dir_path / (window_title + '.png'), dpi=300, bbox_inches="tight") A_str, b_str = coef2str(A, b) lf.info( 'Calibration coefficients calculated: \nA = \n{:s}\nb = \n{:s}', A_str, b_str) coefs[tbl][channel] = {'A': A, 'b': b} # Zeroing Nord direction timerange_nord = cfg['in']['timerange_nord'] if isinstance(timerange_nord, Mapping): timerange_nord = timerange_nord.get(probe_number) if timerange_nord: coefs[tbl]['M']['azimuth_shift_deg'] = zeroing_azimuth( store, tbl, timerange_nord, calc_vel_flat_coef(coefs[tbl]), cfg['in']) else: lf.info('not zeroing North') # Write coefs to each of output tables named same as input for cfg_output in (['in', 'out'] if cfg['out'].get('db_path') else ['in']): lf.info('Writing to {}', cfg[cfg_output]['db_path']) for itbl, tbl in enumerate(cfg['in']['tables'], start=1): # i_search = re.search('\d*$', tbl) # for channel in cfg['in']['channels']: # (col_str, coef_str) = channel_cols(channel) # dict_matrices = {f'//coef//{coef_str}//A': coefs[tbl][channel]['A'], # f'//coef//{coef_str}//C': coefs[tbl][channel]['b'], # } # if channel == 'M': # if coefs[tbl]['M'].get('azimuth_shift_deg'): # dict_matrices[f'//coef//{coef_str}//azimuth_shift_deg'] = coefs[tbl]['M']['azimuth_shift_deg'] # # Coping probe number to coefficient to can manually check when copy manually # if i_search: # try: # dict_matrices['//coef//i'] = int(i_search.group(0)) # except Exception as e: # pass dict_matrices = dict_matrices_for_h5(coefs[tbl], tbl, cfg['in']['channels']) h5copy_coef(None, cfg[cfg_output]['db_path'], tbl, dict_matrices=dict_matrices) print('Ok>', end=' ')
def main(): print('\n' + this_prog_basename(__file__), end=' started. ') # try: # cfg['in']= init_file_names(cfg['in']) # except Ex_nothing_done as e: # print(e.message) # return() # gen_names = ge_names(cfg) cfg = veuszPropagate([ os_path.join(os_path.dirname(file_veuszPropagate), 'veuszPropagate_incl.ini'), # '--path', r'd:\workData\BalticSea\171003_ANS36\inclinometr\171017\171017#??.TXT', # windows '--path', r'/mnt/D/workData/BalticSea/171003Strahov/inclinometr/171017/171017#??.TXT', # in Linux match the case is important '--pattern_path', '171017#01.vsz', '--log', os_path.join(os_path.dirname(file_veuszPropagate), 'logs/viewsPropagate.log'), '--data_yield_prefix', 'Hxyz', '--eval_list', """ "ImportFileCSV(u'{nameRFE}', blanksaredata=True, encoding='ascii', headermode='1st', linked=True, dsprefix='counts', rowsignore=2, skipwhitespace=True)", "TagDatasets(u'source', [u'countsAx', u'countsAy', u'countsAz', u'countsBattery', u'countsDay', u'countsHour', u'countsMinute', u'countsMonth', u'countsMx', u'countsMy', u'countsMz', u'countsSecond', u'countsTemp', u'countsYear'])" """, # '--import_method', 'ImportFileCSV', '--add_custom_list', 'Ag_old', # , Ch', '--add_custom_expressions_list', # """ # 'float64([[1,0,0],[0,1,0],[0,0,1]])/16384.0' # """ None, '--before_next_list', 'restore_config', # 'Close(), ' '--export_pages_int_list', '1,3,4,5', # '--veusz_path', '/usr/lib64/python3.6/site-packages/veusz-2.1.1-py3.6-linux-x86_64.egg/veusz', # '/home/korzh/Python/other_sources/veusz/veusz', '-V', 'DEBUG' ]) # os_path.dirname( # if not cfg: exit(0) file_cal_pattern = cfg['in']['path'].with_name( '171121zeroing/INKL_{:03}_data.txt') iFile = cfg['in']['start_file'] # inclinometers are numbered from 1 cfgin_update = None while True: iFile += 1 try: d, log = cfg['co_send_data'].send(cfgin_update) except (GeneratorExit, StopIteration): print('ok>') break except Exception as e: print('There are error ', standard_error_info(e)) continue i = int(log['fileName'].split('#')[1]) Hxyz = d['Hxyz'] # Hxyz = np.column_stack((a['Mx'], a['My'], a['Mz']))[slice(*iUseTime.flat)].T if len(Hxyz) < 3 or not np.prod( Hxyz.shape): # 3 is ok but may be empty print('\nNo data from Veusz!\n') bBad = True else: file_data = file_cal_pattern.format(i) with open(file_data) as f: Ag_str = f.read() Ag_str = re.sub(r'((?<=\d)([ ]+))|(?=\n)', r',\1', Ag_str) Ag = np.float64(eval(Ag_str)) Ag_str = 'float64({})'.format(Ag_str) if Ag[0, 0] > 10: print('\nBad calibration!\n') bBad = True else: bBad = False if bBad: print('using default coef!') Ag_str = 'float64([[1,0,0],[0,1,0],[0,0,1]])/16384.0' """ AddCustom('definition', u'Ch', u'float64([[60,-160,-650]])') AddCustom('definition', u'Ah_old', u'float64([[50,0,0],\n[0,65,0],\n[0,0,90]])*1e-4') """ # calibrate_plot(Hxyz, Ah, b) # A_str, b_str = coef2str(Ah, b) # b_str= 'float64([{}])'.format(b_str) if not bBad: print( 'calibration coefficient loaded ({}): '.format( os_path.basename(file_data)), 'A = \n', Ag_str) cfgin_update = {'add_custom_expressions': [Ag_str]}