def filter_local( d: Union[pd.DataFrame, dd.DataFrame, Mapping[str, pd.Series], Mapping[str, dd.Series]], cfg_filter: Mapping[str, Any]) -> Union[pd.DataFrame, dd.DataFrame]: """ Filtering values without changing output size: set to NaN if exceed limits :param d: DataFrame :param cfg_filter: This is a dict with dicts "min" and "max" having fields with: - keys equal to column names to filter or regex strings to selelect columns: "*" or "[" must be present to detect it as regex. - values are min and max limits consequently. :return: filtered d with bad values replaced by NaN """ for limit, f_compare in [('min', lambda x, v: x > v), ('max', lambda x, v: x < v)]: # todo: check if is better to use between(left, right, inclusive=True) if not cfg_filter.get(limit): continue for key, v in cfg_filter[limit].items(): if ('*' in key) or ('[' in key): # get multiple keys by regex keys = [c for c in d.columns if re.fullmatch(key, c)] d[keys] = d[keys].where(f_compare(d[keys], v)) key = ', '.join(keys) # for logging only else: try: d[key] = d[key].where(f_compare(d[key], v)) except KeyError as e: # allow redundant parameters in config l.warning('Can not filter this parameer %s', standard_error_info(e)) l.debug('filtering %s(%s) = %g', limit, key, v) return d
def filter_local_arr( d: Mapping[str, Sequence], cfg_filter: Mapping[str, Any]) -> Mapping[str, np.ndarray]: """ Same as filter_local but for dict of arrays Filtering values without changing output size: set to NaN if exceed limits :param d: dict of arrays :param cfg: must have field 'filter'. This is a dict with dicts "min" and "max" having fields with: - keys equal to column names to filter or regex strings to selelect columns: "*" or "[" must be present to detect it as regex. - values are min and max limits consequently. :return: filtered input where filtered values are np.ndarrays having filtered values replaced by NaN """ for limit, f_compare in [('min', lambda x, v: x < v), ('max', lambda x, v: x > v)]: # todo: check if is better to use between(left, right, inclusive=True) if not cfg_filter.get(limit): continue for key, v in cfg_filter[limit].items(): if ('*' in key) or ('[' in key): # get multiple keys by regex # Filter multiple columns at once keys = [c for c in d.columns if re.fullmatch(key, c)] d[keys][f_compare(d[keys], v)] = np.NaN key = ', '.join(keys) # for logging only else: try: d[key][f_compare(d[key], v)] = np.NaN except KeyError as e: # allow redundant parameters in config l.warning('Can not filter this parameer %s', standard_error_info(e)) l.debug('filtering %s(%s) = %g', limit, key, v) return d
def main_init( cfg, cs_store_name, __file__=None, ): """ - prints parameters - prints message that program (__file__ or cs_store_name) started - converts cfg parameters to types according to its prefixes/suffixes names (see ini2dict()) :param cfg: :param cs_store_name: :param __file__: :return: """ # global lf # if cfg.search_path is not None: # override_path = hydra.utils.to_absolute_path(cfg.search_path) # override_conf = OmegaConf.load(override_path) # cfg = OmegaConf.merge(cfg, override_conf) print("Working directory : {}".format(os.getcwd())) # todo: print only if config changed print(OmegaConf.to_yaml(cfg)) # cfg = cfg_from_args(argparser_files(), **kwargs) if not cfg.program.return_: print('Can not initialise') return cfg elif cfg.program.return_ == '<cfg_from_args>': # to help testing return cfg hydra.verbose = 1 if cfg.program.verbose == 'DEBUG' else 0 # made compatible to my old cfg print('\n' + this_prog_basename(__file__) if __file__ else cs_store_name, end=' started. ') try: cfg_t = ini2dict( cfg) # fields named with type pre/suffixes are converted except MissingMandatoryValue as e: lf.error(standard_error_info(e)) raise Ex_nothing_done() except Exception: lf.exception('startup error') # OmegaConf.update(cfg, "in", cfg.input, merge=False) # error # to allow non primitive types (cfg.out['db']) and special words field names ('in'): # cfg = omegaconf.OmegaConf.to_container(cfg) return cfg_t
def filename2date(inF): """ :param inF: base name of source data file (20171015_090558p1) yyyymmdd_HHMMSS* :param cfg: dict with keys: out: dir, paths - pattern path b_images_only, b_update_existed - see command line arguments, in: import_method, header if import_method == 'ImportFile', add_custom :param g: :return: """ a = np.array(inF, dtype={ 'yyyy': ('a4', 0), 'mm': ('a2', 4), 'dd': ('a2', 6), 'HH': ('a2', 9), 'MM': ('a2', 11), 'SS': ('a2', 13) }) try: date = np.array( a['yyyy'].astype(np.object) + b'-' + a['mm'].astype(np.object) + b'-' + a['dd'].astype(np.object) + b'T' + a['HH'].astype(np.object) + b':' + a['MM'].astype(np.object) + b':' + a['SS'].astype(np.object), '|S19', ndmin=1) # date = b'%(yyyy)b-%(mm)b-%(dd)bT%(HH)02b-%(MM)02b-%(SS)02b' % a except Exception as e: print('Can not convert date: ', standard_error_info(e)) raise e return convertNumpyArrayOfStrings(date, 'datetime64[ns]')
def main(new_arg=None): cfg = cfg_from_args(my_argparser(), new_arg) if not cfg or not cfg['program'].get('return'): print('Can not initialise') return cfg elif cfg['program']['return'] == '<cfg_from_args>': # to help testing return cfg l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose']) print('\n' + this_prog_basename(__file__), end=' started. ') try: cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][ 'path'] = init_file_names(**cfg['in'], b_interact=cfg['program']['b_interact'], cfg_search_parent=cfg['out']) h5init(cfg['in'], cfg['out']) except Ex_nothing_done as e: print(e.message) exit() df_dummy = pd.DataFrame( np.full(1, np.NaN, dtype=np.dtype({ 'formats': ['float64', 'float64'], 'names': cfg['out']['tracks_cols'][1:] })), index=(pd.NaT, )) # used for insert separator lines if 'routes_cols' not in cfg['in']: cfg['in']['routes_cols'] = cfg['in']['waypoints_cols'] if 'routes_cols' not in cfg['out']: cfg['out']['routes_cols'] = cfg['out'][ 'waypoints_cols'] # cfg['in']['routes_cols'] # # Writing if True: # try: l.warning('processing ' + str(cfg['in']['nfiles']) + ' file' + 's:' if cfg['in']['nfiles'] > 1 else ':') cfg['out']['log'] = {} set_field_if_no(cfg['out'], 'table_prefix', PurePath(cfg['in']['path']).stem) cfg['out']['table_prefix'] = cfg['out']['table_prefix'].replace( '-', '') if len([t for t in cfg['out']['tables'] if len(t)]) > 1: cfg['out']['tables'] = \ [cfg['out']['table_prefix'] + '_' + s for s in cfg['out']['tables']] cfg['out']['tables_log'] = \ [cfg['out']['table_prefix'] + '_' + s for s in cfg['out']['tables_log']] tables = dict(zip(df_names, cfg['out']['tables'])) tables_log = dict(zip(df_names, cfg['out']['tables_log'])) # Can not save path to DB (useless?) so set for this max file name length: set_field_if_no(cfg['out'], 'logfield_fileName_len', 50) cfg['out']['index_level2_cols'] = cfg['in']['routes_cols'][0] # ############################################################### # ## Cumulate all data in cfg['out']['path_temp'] ################## ## Main circle ############################################################ for i1_file, path_gpx in h5_dispenser_and_names_gen( cfg['in'], cfg['out']): l.info('{}. {}: '.format(i1_file, path_gpx.name)) # Loading data dfs = gpxConvert(cfg, path_gpx) print('write', end=': ') sys_stdout.flush() for key, df in dfs.items(): if (not tables.get(key)) or df.empty: continue elif key == 'tracks': # Save last time to can filter next file cfg['in']['time_last'] = df.index[-1] sort_time = False if key in {'waypoints', 'routes'} else None # monkey patching if 'tracker' in tables[key]: # Also {} must be in tables[key]. todo: better key+'_fun_tracker' in cfg['out']? # Trackers processing trackers_numbers = { '0-3106432': '1', '0-2575092': '2', '0-3124620': '3', '0-3125300': '4', '0-3125411': '5', '0-3126104': '6' } tables_pattern = tables[key] tables_log_pattern = tables_log[key] df['comment'] = df['comment'].str.split(" @", n=1, expand=True)[0] # split data and save to multipe tables df_all = df.set_index(['comment', df.index]) for sn, n in trackers_numbers.items( ): # set(df_all.index.get_level_values(0)) try: df = df_all.loc[sn] except KeyError: continue # redefine saving parameters cfg['out']['table'] = tables_pattern.format( trackers_numbers[sn]) cfg['out']['table_log'] = tables_log_pattern.format( trackers_numbers[sn]) call_with_valid_kwargs(df_filter_and_save_to_h5, df**cfg, input=cfg['in'], sort_time=sort_time) else: cfg['out']['table'] = tables[key] cfg['out']['table_log'] = tables_log[key] call_with_valid_kwargs(df_filter_and_save_to_h5, df, **cfg, input=cfg['in'], sort_time=sort_time) # try: # if cfg['out']['b_remove_duplicates']: # for tbls in cfg['out']['tables_have_wrote']: # for tblName in tbls: # cfg['out']['db'][tblName].drop_duplicates(keep='last', inplace= True) # print('Create index', end=', ') # create_table_index calls create_table which docs sais "cannot index Time64Col() or ComplexCol" # so load it, index, then save # level2_index = None # df = cfg['out']['db'][tblName] # last commented # df.set_index([navp_all_index, level2_index]) # df.sort_index() # cfg['out']['db'][tblName].sort_index(inplace=True) # if df is not None: # resave # df_log = cfg['out']['db'][tblName] # cfg['out']['db'].remove(tbls[0]) # cfg['out']['db'][tbls[0]] = df # cfg['out']['db'][tbls[1]] = df_log try: pass except Exception as e: print('The end. There are error ', standard_error_info(e)) # import traceback, code # from sys import exc_info as sys_exc_info # # tb = sys_exc_info()[2] # type, value, # traceback.print_exc() # last_frame = lambda tb=tb: last_frame(tb.tb_next) if tb.tb_next else tb # frame = last_frame().tb_frame # ns = dict(frame.f_globals) # ns.update(frame.f_locals) # code.interact(local=ns) # finally: # cfg['out']['db'].close() # failed_storages= h5move_tables(cfg['out'], cfg['out']['tables_have_wrote']) try: failed_storages = h5move_tables(cfg['out'], tbl_names=cfg['out'].get( 'tables_have_wrote', set())) print('Finishing...' if failed_storages else 'Ok.', end=' ') # Sort if have any processed data that needs it (not the case for the routes and waypoints), else don't because ``ptprepack`` not closes hdf5 source if it not finds data if cfg['in'].get('time_last'): cfg['out']['b_remove_duplicates'] = True h5index_sort( cfg['out'], out_storage_name=f"{cfg['out']['db_path'].stem}-resorted.h5", in_storages=failed_storages, tables=cfg['out'].get('tables_have_wrote', set())) except Ex_nothing_done: print('ok')
def h5_append(cfg_out: Dict[str, Any], df: Union[pd.DataFrame, dd.DataFrame], log, log_dt_from_utc=pd.Timedelta(0), tim=None): ''' Append dataframe to Store: df to cfg_out['table'] ``table`` node and append chield table with 1 row metadata including 'index' and 'DateEnd' which is calculated as first and last elements of df.index :param df: pandas or dask datarame to append. If dask then log_dt_from_utc must be None (not assign log metadata here) :param log: dict which will be appended to child tables, cfg_out['tables_log'] :param cfg_out: dict with fields: db: opened hdf5 store in write mode table: name of table to update (or tables: list, then used only 1st element) table_log: name of chield table (or tables_log: list, then used only 1st element) tables: None - to return with done nothing! list of str - to assign cfg_out['table'] = cfg_out['tables'][0] tables_log: list of str - to assign cfg_out['table_log'] = cfg_out['tables_log'][0] b_insert_separator: (optional), freq (optional) data_columns: optional, list of column names to write. chunksize: may be None but then must be chunksize_percent to calcW ake Up: chunksize = len(df) * chunksize_percent / 100 :param log_dt_from_utc: 0 or pd.Timedelta - to correct start and end time: index and DateEnd. Note: if log_dt_from_utc is None then start and end time: 'Date0' and 'DateEnd' fields of log must be filled right already :return: None :updates: log: 'Date0' and 'DateEnd' cfg_out: only if not defined already: cfg_out['table_log'] = cfg_out['tables_log'][0] table_log tables_have_wrote list appended (or created) with tuple `(table, table_log)` ''' df_len = len(df) if tim is None else len( tim) # use computed values if possible for faster dask if df_len: # dask.dataframe.empty is not implemented if cfg_out.get('b_insert_separator'): # Add separatiion row of NaN msg_func = f'{df_len}rows+1dummy' cfg_out.setdefault('fs') df = h5_append_dummy_row(df, cfg_out['fs'], tim) df_len += 1 else: msg_func = f'{df_len}rows' # Save to store # check/set tables names if 'tables' in cfg_out: if cfg_out['tables'] is None: l.info('selected(%s)... ', msg_func) return set_field_if_no(cfg_out, 'table', cfg_out['tables'][0]) l.info('h5_append(%s)... ', msg_func) set_field_if_no(cfg_out, 'nfiles', 1) if 'chunksize' in cfg_out and cfg_out['chunksize'] is None: if ('chunksize_percent' in cfg_out): # based on first file cfg_out['chunksize'] = int( df_len * cfg_out['chunksize_percent'] / 1000) * 10 if cfg_out['chunksize'] < 10000: cfg_out['chunksize'] = 10000 else: cfg_out['chunksize'] = 10000 if df_len <= 10000 and isinstance(df, dd.DataFrame): df = df.compute() # dask not writes "all NaN" rows # , compute=False # cfg_out['db'].append(cfg_out['table'], df, data_columns=True, index=False, # chunksize=cfg_out['chunksize']) table = None try: table = df_data_append_fun(df, cfg_out['table'], cfg_out) except ValueError as e: table = h5append_on_inconsistent_index(cfg_out, cfg_out['table'], df, df_data_append_fun, e, msg_func) except TypeError as e: # (, AttributeError)? if isinstance(df, dd.DataFrame): last_nan_row = df.loc[df.index.compute()[-1]].compute() # df.compute().query("index >= Timestamp('{}')".format(df.index.compute()[-1].tz_convert(None))) ??? works # df.query("index > Timestamp('{}')".format(t_end.tz_convert(None)), meta) #df.query(f"index > {t_end}").compute() if all(last_nan_row.isna()): l.exception( f'{msg_func}: dask not writes separator? Repeating using pandas' ) table = df_data_append_fun( last_nan_row, cfg_out['table'], cfg_out, min_itemsize={ c: 1 for c in (cfg_out['data_columns'] if cfg_out. get('data_columns', True ) is not True else df.columns) }) # sometimes pandas/dask get bug (thinks int is a str?): When I add row of NaNs it tries to find ``min_itemsize`` and obtain NaN (for float too, why?) this lead to error else: l.exception(msg_func) else: l.error('%s: Can not write to store. %s', msg_func, standard_error_info(e)) raise (e) except Exception as e: l.error(f'%s: Can not write to store. %s', msg_func, standard_error_info(e)) raise (e) # run even if df is empty becouse of possible needs to write log only table_log = h5add_log(cfg_out, df, log, tim, log_dt_from_utc) _t = (table, table_log) if 'tables_have_wrote' in cfg_out: cfg_out['tables_have_wrote'].add(_t) else: cfg_out['tables_have_wrote'] = {_t}
def h5append_on_inconsistent_index(cfg_out, tbl_parent, df, df_append_fun, e, msg_func): """ :param cfg_out: :param tbl_parent: :param df: :param df_append_fun: :param e: :param msg_func: :return: """ if tbl_parent is None: tbl_parent = cfg_out['table'] error_info_list = [s for s in e.args if isinstance(s, str)] msg = msg_func + ' Error:'.format( e.__class__) + '\n==> '.join(error_info_list) if not error_info_list: l.error(msg) raise e b_correct_time = False b_correct_str = False b_correct_cols = False str_check = 'invalid info for [index] for [tz]' if error_info_list[0].startswith( str_check) or error_info_list[0] == 'Not consistent index': if error_info_list[0] == 'Not consistent index': msg += 'Not consistent index detected' l.error( msg + 'Not consistent index time zone? Changing index to standard UTC') b_correct_time = True elif error_info_list[0].startswith('Trying to store a string with len'): b_correct_str = True l.error(msg + error_info_list[0]) # ? elif error_info_list[0].startswith( 'cannot match existing table structure'): b_correct_cols = True l.error(f'{msg} => Adding columns...') # raise e #? elif error_info_list[0].startswith('invalid combination of [values_axes] on appending data') or \ error_info_list[0].startswith('invalid combination of [non_index_axes] on appending data'): # old pandas version has word "combinate" insted of "combination"! b_correct_cols = True l.error(f'{msg} => Adding columns/convering type...') else: # Can only append to Tables - need resave? l.error(f'{msg} => Can not handle this error!') raise e # Align types # ----------- # Make index to be UTC df_cor = cfg_out['db'][tbl_parent] b_df_cor_changed = False def align_columns(df, df_ref, columns=None): """ :param df: changing dataframe. Will Updated Implisitly! :param df_ref: reference dataframe :param columns: :return: updated df """ if columns is None: columns = df.columns df = df.reindex(df_ref.columns, axis="columns", copy=False) for col, typ in df_ref[columns].dtypes.items(): fill_value = np.array( 0 if np.issubdtype(typ, np.integer) else np.NaN if np.issubdtype(typ, np.floating) else '', dtype=typ) df[col] = fill_value return df if b_correct_time: # change stored to UTC df_cor.index = pd.DatetimeIndex(df_cor.index.tz_convert(tz=tzUTC)) b_df_cor_changed = True elif b_correct_cols: new_cols = list(set(df.columns).difference(df_cor.columns)) if new_cols: df_cor = align_columns(df_cor, df, columns=new_cols) b_df_cor_changed = True # df_cor = df_cor.reindex(columns=df.columns, copy=False) # add columns to df same as in store new_cols = list(set(df_cor.columns).difference(df.columns)) if new_cols: if isinstance(df, dd.DataFrame): df = df.compute() df = align_columns(df, df_cor, columns=new_cols) for col, dtype in zip(df_cor.columns, df_cor.dtypes): d = df_cor[col] # if isinstance(d[0], pd.datetime): if dtype != df[col].dtype: if b_correct_time: if isinstance(d[0], pd.datetime): df_cor[col] = d.dt.tz_convert(tz=df[col].dt.tz) b_df_cor_changed = True elif b_correct_str: # todo: correct str length pass else: try: dtype_max = np.result_type(df_cor[col].dtype, df[col].dtype) if df[col].dtype != dtype_max: df[col] = df[col].astype(dtype_max) if df_cor[col].dtype != dtype_max: df_cor[col] = df_cor[col].astype(dtype_max) b_df_cor_changed = True except e: l.exception('Col "%s" have not numpy dtype?', col) df_cor[col] = df_cor[col].astype(df[col].dtype) b_df_cor_changed = True # pd.api.types.infer_dtype(df_cor.loc[df_cor.index[0], col], df.loc[df.index[0], col]) if b_df_cor_changed: # Update all cfg_out['db'] store data try: with ReplaceTableKeepingChilds([df_cor, df], tbl_parent, cfg_out, df_append_fun): pass return tbl_parent except Exception as e: l.error('%s Can not write to store. May be data corrupted. %s', msg_func, standard_error_info(e)) raise (e) except HDF5ExtError as e: l.exception(e) raise (e) else: # Append corrected data to cfg_out['db'] store try: return df_append_fun(df, tbl_parent, cfg_out) except Exception as e: l.error('%s Can not write to store. May be data corrupted. %s', msg_func, standard_error_info(e)) raise (e) except HDF5ExtError as e: l.exception(e) raise (e)
Load or set default Ag_old, Cg: scaling coefficients for inclinometer Ah_old, Ch: scaling coefficients for magnitometer """ try: with h5py.File(cfg['out']['db_path_coef' if 'db_path_coef' in cfg['out'] else 'db_path'] , "r") as h5source: tblD = cfg['out']['table_coef' if 'table_coef' in cfg['out'] else 'table'] print(f'loading coefficient from {h5source.file.name}/{tblD}') Ag_old = h5source[tblD + '//coef//G//A'].value Cg = h5source[tblD + '//coef//G//C'].value Ah_old = h5source[tblD + '//coef//H//A'].value Ch = h5source[tblD + '//coef//H//C'].value except Exception as e: print(standard_error_info(e), '- Can not load coef. Using default!\n') Ah_old = np.float64([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) # /500.0 Ag_old = np.float64([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) / 16384.0 Cg = np.float64([[0, 0, 0]]) Ch = np.float64([[0, 0, 0]]) # @-others # @-<<loading>> # @+others # @+node:korzh.20180603123200.1: ** zeroing try: if True: # zeroing start_end = h5q_interval2coord( db_path=cfg['out']['db_path'],
def griddata_by_surfer( ctd, path_stem_pattern: Union[str, Path] = r'%TEMP%\xyz{}', margins: Union[bool, Tuple[float, float], None] = True, xCol: str='Lon', yCol: str='Lat', zCols: Sequence[str]= None, SearchEnable=True, BlankOutsideHull=1, DupMethod=15, ShowReport=False, # DupMethod=15=constants.srfDupAvg **kwargs): """ Grid by Surfer :param ctd: pd.DataFrame :param path_stem_pattern: :param margins: extend grid size on all directions: - True - use 10% if limits (xMin...) passed else use InflateHull value - Tuple[float, float] - use this values to add to edges. Note: limits (xMin...) args must be provided in this case :param zCols: z column indexes in ctd :param xCol: x column index in ctd :param yCol: y column index in ctd :param kwargs: other Surfer.GridData4() arguments :return: """ global Surfer if not Surfer: gencache.EnsureModule('{54C3F9A2-980B-1068-83F9-0000C02A351C}', 0, 1, 4) try: Surfer = Dispatch("Surfer.Application") except pywintypes.com_error as e: print("Open Surfer! ", standard_error_info(e)) try: Surfer = Dispatch("Surfer.Application") except pywintypes.com_error as e: print("Open Surfer! ", standard_error_info(e)) raise try: tmpF = f"{path_stem_pattern.format('~temp')}.csv" except AttributeError: path_stem_pattern = str(path_stem_pattern) tmpF = f"{path_stem_pattern.format('~temp')}.csv" kwargs['xCol'] = ctd.dtype.names.index(xCol) + 1 kwargs['yCol'] = ctd.dtype.names.index(yCol) + 1 if zCols is None: zCols = list(ctd.dtype.names) zCols.remove(xCol) zCols.remove(yCol) izCols = [ctd.dtype.names.index(zCol) + 1 for zCol in zCols] # ctd.columns.get_indexer(zCols) + 1 np.savetxt(tmpF, ctd, header=','.join(ctd.dtype.names), delimiter=',', comments='') if margins: if isinstance(margins, bool): margins = [0, 0] for i, coord in enumerate(('x', 'y')): if (f'{coord}Min' in kwargs) and (f'{coord}Max' in kwargs): margins[i] = (kwargs[f'{coord}Max'] - kwargs[f'{coord}Min']) / 10 elif kwargs.get('InflateHull'): margins[i] = kwargs['InflateHull'] kwargs['xMin'] -= margins[0] kwargs['xMax'] += margins[0] kwargs['yMin'] -= margins[1] kwargs['yMax'] += margins[1] if kwargs.get('SearchRad1') is None: kwargs['SearchRad1'] = margins[1] * 3 if kwargs.get('InflateHull') is None: kwargs['InflateHull'] = margins[1] # const={'srfDupAvg': 15, 'srfGridFmtS7': 3} # gdal_geotransform = (x_min, cfg['out']['x_resolution'], 0, -y_min, 0, -cfg['y_resolution_use']) for i, kwargs['zCol'] in enumerate(izCols): outGrd = path_stem_pattern.format(zCols[i]) + '.grd' try: Surfer.GridData4( Algorithm=constants.srfKriging, DataFile=tmpF, OutGrid=outGrd, SearchEnable=(SearchEnable and ctd.size > 3), BlankOutsideHull=BlankOutsideHull, DupMethod=DupMethod, ShowReport=ShowReport, **kwargs) except pywintypes.com_error as e: print(standard_error_info(e)) if i >= 0: # True but in debug mode you can change to not raise and continue without side effects by set i=-1 raise return margins
def main(): print('\n' + this_prog_basename(__file__), end=' started. ') # try: # cfg['in']= init_file_names(cfg['in']) # except Ex_nothing_done as e: # print(e.message) # return() # gen_names = ge_names(cfg) cfg = veuszPropagate([ os_path.join(os_path.dirname(file_veuszPropagate), 'veuszPropagate_incl.ini'), # '--path', r'd:\workData\BalticSea\171003_ANS36\inclinometr\171017\171017#??.TXT', # windows '--path', r'/mnt/D/workData/BalticSea/171003Strahov/inclinometr/171017/171017#??.TXT', # in Linux match the case is important '--pattern_path', '171017#01.vsz', '--log', os_path.join(os_path.dirname(file_veuszPropagate), 'logs/viewsPropagate.log'), '--data_yield_prefix', 'Hxyz', '--eval_list', """ "ImportFileCSV(u'{nameRFE}', blanksaredata=True, encoding='ascii', headermode='1st', linked=True, dsprefix='counts', rowsignore=2, skipwhitespace=True)", "TagDatasets(u'source', [u'countsAx', u'countsAy', u'countsAz', u'countsBattery', u'countsDay', u'countsHour', u'countsMinute', u'countsMonth', u'countsMx', u'countsMy', u'countsMz', u'countsSecond', u'countsTemp', u'countsYear'])" """, # '--import_method', 'ImportFileCSV', '--add_custom_list', 'Ag_old', # , Ch', '--add_custom_expressions_list', # """ # 'float64([[1,0,0],[0,1,0],[0,0,1]])/16384.0' # """ None, '--before_next_list', 'restore_config', # 'Close(), ' '--export_pages_int_list', '1,3,4,5', # '--veusz_path', '/usr/lib64/python3.6/site-packages/veusz-2.1.1-py3.6-linux-x86_64.egg/veusz', # '/home/korzh/Python/other_sources/veusz/veusz', '-V', 'DEBUG' ]) # os_path.dirname( # if not cfg: exit(0) file_cal_pattern = cfg['in']['path'].with_name( '171121zeroing/INKL_{:03}_data.txt') iFile = cfg['in']['start_file'] # inclinometers are numbered from 1 cfgin_update = None while True: iFile += 1 try: d, log = cfg['co_send_data'].send(cfgin_update) except (GeneratorExit, StopIteration): print('ok>') break except Exception as e: print('There are error ', standard_error_info(e)) continue i = int(log['fileName'].split('#')[1]) Hxyz = d['Hxyz'] # Hxyz = np.column_stack((a['Mx'], a['My'], a['Mz']))[slice(*iUseTime.flat)].T if len(Hxyz) < 3 or not np.prod( Hxyz.shape): # 3 is ok but may be empty print('\nNo data from Veusz!\n') bBad = True else: file_data = file_cal_pattern.format(i) with open(file_data) as f: Ag_str = f.read() Ag_str = re.sub(r'((?<=\d)([ ]+))|(?=\n)', r',\1', Ag_str) Ag = np.float64(eval(Ag_str)) Ag_str = 'float64({})'.format(Ag_str) if Ag[0, 0] > 10: print('\nBad calibration!\n') bBad = True else: bBad = False if bBad: print('using default coef!') Ag_str = 'float64([[1,0,0],[0,1,0],[0,0,1]])/16384.0' """ AddCustom('definition', u'Ch', u'float64([[60,-160,-650]])') AddCustom('definition', u'Ah_old', u'float64([[50,0,0],\n[0,65,0],\n[0,0,90]])*1e-4') """ # calibrate_plot(Hxyz, Ah, b) # A_str, b_str = coef2str(Ah, b) # b_str= 'float64([{}])'.format(b_str) if not bBad: print( 'calibration coefficient loaded ({}): '.format( os_path.basename(file_data)), 'A = \n', Ag_str) cfgin_update = {'add_custom_expressions': [Ag_str]}
def csv_to_h5( read_csv_args, to_hdf_args, dates_formats: Mapping[str, str], correct_fun: Tuple[None, bool, Callable[[pd.DataFrame], None]] = None, processing: Optional[Mapping[Tuple[Tuple[str], Tuple[str]], Callable[[Any], Any]]] = None, out_cols: Optional[Sequence] = None, continue_row=False, vaex_format: Optional[bool]=None ): """ Read csv and write to hdf5 :param read_csv_args: dict, must have keys: filepath_or_buffer, chunksize :param to_hdf_args: path_or_buf: default = read_csv_args['filepath_or_buffer'].with_suffix('vaex.h5' if vaex_format else '.h5') mode: default = 'w' if not continue_row else 'a', key: hdf5 group name in hdf5 file where store data ... :param dates_formats: column: csv column name wich need to be convert from str to DateTime, date_format: date formats :param processing: dict with keys: ((_input cols_), (_output cols_)) and values: function(_input cols_) that will return _output cols_ :param out_cols: default is all excluding columns that in inputs but not in output of custom param:processing :param continue_row: csv row number (excluding header) to start with shifting index. If output file exist and continue_row = True then continue converting starting from row equal to last index in it, useful to continue after program interrupting or csv appending. If not exist then start from row 0 giving it index 0. If continue_row = integer then start from this row, giving starting index = continue_row :param correct_fun: function applied to each chunk returned by read_csv() which is a frame of column data of type str :param vaex_format: bool how to write chunks: - True: to many vaex hdf5 files. They at end will be converted to single vaex hdf5 file - False: appending to single pandas hdf5 table - None: evaluates to True if to_hdf_args['path_or_buf'] has next to last suffix ".vaex" else to False :return: """ if to_hdf_args.get('path_or_buf'): if vaex_format is None: vaex_format = Path(str(to_hdf_args['path_or_buf']).strip()).suffixes[:-1] == ['.vaex'] else: # give default name to output file to_hdf_args['path_or_buf'] = Path(read_csv_args['filepath_or_buffer']).with_suffix( f'{".vaex" if vaex_format else ""}.h5' ) # Deal with vaex/pandas storing difference if vaex_format: open_for_pandas_to_hdf = None tmp_save_pattern, tmp_search_pattern = h5pandas_to_vaex_file_names( path_out_str=str(to_hdf_args['path_or_buf']) ) ichunk = None else: def open_for_pandas_to_hdf(path_or_buf): return pd.HDFStore( to_hdf_args['path_or_buf'], to_hdf_args.get('mode', 'a' if continue_row else 'w') ) # Find csv row to start msg_start = f'Converting in chunks of {read_csv_args["chunksize"]} rows.' if continue_row is True: # isinstance(continue_same_csv, bool) try: if vaex_format: hdf5_list = glob.glob(tmp_search_pattern) if len(hdf5_list): # continue interrupted csv_to_h5() hdf5_list.sort() file_last = hdf5_list[-1] lf.info('Found {:d} temporary files, continue from index found in last file', len(hdf5_list)) "table/columns/index" else: # add next csv data file_last = to_hdf_args['path_or_buf'] with h5py.File(file_last, mode='r') as to_hdf_buf: continue_row = to_hdf_buf['table/columns/index/data'][-1] + 1 else: with pd.HDFStore(to_hdf_args['path_or_buf'], mode='r') as to_hdf_buf: continue_row = to_hdf_buf.select(to_hdf_args['key'], columns=[], start=-1).index[-1] + 1 except (OSError) as e: msg_start += ' No output file.' continue_row = None except KeyError as e: msg_start += ' No data in output file.' continue_row = None else: msg_start += ' Starting from next to last loaded csv row:' elif continue_row: msg_start += ' Starting from specified csv data row:' if continue_row: lf.info('{:s} {:d}...', msg_start, continue_row) read_csv_args['skiprows'] = read_csv_args.get('skiprows', 0) + continue_row else: lf.info('{:s} begining from csv row 0, giving it index 0...', msg_start) dtypes = read_csv_args['dtype'] # Set default output cols if not set if out_cols is None and processing: # we will out all we will have except processing inputs if they are not mentioned in processing outputs cols_in_used = set() cols_out_used = set() for (c_in, c_out) in processing.keys(): cols_in_used.update(c_in) cols_out_used.update(c_out) cols2del = cols_in_used.difference(cols_out_used) out_cols = dtypes.keys() for col in cols2del: del out_cols[col] cols_out_used = set(out_cols if out_cols is not None else dtypes.keys()) # Group cols for conversion by types specified str_cols = [] int_and_nans_cols = [] other_cols = [] for col, typ in dtypes.items(): if out_cols and col not in cols_out_used: continue kind = typ[0] (str_cols if kind == 'S' else int_and_nans_cols if kind == 'I' else other_cols).append(col) str_not_dates = list(set(str_cols).difference(dates_formats.keys())) min_itemsize = {col: int(dtypes[col][1:]) for col in str_not_dates} # Read csv, process, write hdf5 with open(read_csv_args['filepath_or_buffer'], 'r') as read_csv_buf, \ FakeContextIfOpen(open_for_pandas_to_hdf, to_hdf_args['path_or_buf']) as to_hdf_buf: read_csv_args.update({ 'filepath_or_buffer': read_csv_buf, 'memory_map': True, 'dtype': 'string' # switch off read_csv dtypes convertion (because if it fails it is hard to correct: }) # to read same csv place by pandas) to_hdf_args.update({ 'path_or_buf': to_hdf_buf, 'format': 'table', 'data_columns': True, 'append': True, 'min_itemsize': min_itemsize }) # rows_processed = 0 # rows_in_chunk = read_csv_args['chunksize'] for ichunk, chunk in enumerate(pd.read_csv(**read_csv_args)): if continue_row: if chunk.size == 0: ichunk = np.ceil(continue_row / read_csv_args['chunksize']).astype(int) - 1 break # continue_row is > data rows else: chunk.index += continue_row lf.extra['id'] = f'chunk start row {chunk.index[0]:d}' if ichunk % 10 == 0: print(f'{ichunk}', end=' ') else: print('.', end='') if correct_fun: correct_fun(chunk) # Convert to user specified types # 1. dates str to DateTime for col, f in dates_formats.items(): # the convertion of 'bytes' to 'strings' is needed for pd.to_datetime() try: chunk[col] = pd.to_datetime(chunk[col], format=f) except ValueError as e: lf.error( 'Conversion to datetime("{:s}" formatted as "{:s}") {:s} -> ' 'Replacing malformed strings by NaT...', col, f, standard_error_info(e)) chunk[col] = pd.to_datetime(chunk[col], format=f, exact=False, errors='coerce') # 2. str to numeric for other_cols and int_and_nans_cols (which is limited support pandas extension dtypes) # but we use numpy types instead replasing nans by -1 to able write to hdf5 chunk[other_cols] = chunk[other_cols].fillna('NaN') # <NA> to numpy recognized eq meaning string chunk[int_and_nans_cols] = chunk[int_and_nans_cols].fillna('-1') for col in (int_and_nans_cols + other_cols): # for col, typ in zip(nans.columns, chunk[nans.columns].dtypes): typ = dtypes[col] if col in int_and_nans_cols: is_integer = True typ = f'i{typ[1:]}' # typ.numpy_dtype else: is_integer = np.dtype(typ).kind == 'i' try: chunk[col] = chunk[col].astype(typ) continue except (ValueError, OverflowError) as e: # Cleaning. In case of OverflowError we do it here to prevent ValueError while handling of OverflowError below. pattern_match = r'^[\d]$' if is_integer else r'^-?[\d.]$' ibad = ~chunk[col].str.match(pattern_match) rep_val = '-1' if is_integer else 'NaN' # ibad = np.flatnonzero(chunk[col] == re.search(r'(?:")(.*)(?:")', e.args[0]).group(1), 'ascii') lf.error('Conversion {:s}("{:s}") {:s} -> replacing {:d} values not maching pattern "{:s}" with "{' ':s}" and again...', typ, col, standard_error_info(e), ibad.sum(), pattern_match, rep_val) chunk.loc[ibad, col] = rep_val # astype(str).replace(regex=True, to_replace=r'^.*[^\d.].*$', value= try: chunk[col] = chunk[col].astype(typ) except (OverflowError, ValueError) as e: # May be bad value from good symbols: r'^\d*\.\d*\.+\d*$' but instead checking it we do coerce_to_exact_dtype() on ValueError here too lf.error('Conversion {:s}("{:s}") {:s} -> Replacing malformed strings and big numbers' ' by NaN ...', typ, col, standard_error_info(e)) chunk[col] = coerce_to_exact_dtype(chunk[col], dtype=typ) # Limit big strings length and convert StringDtype to str to can save by to_hdf() for col, max_len in min_itemsize.items(): # for col, typ in zip(nans.columns, chunk[nans.columns].dtypes): chunk[col] = chunk[col].str.slice(stop=max_len) # apply(lambda x: x[:max_len]) not handles <NA> chunk[str_not_dates] = chunk[str_not_dates].astype(str) # Apply specified data processing if processing: for (cols_in, c_out), fun in processing.items(): cnv_result = fun(chunk[list(cols_in)]) chunk[list(c_out)] = cnv_result # # Bad rows check # is_different = chunk['wlaWID'].fillna('') != chunk['wlaAPIHartStandard'].fillna('') # if is_different.any(): # i_bad = np.flatnonzero(is_different.values) # lf.debug('have wlaWID != wlaAPIHartStandard in rows {:s}', chunk.index[i_bad]) # # chunk= chunk.drop(chunk.index[i_bad]) # - deleting # pass # Check unique index # if chunk['wlaWID'].duplicated() try: if vaex_format: df = vaex.from_pandas(chunk if out_cols is None else chunk[out_cols]) df.export_hdf5(tmp_save_pattern.format(ichunk)) else: # better to move this command upper and proc. by vaex instead of pandas (chunk if out_cols is None else chunk[out_cols]).to_hdf(**to_hdf_args) #rows_processed += rows_in_chunk # think we red always the same length exept last which length value will not be used except Exception as e: lf.exception('write error') pass try: del lf.extra['id'] except KeyError: lf.info('was no more data rows to read') # If vaex store was specified then we have chunk files that we combine now by export_hdf5(): if vaex_format: h5pandas_to_vaex_combine(tmp_search_pattern, str(to_hdf_args['path_or_buf']), check_files_number=ichunk+1)
def main(): print('\n' + this_prog_basename(__file__), end=' started. ') # try: # cfg['in']= init_file_names(cfg['in']) # except Ex_nothing_done as e: # print(e.message) # return() # gen_names = ge_names(cfg) gen_data = veuszPropagate([os_path.join(os_path.dirname( file_veuszPropagate), 'veuszPropagate.ini'), '--data_yield_prefix', 'Hxyz', '--path', cfg['in']['path'], '--pattern_path', r'd:\workData\BalticSea\171003_ANS36\inclinometr\171015_intercal_on_board\~pattern~.vsz', '--eval_list', """ 'ImportFileCSV(u"{nameRFE}", blanksaredata=True, encoding="ascii", headermode="1st", linked=True, dsprefix=u"counts", rowsignore=2, skipwhitespace=True)', "TagDatasets(u'source', [u'countsAx', u'countsAy', u'countsAz', u'countsBattery', u'countsDay', u'countsHour', u'countsMinute', u'countsMonth', u'countsMx', u'countsMy', u'countsMz', u'countsSecond', u'countsTemp', u'countsYear'])" """, # '--import_method', 'ImportFileCSV', '--add_custom_list', 'Ah_old, Ch', '--add_custom_expressions_list', """ 'float64([[ 1, 0, 0],\ [0, 1, 0],\ [0, 0, 1]])\ ', 'float64([[0, 0, 0]])' """, '--before_next', 'restore_config', '--export_pages_int_list', '1']) cfgin_update = None while True: try: d = gen_data.send(cfgin_update) except (GeneratorExit, StopIteration): print('ok>') break except Exception as e: print('There are error ', standard_error_info(e)) continue Hxyz = d['Hxyz'] # Hxyz = np.column_stack((a['Mx'], a['My'], a['Mz']))[slice(*iUseTime.flat)].T if len(Hxyz) < 3 or not np.prod(Hxyz.shape): # 3 is ok but may be empty print('\nNo data from Veusz!\n') bBad = True else: Ah, b = calibrate(Hxyz) if Ah[0, 0] > 10: print('\nBad calibration!\n') bBad = True else: bBad = False if bBad: print('use 1st coef!') b = np.float64([[46, -166, -599]]) Ah = np.float64([[0.0054, -0.0001, -0.0001], [-0.0001, 0.0069, -0.0001], [-0.0001, -0.0001, 0.0089]]) """ AddCustom('definition', u'Ch', u'float64([[60,-160,-650]])') AddCustom('definition', u'Ah_old', u'float64([[50,0,0],\n[0,65,0],\n[0,0,90]])*1e-4') """ # calibrate_plot(Hxyz, Ah, b) A_str, b_str = coef2str(Ah, b) # b_str= 'float64([{}])'.format(b_str) if not bBad: print('calibration coefficients calculated:', '\nA = \n', A_str, '\nb = \n', b_str) cfgin_update = {'add_custom_expressions': [A_str, b_str]}
return wrap @with_in_config def main(cfg): """ :param new_arg: list of strings, command line arguments Note: if new_arg=='<cfg_from_args>' returns cfg but it will be None if argument argv[1:] == '-h' or '-v' passed to this code argv[1] is cfgFile. It was used with cfg files: 'csv2h5_nav_supervisor.ini' 'csv2h5_IdrRedas.ini' 'csv2h5_Idronaut.ini' :return: """ def version(): """Show the version""" return 'version {0}'.format(VERSION) if __name__ == '__main__': run(main, alt=version) except ModuleNotFoundError as e: print(standard_error_info(e))
def parse_csv(filename, cfg_in): """ Guess csv structure :param filename: :param cfg_in: :param known_structure: list of strings formats in order of columns, from start but may be not all (next is auto treeted) :return: lst_types, offset, headers * quotechar - specifies a one-character string to use as the quoting character. It defaults to '"'. * delimiter - specifies a one-character string to use as the field separator. It defaults to ','. * skipinitialspace - specifies how to interpret whitespace which immediately follows a delimiter. It defaults to False, which means that whitespace immediately following a delimiter is part of the following field. * lineterminator - specifies the character sequence which should terminate rows. * quoting - controls when quotes should be generated by the writer. It can take on any of the following module constants: csv.QUOTE_MINIMAL means only when required, for example, when a field contains either the quotechar or the delimiter csv.QUOTE_ALL means that quotes are always placed around fields. csv.QUOTE_NONNUMERIC means that quotes are always placed around fields which do not parse as integers or floating point numbers. csv.QUOTE_NONE means that quotes are never placed around fields. * escapechar - specifies a one-character string used to escape the delimiter when quoting is set to QUOTE_NONE. * doublequote - controls the handling of quotes inside fields. When True, two consecutive quotes are interpreted as one during read, and when writing, each quote character embedded in the data is written as two quotes Example: parse_csv(filename, ['%H:%M:%S']) """ set_field_if_no(cfg_in, 'types', []) set_field_if_no(cfg_in, 'delimiter') with open(filename, 'rb') as fh: ext = os_path.splitext(filename)[1] # Load a file object: try: # If you are sure that file is csv use CSVTableSet(fh) from magic import MagicException # because any_tableset uses libmagic table_set = any_tableset(fh, mimetype=None, extension=ext, delimiter=cfg_in['delimiter']) except (ImportError, MagicException) as e: print('There are error ', standard_error_info(e), '\n=> Loading file as csv without trying other formats') table_set = CSVTableSet(fh, delimiter=cfg_in['delimiter']) # A table set is a collection of tables: row_set = table_set.tables[0] # A row set is an iterator over the table, but it can only # be run once. To peek, a sample is provided: # guess header names and the offset of the header: offset, headers = headers_guess(row_set.sample) # tolerance=1 row_set.register_processor(headers_processor(headers)) # add one to begin with content, not the header: row_set.register_processor(offset_processor(offset + 1)) # guess column types: lst_types = type_guess(row_set.sample, strict=True) row_sample = next(row_set.sample) # check not detected types def formats2types(formats_str): for f in formats_str: if f: if is_date_format(f): yield (types.DateType(f)) else: yield (TimeType()) else: yield (None) known_types = formats2types(cfg_in['types']) for n, (t, s, kt) in enumerate(zip(lst_types, row_sample, known_types)): if t.result_type == types.StringType.result_type: # not auto detected? -> check known_types if kt.test(s.value): lst_types[n] = kt # t= kt else: # known_types fits element print( "col'" 's#{:d} value "{}" type not match provided type of {}'. format(n, s.value, type(kt))) # kt = types.DateType('mm/dd/yyyy') # kt.test('0'+s.value) # detect? else: pass # not works for time type: # print(jts.headers_and_typed_as_jts(headers, # list(map(jts.celltype_as_string, lst_types))).as_json()) return lst_types, offset, headers
def load_to_veusz(in_fulls, cfg, veusze=None): """ Generate Veusz embedded instances by opening vsz-file(s) and modify it by executing commands specified in cfg :param in_full: full name of source data file to load in veusz pattern (usually csv) :param cfg: dict with keys: out: path, paths - pattern paths b_images_only, b_update_existed - command line arguments - see my_argparser() in: before_next - modify Veusz pattern data by execute Veusz commands if have fields: 'Close()' - reopen same pattern each cycle import_method, header if import_method == 'ImportFile', add_custom, add_custom_expressions - Veusz Castom Definitions eval - any Veusz command :param veusze: Veusz embedded object. If it is None creates new Veusz embedded object loading pattern path :yields (veusze, log) veusze: Veusz embedded object. log: dict, {'out_name': inF, 'out_vsz_full': out_vsz_full} Note 1: Uses global load_vsz function if veusze = None or cfg['out']['b_images_only'] which is defined by call load_vsz_closure() Note 2: If 'restore_config' in cfg['program']['before_next'] then sets cfg['in']= cfg['in_saved'] """ global load_vsz filename_fun = eval(compile(cfg['out']['filename_fun'], '', 'eval')) ifile = 0 for in_full in in_fulls: ifile += 1 in_full = Path(in_full) out_name = filename_fun(in_full.stem) + cfg['out']['add_to_filename'] out_vsz_full = Path( cfg['out']['path']).with_name(out_name).with_suffix('.vsz') # if ifile < cfg['in']['start_file']: # continue # do not update existed vsz/images if not specified 'b_update_existed' if not cfg['out']['b_update_existed']: if cfg['out']['b_images_only']: glob = Path(cfg['out']['export_dir']).glob( f"*{out_name}*.{cfg['out']['export_format']}") if any(glob): continue elif out_vsz_full.is_file(): continue # yield (None, None) if in_full.stem != out_vsz_full.stem: l.info('%d. %s -> %s, ', ifile, in_full.name, out_vsz_full.name) else: l.info('%d. %s, ', ifile, in_full.name) sys_stdout.flush() log = {'out_name': out_name, 'out_vsz_full': out_vsz_full} if veusze: try: b_closed = veusze.IsClosed() except Exception as e: l.error('IsClosed() error', exc_info=True) b_closed = True if b_closed: veusze = None elif 'Close()' in cfg['program'][ 'before_next']: # or in_ext == '.vsz' veusze.Close() veusze = None def do_load_vsz(in_full, veusze, load_vsz): in_ext = in_full.suffix.lower() # load same filePattern (last in list) if data file not "vsz": vsz_load = in_full if in_ext == '.vsz' else cfg['out']['paths'][-1] if cfg['out']['b_images_only']: veusze = load_vsz( vsz_load, veusze)[0] # veusze.Load(in_full.with_suffix('.vsz')) else: if 'restore_config' in cfg['program']['before_next']: cfg['in'] = cfg['in_saved'].copy( ) # useful if need restore add_custom_expressions? if not veusze: veusze = load_vsz(vsz_load)[0] # , veusze=veusze if cfg['program']['verbose'] == 'DEBUG': veusze.SetVerbose() # nothing changes # Relative path from new vsz to data, such as u'txt/160813_0010.txt' try: file_name_r = in_full.relative_to( cfg['out']['path'].parent) except ValueError as e: # l.exception('path not related to pattern') file_name_r = in_full veusze_commands(veusze, cfg['in'], file_name_r) return veusze try: veusze = do_load_vsz(in_full, veusze, load_vsz) except asyncio.TimeoutError as e: l.warning('Recreating window because of %s', standard_error_info(e)) veusze.remote.terminate() veusze.remote = None # veusze.Close() # veusze.exitQt() # not works # import socket # veusze.serv_socket.shutdown(socket.SHUT_RDWR) # veusze.serv_socket.close() # veusze.serv_socket, veusze.from_pipe = -1, -1 # veusze.serv_socket.shutdown(socket.SHUT_RDWR); veusze.serv_socket.close() # veusze.startRemote() # no effect load_vsz = load_vsz_closure( cfg['program']['veusz_path'], cfg['program']['load_timeout_s'], cfg['program'] ['b_execute_vsz']) # this is only unfreezes Veusz veusze = do_load_vsz(in_full, None, load_vsz) yield (veusze, log)
plt.clabel(CS, fmt='%g', fontsize=9, inline=1) # if Dep_filt is not None: # plt.plot(nav_dist, -nav_dep, color='m', alpha=0.5, label='Depth') # bot if bFirst: plt.plot(ctd['Lon'], ctd['Lat'], color='m', alpha=0.5, label='run path') CBI = plt.colorbar(im, shrink=0.8) # CB = plt.colorbar(CS, shrink=0.8, extend='both') # plt.gcf().set_size_inches(9, 3) plt.savefig( path_stem.with_suffix('.png')) # , dpi= 200 # plt.show() plt.close() pass except Exception as e: print('\nCan no draw contour! ', standard_error_info(e)) # gdal_drv_grid.Register() write_grd_this_geotransform(path_stem.with_suffix('.grd'), z) if bFirst: bFirst = False except Exception as e: print('\nError! ', standard_error_info(e)) raise e
str = fdata.readline() fdata = BytesIO(str.encode()) if 'b_raise_on_err' in cfg['in'] and not cfg['in']['b_raise_on_err']: try: a = np.genfromtxt(fdata, dtype=cfg['in']['dtype'], delimiter=cfg['in']['delimiter'], usecols=cfg['in']['cols_load'], converters=cfg['in']['converters'], skip_header=cfg['in']['skiprows'], comments=cfg['in']['comments'], invalid_raise=False) # ,autostrip= True # warnings.warn("Mean of empty slice.", RuntimeWarning) except Exception as e: print(*standard_error_info(e), ' - Bad file. skip!\n') continue else: try: a = np.loadtxt(fdata, dtype=cfg['in']['dtype'], delimiter=cfg['in']['delimiter'], usecols=cfg['in']['cols_load'], converters=cfg['in']['converters'], skiprows=cfg['in']['skiprows']) except Exception as e: print('{}\n Try set [in].b_raise_on_err= False'.format(e)) raise (e) # Process a and get date date in ISO standard format try: date = fun_proc_loaded(a, cfg['in'])