import os import numpy as np import pandas as pd from smp_base.common import get_module_logger from smp_graphs.common import read_puppy_hk_pickles from smp_graphs.block import Block2, decInit, decStep from smp_graphs.block import PrimBlock2 import logging logger = get_module_logger(modulename='block_ols', loglevel=logging.INFO) class FileBlock2(Block2): """FileBlock2 class File block: read data from file (data file, logfile, rosbag, ...) and output blocksize lines each step. Known filetypes: hdf5, npy, puppylog, wav, ... """ defaults = { 'block_group': 'data', } @decInit() def __init__(self, conf={}, paren=None, top=None): # ad hoc default # if not conf['params'].has_key('type'): conf['params']['type'] = 'puppy'
- TODO: Merge with *general caching* idea - TODO: Merge / consolidate with realtime kernel and flexible on demand logging and logfile saving """ import numpy as np import tables as tb import logging from logging import INFO as logging_INFO from logging import CRITICAL as logging_CRITICAL from smp_base.common import get_module_logger # loglevel_DEFAULT = logging_CRITICAL loglevel_DEFAULT = logging.DEBUG logger = get_module_logger(modulename='logging', loglevel=loglevel_DEFAULT) # declare global h5 file handle h5file = 0 loginit = True lognodes = {} ################################################################################ # raw tables logging def log_tb_init(config): """log_tb: log to tables log init function: global, called from topblock init """ # global handles
import numpy as np # # call graph foo / temp # from pycallgraph import PyCallGraph # from pycallgraph import Config # from pycallgraph.output import GraphvizOutput from smp_base.common import get_module_logger from smp_graphs.block import decInit, decStep, Block2, PrimBlock2, LoopBlock2 from smp_graphs.funcs_models import model from smp_graphs.graph import nxgraph_node_by_id_recursive from logging import DEBUG as LOGLEVEL logger = get_module_logger(modulename='block_models', loglevel=LOGLEVEL - 0) class CodingBlock2(PrimBlock2): """CodingBlock2 Mean-variance-residual coding block, recursive estimate of input's mu and sigma """ @decInit() def __init__(self, conf={}, paren=None, top=None): PrimBlock2.__init__(self, conf=conf, paren=paren, top=top) for ink, inv in self.inputs.items(): # print inv for outk in ["mu", "sig", "std"]:
"""Some plotting utils""" import matplotlib.pyplot as plt from matplotlib import rc from matplotlib import rcParams from matplotlib.font_manager import FontProperties import logging from smp_base.common import get_module_logger loglevel_debug = logging.DEBUG - 1 logger = get_module_logger(modulename='plot_utils', loglevel=logging.DEBUG) def set_latex_header(): # plotting parameters rc('text', usetex=True) rc('font', serif="Times New Roman") rc('font', family='sans') rc('font', style="normal", variant="normal", weight=700, stretch="normal", size=10.0) rcParams[ "text.latex.preamble"] = r"\usepackage{amsmath}\usepackage{amsfonts}\usepackage{amssymb}\usepackage{latexsym}\usepackage{bm}" def set_fontprops(): fontP = FontProperties()
smp/infth/infth_EH-2D.py, smp/infth/infth_EH-2D_clean.py [1] https://github.com/jlizier/jidt """ import sys, os from pprint import pformat import numpy as np import smp_base.config as config # from smp_base.measures import meas from smp_base.measures import meas import logging from smp_base.common import get_module_logger logger = get_module_logger(modulename='measures_infth', loglevel=logging.DEBUG) try: from jpype import getDefaultJVMPath, isJVMStarted, startJVM, attachThreadToJVM, isThreadAttachedToJVM from jpype import JPackage HAVE_JPYPE = True except ImportError as e: print("Couldn't import jpype, %s" % e) HAVE_JPYPE = False # sys.exit(1) def init_jpype(jarloc=None, jvmpath=None): if not HAVE_JPYPE: print( "Cannot initialize jpype because it couldn't be imported. Make sure jpype is installed"
import os, sys import argparse import subprocess import logging from smp_base.common import get_module_logger logger = get_module_logger(modulename='smp_data_check', loglevel=logging.DEBUG) datafiles = { 'sin_sweep_0-6.4Hz_newB.pickle': { 'md5': '16da961befff74e296a3743d0ff76d7f' } } def main(args): # print "args.data_root", args.data_root do_data_root = False do_data_dir = False do_fetch = False if os.path.exists(args.data_root): logger.info('Data root at %s exists, checking for dir \'data\'', args.data_root) data_dir = '{0}/{1}'.format(args.data_root, 'data') if os.path.exists(data_dir): logger.info('Data directory at %s exists, checking for subdirs', data_dir) else: do_data_dir = True
import copy from functools import partial import numpy as np from pprint import pformat # import transfer functions: linear, nonlin_1, nonlin_2, ... from smp_base.funcs import * # model funcs from smp_graphs, FIXME: move that into smp_base # from smp_graphs.funcs_models import model from smp_base.models_funcmapped import model import logging from smp_base.common import get_module_logger logger = get_module_logger(modulename='systems', loglevel=logging.INFO) # dummy block ref class bla(object): pass class SMPSys(object): """SMPSys class Basic smp system class - Takes the configuration dict and copies all items to corresponding class members - Checks for presence of ROS libraries - Initializes pubs/subs dictionaries
[1] Alain, Guillaume, and Yoshua Bengio. "Understanding Intermediate Layers Using Linear Classifier Probes." ArXiv:1610.01644 [Cs, Stat], October 5, 2016. http://arxiv.org/abs/1610.01644. """ import numpy as np import sklearn from sklearn import linear_model from sklearn import kernel_ridge from sklearn.model_selection import train_test_split import logging from smp_base.common import get_module_logger, compose from smp_base.measures import measures as measures_available loglevel_debug = logging.DEBUG - 0 logger = get_module_logger(modulename='measures_probes', loglevel=logging.DEBUG) def meas_linear_regression_probe(data, alpha=0.0, *args, **kwargs): """meas_linear_regression_probe Linear regression probe: evaluate random vector with respect to regression task """ assert 'X' in data and 'Y' in data, "Data requires 'X' and 'Y' keys with input and labels resp." # linear model lm = linear_model.Ridge(alpha=alpha) # data X_train, X_test, y_train, y_test = train_test_split(data["X"],
"""smp_graphs utils_conf some utils for use in configuration files """ from functools import partial from collections import OrderedDict import numpy as np from smp_base.measures import meas from smp_graphs.block_cls import PointmassBlock2, SimplearmBlock2 import logging from smp_base.common import get_module_logger logger = get_module_logger(modulename = 'utils_conf', loglevel = logging.DEBUG) """system block - a robot """ def get_systemblock_pm( dim_s0 = 2, dim_s1 = 2, dt = 0.1, lag = 1, **kwargs): """configuration utility function pointmass (pm) Generate configuration for pointmass system block using :mod:`smp_sys.systems.PointmassSys` and :mod:`smp_sys.systems.Pointmass2Sys` """ global np, PointmassBlock2, meas logger.debug( 'get_systemblock_pm: dim_s0 = %d, dt= %f, lag = %d, kwargs = %s' % (
Helper func for parameterized measurement stack for use in configuration files """ from collections import OrderedDict import numpy as np # from smp_base.measures import meas_div from smp_graphs.block import Block2, FuncBlock2 from smp_graphs.block_models import ModelBlock2 from smp_graphs.block_meas import MomentBlock2, MeasBlock2 from smp_graphs.block_meas_infth import MIBlock2, InfoDistBlock2 from smp_graphs.funcs import f_sum, f_envelope, f_rootmeansquare from smp_base.common import get_module_logger logger = get_module_logger(modulename = 'utils_conf_meas', loglevel = 'info') def make_input_matrix(id = 'xcorr', base = 'xcorr', xdim = 1, ydim = 1, with_t = False, scan = (-1, 0)): """generate input config dict for a plot block for xcorr and similar inputs Returns single input items as arrays. """ # import numpy as np # global scanstart, scanstop, scanlen (scanstart, scanstop) = scan scanlen = scanstop - scanstart d = {'d3_%d_%d' % (i, j): {'bus': '%s/%s_%d_%d' % (id, base, i, j)} for j in range(xdim) for i in range(ydim)} if with_t: d['t'] = {'val': np.linspace(scanstart, scanstop-1, scanlen)} # print d
from matplotlib import colors from smp_graphs.utils import print_dict from smp_graphs.common import loop_delim, dict_replace_idstr_recursive, conf_strip_variables, compress_loop_id # colors colors_ = list(six.iteritems(colors.cnames)) from smp_base.plot import plot_colors import logging from smp_base.common import get_module_logger # loglevel_DEFAULT = logging.CRITICAL loglevel_DEFAULT = logging.DEBUG logger = get_module_logger(modulename = 'graph', loglevel = loglevel_DEFAULT) # colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS) # colors = dict(mcolors.BASE_COLORS) def nxgraph_nodes_iter_recursive(G, filt = None, data = False, level = 0, iterfunc = None): if iterfunc is None: def iterfunc(node, data, level): print "node = %s, data = %s, level = %d" % (node, data, level) return level r = [] for n in G.nodes(data = data): if type(n) is tuple: d = n[1]
- tap_struct: restructure flat prediction TODO: - moving over legacy tapping funcs from :file:`funcs_models.py` - move developmental models into smp_base to be able to reuse them outside of smp_graphs. [1] https://arxiv.org/abs/1704.07622 """ import numpy as np from smp_base.common import get_module_logger import logging logger = get_module_logger(modulename='tapping', loglevel=logging.DEBUG) def tap_tupoff(tup=(), off=0): """block_models.tap_tupoff Return the input tuple with constant offset added to both elements """ assert len( tup) == 2, "block_models.py.tap_tupoff wants 2-tuple, got %d-tuple" % ( len(tup), ) return (tup[0] + off, tup[1] + off) def tap(ref, inkey=None, lag=None, off=0, source='inputs', outidx=None): """tap stuff
.. todo:: Almost any other related library has this, for example sklearn, consolidate .. todo:: Make it a class of @staticmethods """ from smp_base.impl import smpi import numpy as np from scipy.io import wavfile as wavfile # import mdp import logging from smp_base.common import get_module_logger, compose loglevel_debug = logging.DEBUG logger = get_module_logger(modulename='datasets', loglevel=logging.INFO) Oger = smpi('Oger') def get_mackey_glass(sample_len=1000, n_samples=1): # check Oger imported by name exists in globals and its value a module if 'Oger' in globals() and Oger is not None: logger.debug('get_mackey_glass Oger = {0}'.format(Oger)) return Oger.datasets.mackey_glass(sample_len=sample_len, n_samples=n_samples) logger.warn('datasets.get_mackey_glass: Oger not found, returning zeros') # return np.zeros((n_samples, 1, sample_len)) return [[np.zeros((sample_len, 1))] for _ in list(range(n_samples))]
from smp_graphs.common import md5, get_config_raw, check_datadir, code_compile_and_run from smp_graphs.graph import nxgraph_plot, recursive_draw, recursive_hierarchical from smp_graphs.graph import nxgraph_flatten, nxgraph_add_edges, nxgraph_get_node_colors from smp_graphs.graph import nxgraph_nodes_iter, nxgraph_to_smp_graph from smp_graphs.graph import nxgraph_load, nxgraph_store # filter warnings import warnings warnings.filterwarnings('ignore', category=pd.io.pytables.PerformanceWarning) from logging import INFO as logging_INFO from logging import DEBUG as logging_DEBUG # import logging loglevel_DEFAULT = logging_INFO # loglevel_DEFAULT = logging_DEBUG logger = get_module_logger(modulename='experiment', loglevel=loglevel_DEFAULT) # # 'application' code # logger.debug('debug message') # logger.info('info message') # logger.warn('warn message') # logger.error('error message') # logger.critical('critical message') # logging.basicConfig( # logger = logging.getLogger(__name__) # filename = 'example.log', # level = logging.DEBUG, # format = '%(levelname)s:%(message)s', # ) # logger = logging.getLogger()
import os, sys, pickle, re, ast from collections import OrderedDict # import functools from functools import partial from types import FunctionType import logging import smp_graphs from smp_base.common import set_attr_from_dict from smp_base.common import get_module_logger from smp_graphs.utils import print_dict logger = get_module_logger(modulename='common', loglevel=logging.INFO) ################################################################################ # static config templates conf_header = """from smp_graphs.experiment import make_expr_id from collections import OrderedDict from functools import partial import copy import numpy as np from smp_graphs.block import Block2, ConstBlock2, CountBlock2, DelayBlock2, UniformRandomBlock2 from smp_graphs.block import FuncBlock2, LoopBlock2, SeqLoopBlock2 from smp_graphs.block import dBlock2, IBlock2, DelayBlock2 from smp_graphs.block_ols import FileBlock2
# Oswald Berthold 2017 # # - oneoverfnoise is a python port of gennoise.c by paul bourke # - levyflight is homegrown (?) from __future__ import print_function import argparse import numpy as np import matplotlib.pylab as plt from sklearn.preprocessing import normalize import logging from smp_base.common import get_module_logger logger = get_module_logger(modulename='gennoise', loglevel=logging.INFO) N = 8192 TWOPOWER = 13 TWOPI = 6.283185307179586476925287 def next_point(prev): "choose next destination" mode = "np" alpha = 1.5 # angle = random.uniform(0,(2*math.pi)) if mode == "random": angle = random.normalvariate(0, 1.8) distance = 2 * random.paretovariate(alpha) # distance = 2 * random.weibullvariate(1.0, 0.9)