import os
import numpy as np
import pandas as pd
from smp_base.common import get_module_logger
from smp_graphs.common import read_puppy_hk_pickles
from smp_graphs.block import Block2, decInit, decStep
from smp_graphs.block import PrimBlock2
import logging
logger = get_module_logger(modulename='block_ols', loglevel=logging.INFO)
class FileBlock2(Block2):
"""FileBlock2 class
File block: read data from file (data file, logfile, rosbag, ...)
and output blocksize lines each step.
Known filetypes: hdf5, npy, puppylog, wav, ...
"""
defaults = {
'block_group': 'data',
}
@decInit()
def __init__(self, conf={}, paren=None, top=None):
# ad hoc default
# if not conf['params'].has_key('type'): conf['params']['type'] = 'puppy'
- TODO: Merge with *general caching* idea
- TODO: Merge / consolidate with realtime kernel and flexible on demand logging and logfile saving
"""
import numpy as np
import tables as tb
import logging
from logging import INFO as logging_INFO
from logging import CRITICAL as logging_CRITICAL
from smp_base.common import get_module_logger
# loglevel_DEFAULT = logging_CRITICAL
loglevel_DEFAULT = logging.DEBUG
logger = get_module_logger(modulename='logging', loglevel=loglevel_DEFAULT)
# declare global h5 file handle
h5file = 0
loginit = True
lognodes = {}
################################################################################
# raw tables logging
def log_tb_init(config):
"""log_tb: log to tables
log init function: global, called from topblock init
"""
# global handles
import numpy as np
# # call graph foo / temp
# from pycallgraph import PyCallGraph
# from pycallgraph import Config
# from pycallgraph.output import GraphvizOutput
from smp_base.common import get_module_logger
from smp_graphs.block import decInit, decStep, Block2, PrimBlock2, LoopBlock2
from smp_graphs.funcs_models import model
from smp_graphs.graph import nxgraph_node_by_id_recursive
from logging import DEBUG as LOGLEVEL
logger = get_module_logger(modulename='block_models', loglevel=LOGLEVEL - 0)
class CodingBlock2(PrimBlock2):
"""CodingBlock2
Mean-variance-residual coding block, recursive estimate of input's mu and sigma
"""
@decInit()
def __init__(self, conf={}, paren=None, top=None):
PrimBlock2.__init__(self, conf=conf, paren=paren, top=top)
for ink, inv in self.inputs.items():
# print inv
for outk in ["mu", "sig", "std"]:
"""Some plotting utils"""
import matplotlib.pyplot as plt
from matplotlib import rc
from matplotlib import rcParams
from matplotlib.font_manager import FontProperties
import logging
from smp_base.common import get_module_logger
loglevel_debug = logging.DEBUG - 1
logger = get_module_logger(modulename='plot_utils', loglevel=logging.DEBUG)
def set_latex_header():
# plotting parameters
rc('text', usetex=True)
rc('font', serif="Times New Roman")
rc('font', family='sans')
rc('font',
style="normal",
variant="normal",
weight=700,
stretch="normal",
size=10.0)
rcParams[
"text.latex.preamble"] = r"\usepackage{amsmath}\usepackage{amsfonts}\usepackage{amssymb}\usepackage{latexsym}\usepackage{bm}"
def set_fontprops():
fontP = FontProperties()
smp/infth/infth_EH-2D.py, smp/infth/infth_EH-2D_clean.py
[1] https://github.com/jlizier/jidt
"""
import sys, os
from pprint import pformat
import numpy as np
import smp_base.config as config
# from smp_base.measures import meas
from smp_base.measures import meas
import logging
from smp_base.common import get_module_logger
logger = get_module_logger(modulename='measures_infth', loglevel=logging.DEBUG)
try:
from jpype import getDefaultJVMPath, isJVMStarted, startJVM, attachThreadToJVM, isThreadAttachedToJVM
from jpype import JPackage
HAVE_JPYPE = True
except ImportError as e:
print("Couldn't import jpype, %s" % e)
HAVE_JPYPE = False
# sys.exit(1)
def init_jpype(jarloc=None, jvmpath=None):
if not HAVE_JPYPE:
print(
"Cannot initialize jpype because it couldn't be imported. Make sure jpype is installed"
import os, sys
import argparse
import subprocess
import logging
from smp_base.common import get_module_logger
logger = get_module_logger(modulename='smp_data_check', loglevel=logging.DEBUG)
datafiles = {
'sin_sweep_0-6.4Hz_newB.pickle': {
'md5': '16da961befff74e296a3743d0ff76d7f'
}
}
def main(args):
# print "args.data_root", args.data_root
do_data_root = False
do_data_dir = False
do_fetch = False
if os.path.exists(args.data_root):
logger.info('Data root at %s exists, checking for dir \'data\'',
args.data_root)
data_dir = '{0}/{1}'.format(args.data_root, 'data')
if os.path.exists(data_dir):
logger.info('Data directory at %s exists, checking for subdirs',
data_dir)
else:
do_data_dir = True
import copy
from functools import partial
import numpy as np
from pprint import pformat
# import transfer functions: linear, nonlin_1, nonlin_2, ...
from smp_base.funcs import *
# model funcs from smp_graphs, FIXME: move that into smp_base
# from smp_graphs.funcs_models import model
from smp_base.models_funcmapped import model
import logging
from smp_base.common import get_module_logger
logger = get_module_logger(modulename='systems', loglevel=logging.INFO)
# dummy block ref
class bla(object):
pass
class SMPSys(object):
"""SMPSys class
Basic smp system class
- Takes the configuration dict and copies all items to
corresponding class members
- Checks for presence of ROS libraries
- Initializes pubs/subs dictionaries
[1] Alain, Guillaume, and Yoshua Bengio. "Understanding Intermediate Layers Using Linear Classifier Probes." ArXiv:1610.01644 [Cs, Stat], October 5, 2016. http://arxiv.org/abs/1610.01644.
"""
import numpy as np
import sklearn
from sklearn import linear_model
from sklearn import kernel_ridge
from sklearn.model_selection import train_test_split
import logging
from smp_base.common import get_module_logger, compose
from smp_base.measures import measures as measures_available
loglevel_debug = logging.DEBUG - 0
logger = get_module_logger(modulename='measures_probes',
loglevel=logging.DEBUG)
def meas_linear_regression_probe(data, alpha=0.0, *args, **kwargs):
"""meas_linear_regression_probe
Linear regression probe: evaluate random vector with respect to
regression task
"""
assert 'X' in data and 'Y' in data, "Data requires 'X' and 'Y' keys with input and labels resp."
# linear model
lm = linear_model.Ridge(alpha=alpha)
# data
X_train, X_test, y_train, y_test = train_test_split(data["X"],
"""smp_graphs utils_conf
some utils for use in configuration files
"""
from functools import partial
from collections import OrderedDict
import numpy as np
from smp_base.measures import meas
from smp_graphs.block_cls import PointmassBlock2, SimplearmBlock2
import logging
from smp_base.common import get_module_logger
logger = get_module_logger(modulename = 'utils_conf', loglevel = logging.DEBUG)
"""system block
- a robot
"""
def get_systemblock_pm(
dim_s0 = 2, dim_s1 = 2, dt = 0.1, lag = 1, **kwargs):
"""configuration utility function pointmass (pm)
Generate configuration for pointmass system block using
:mod:`smp_sys.systems.PointmassSys` and
:mod:`smp_sys.systems.Pointmass2Sys`
"""
global np, PointmassBlock2, meas
logger.debug(
'get_systemblock_pm: dim_s0 = %d, dt= %f, lag = %d, kwargs = %s' % (
Helper func for parameterized measurement stack for use in
configuration files
"""
from collections import OrderedDict
import numpy as np
# from smp_base.measures import meas_div
from smp_graphs.block import Block2, FuncBlock2
from smp_graphs.block_models import ModelBlock2
from smp_graphs.block_meas import MomentBlock2, MeasBlock2
from smp_graphs.block_meas_infth import MIBlock2, InfoDistBlock2
from smp_graphs.funcs import f_sum, f_envelope, f_rootmeansquare
from smp_base.common import get_module_logger
logger = get_module_logger(modulename = 'utils_conf_meas', loglevel = 'info')
def make_input_matrix(id = 'xcorr', base = 'xcorr', xdim = 1, ydim = 1, with_t = False, scan = (-1, 0)):
"""generate input config dict for a plot block for xcorr and similar inputs
Returns single input items as arrays.
"""
# import numpy as np
# global scanstart, scanstop, scanlen
(scanstart, scanstop) = scan
scanlen = scanstop - scanstart
d = {'d3_%d_%d' % (i, j): {'bus': '%s/%s_%d_%d' % (id, base, i, j)} for j in range(xdim) for i in range(ydim)}
if with_t:
d['t'] = {'val': np.linspace(scanstart, scanstop-1, scanlen)}
# print d
from matplotlib import colors
from smp_graphs.utils import print_dict
from smp_graphs.common import loop_delim, dict_replace_idstr_recursive, conf_strip_variables, compress_loop_id
# colors
colors_ = list(six.iteritems(colors.cnames))
from smp_base.plot import plot_colors
import logging
from smp_base.common import get_module_logger
# loglevel_DEFAULT = logging.CRITICAL
loglevel_DEFAULT = logging.DEBUG
logger = get_module_logger(modulename = 'graph', loglevel = loglevel_DEFAULT)
# colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS)
# colors = dict(mcolors.BASE_COLORS)
def nxgraph_nodes_iter_recursive(G, filt = None, data = False, level = 0, iterfunc = None):
if iterfunc is None:
def iterfunc(node, data, level):
print "node = %s, data = %s, level = %d" % (node, data, level)
return level
r = []
for n in G.nodes(data = data):
if type(n) is tuple:
d = n[1]
- tap_struct: restructure flat prediction
TODO:
- moving over legacy tapping funcs from :file:`funcs_models.py`
- move developmental models into smp_base to be able to reuse them
outside of smp_graphs.
[1] https://arxiv.org/abs/1704.07622
"""
import numpy as np
from smp_base.common import get_module_logger
import logging
logger = get_module_logger(modulename='tapping', loglevel=logging.DEBUG)
def tap_tupoff(tup=(), off=0):
"""block_models.tap_tupoff
Return the input tuple with constant offset added to both elements
"""
assert len(
tup) == 2, "block_models.py.tap_tupoff wants 2-tuple, got %d-tuple" % (
len(tup), )
return (tup[0] + off, tup[1] + off)
def tap(ref, inkey=None, lag=None, off=0, source='inputs', outidx=None):
"""tap stuff
.. todo:: Almost any other related library has this, for example sklearn, consolidate
.. todo:: Make it a class of @staticmethods
"""
from smp_base.impl import smpi
import numpy as np
from scipy.io import wavfile as wavfile
# import mdp
import logging
from smp_base.common import get_module_logger, compose
loglevel_debug = logging.DEBUG
logger = get_module_logger(modulename='datasets', loglevel=logging.INFO)
Oger = smpi('Oger')
def get_mackey_glass(sample_len=1000, n_samples=1):
# check Oger imported by name exists in globals and its value a module
if 'Oger' in globals() and Oger is not None:
logger.debug('get_mackey_glass Oger = {0}'.format(Oger))
return Oger.datasets.mackey_glass(sample_len=sample_len,
n_samples=n_samples)
logger.warn('datasets.get_mackey_glass: Oger not found, returning zeros')
# return np.zeros((n_samples, 1, sample_len))
return [[np.zeros((sample_len, 1))] for _ in list(range(n_samples))]
from smp_graphs.common import md5, get_config_raw, check_datadir, code_compile_and_run
from smp_graphs.graph import nxgraph_plot, recursive_draw, recursive_hierarchical
from smp_graphs.graph import nxgraph_flatten, nxgraph_add_edges, nxgraph_get_node_colors
from smp_graphs.graph import nxgraph_nodes_iter, nxgraph_to_smp_graph
from smp_graphs.graph import nxgraph_load, nxgraph_store
# filter warnings
import warnings
warnings.filterwarnings('ignore', category=pd.io.pytables.PerformanceWarning)
from logging import INFO as logging_INFO
from logging import DEBUG as logging_DEBUG
# import logging
loglevel_DEFAULT = logging_INFO
# loglevel_DEFAULT = logging_DEBUG
logger = get_module_logger(modulename='experiment', loglevel=loglevel_DEFAULT)
# # 'application' code
# logger.debug('debug message')
# logger.info('info message')
# logger.warn('warn message')
# logger.error('error message')
# logger.critical('critical message')
# logging.basicConfig(
# logger = logging.getLogger(__name__) # filename = 'example.log',
# level = logging.DEBUG,
# format = '%(levelname)s:%(message)s',
# )
# logger = logging.getLogger()
import os, sys, pickle, re, ast from collections import OrderedDict # import functools from functools import partial from types import FunctionType import logging import smp_graphs from smp_base.common import set_attr_from_dict from smp_base.common import get_module_logger from smp_graphs.utils import print_dict logger = get_module_logger(modulename='common', loglevel=logging.INFO) ################################################################################ # static config templates conf_header = """from smp_graphs.experiment import make_expr_id from collections import OrderedDict from functools import partial import copy import numpy as np from smp_graphs.block import Block2, ConstBlock2, CountBlock2, DelayBlock2, UniformRandomBlock2 from smp_graphs.block import FuncBlock2, LoopBlock2, SeqLoopBlock2 from smp_graphs.block import dBlock2, IBlock2, DelayBlock2 from smp_graphs.block_ols import FileBlock2
# Oswald Berthold 2017
#
# - oneoverfnoise is a python port of gennoise.c by paul bourke
# - levyflight is homegrown (?)
from __future__ import print_function
import argparse
import numpy as np
import matplotlib.pylab as plt
from sklearn.preprocessing import normalize
import logging
from smp_base.common import get_module_logger
logger = get_module_logger(modulename='gennoise', loglevel=logging.INFO)
N = 8192
TWOPOWER = 13
TWOPI = 6.283185307179586476925287
def next_point(prev):
"choose next destination"
mode = "np"
alpha = 1.5
# angle = random.uniform(0,(2*math.pi))
if mode == "random":
angle = random.normalvariate(0, 1.8)
distance = 2 * random.paretovariate(alpha)
# distance = 2 * random.weibullvariate(1.0, 0.9)
