def useDill():
# load dill
import dill
# disable special function handling
from types import FunctionType
from IPython.utils.pickleutil import can_map
can_map.pop(FunctionType, None)
# fallback to pickle instead of cPickle, so that dill can take over
import pickle
from IPython.kernel.zmq import serialize
serialize.pickle = pickle
def useDill():
# load dill
import dill
# disable special function handling
from types import FunctionType
from IPython.utils.pickleutil import can_map
can_map.pop(FunctionType, None)
# fallback to pickle instead of cPickle, so that dill can take over
import pickle
from IPython.kernel.zmq import serialize
serialize.pickle = pickle
def use_cloudpickle():
"""use cloudpickle to expand serialization support
This is the same things as IPython's pickleutils.use_dill
but for cloudpickle.
"""
from cloud.serialization import cloudpickle
global pickle
pickle = cloudpickle
try:
from IPython.kernel.zmq import serialize
except ImportError:
pass
else:
serialize.pickle = cloudpickle
# disable special function handling, let cloudpickle take care of it
can_map.pop(FunctionType, None)
def train_hmm_n_times(file_id, nstates, trials=20, iter=1000, pickle=True,
phase=2, cond=None, units=constants.XY, parallel=True):
"""
Trains multiple HMM's (as many as trials parameter per nstate) and chooses the one with the
lowest BIC, so as to avoid local optima. units parameter can be "xy", "amp_and_freq", or
"amp_and_mel", which specifies the kind of data to fit the HMM to.
"""
def pick_lowest_bic(models):
hmm, d, bic = None, None, 9999999999
for hmm_ in models:
# hmm_ = HMM(hmm__, training_data=hmm__.obs, hmm_type="ghmm")
if hmm_.bic < bic:
bic = hmm_.bic
hmm = hmm_
if hmm is None:
raise Exception("There are no valid models, WTF?!?")
# return None
# Hmm = HMM(hmm, training_data=d, hmm_type="hmmlearn")
# print_n_flush( "New hmm and data (%s)" % d)
# Hmm.from_R(hmm)
return hmm
import GHmmWrapper
reload(GHmmWrapper)
from GHmmWrapper import get_range_of_multiple_traj
# reload(ExperimentalData)
from leaparticulator.data.functions import fromFile
from leaparticulator.data.hmm import reconstruct_hmm
from LeapTheremin import palmToAmpAndFreq, palmToAmpAndMel
responses, test_results, responses_p, test_p, images = fromFile(id_to_log(file_id))
multivariate = False
reverse_cond = cond in ("2r", "1r")
interval = 1
pick_var = 0
if reverse_cond:
interval = -1
pick_var = 1
if cond in ("2", "2r"):
if phase == 1:
multivariate = True
else:
if phase == 2:
multivariate = True
formatData = None
if multivariate:
if units == constants.XY:
formatData = lambda r, phase: [[frame.get_stabilized_position()[:2][::interval] for frame in rr] for rr in
r["127.0.0.1"][str(phase)].values()]
elif units == constants.AMP_AND_FREQ:
# -interval, because amp_and_freq returns y,x and not x,y.
formatData = lambda r, phase: [
[palmToAmpAndFreq(frame.get_stabilized_position())[::-interval] for frame in rr] for rr in
r["127.0.0.1"][str(phase)].values()]
elif units == constants.AMP_AND_MEL:
# -interval, because amp_and_freq returns y,x and not x,y.
formatData = lambda r, phase: [
[palmToAmpAndMel(frame.get_stabilized_position())[::-interval] for frame in rr] for rr in
r["127.0.0.1"][str(phase)].values()]
else:
if units == constants.XY:
formatData = lambda r, phase: [[frame.get_stabilized_position()[pick_var] for frame in rr] for rr in
r["127.0.0.1"][str(phase)].values()]
elif units == constants.AMP_AND_FREQ:
# -interval, because amp_and_freq returns y,x and not x,y.
formatData = lambda r, phase: [
[palmToAmpAndFreq(frame.get_stabilized_position())[::-interval][pick_var] for frame in rr] for rr in
r["127.0.0.1"][str(phase)].values()]
elif units == constants.AMP_AND_MEL:
# -interval, because amp_and_freq returns y,x and not x,y.
formatData = lambda r, phase: [
[palmToAmpAndMel(frame.get_stabilized_position())[::-interval][pick_var] for frame in rr] for rr in
r["127.0.0.1"][str(phase)].values()]
data = formatData(responses, phase) + formatData(responses_p, phase)
print_n_flush("Sample data: %s" % data[0][:3])
# data = [[frame.get_stabilized_position()[:2] for frame in response] for response in data]
# data.append()
lview = client = None
if parallel:
from IPython.parallel import Client
client = Client(profile="default")
from types import FunctionType
from IPython.utils.pickleutil import can_map
can_map.pop(FunctionType, None)
import pickle
from IPython.kernel.zmq import serialize
serialize.pickle = pickle
client[:].use_dill()
reg = "import copy_reg, ExperimentalData;copy_reg.constructor(ExperimentalData.reconstruct_hmm);copy_reg.pickle(ExperimentalData.HMM, ExperimentalData.reduce_hmm, ExperimentalData.reconstruct_hmm)"
# print type(data), type(data[0])
client[:].execute(reg)
# print data
lview = client.load_balanced_view() # default load-balanced
lview.block = True
to_return = []
range_x, range_y = get_range_of_multiple_traj(data)
for n in nstates:
print_n_flush("Doing %d state models..." % n)
args = [(data, n, range_x, range_y)] * trials
if not parallel:
hmms = map(fn, args) #[(data,nstates,range_x,range_y)] * trials)
else:
hmms = lview.map(fn, args) #[(data,nstates,range_x,range_y)] * trials)
hmms = [reconstruct_hmm(matrix, data) for matrix, data in hmms]
to_return.append(pick_lowest_bic(hmms))
if pickle:
pickle_results(to_return, nstates, trials, iter, id_to_log(file_id), phase, units=units)
return to_return
# todo: rewrite
from IPython import parallel
import util as ut, os, sys, time
#from IPython.parallel.util import interactive
import subprocess, numpy as np
if 0:
import dill
from types import FunctionType
from IPython.utils.pickleutil import can_map
can_map.pop(FunctionType, None)
import pickle
from IPython.kernel.zmq import serialize
serialize.pickle = pickle
ip_parallel = parallel
p = None
part_info = None
builtin_map = __builtins__['map']
def relative_module(m):
return hasattr(m, '__file__') \
and ((not m.__file__.startswith('/')) \
or m.__file__.startswith(os.getcwd()))
def safe_reload(m):
if relative_module(m) and (not hasattr(m, '__no_autoreload__')):
print 'reloading'
reload(m)
imp.find_module("dill")
return True
except ImportError:
return False
# XXX Currently disabled: needs more testing
# if _dill_installed():
if False:
import dill
# disable special function handling
from types import FunctionType
from IPython.utils.pickleutil import can_map
can_map.pop(FunctionType, None)
# fallback to pickle instead of cPickle, so that dill can take over
import pickle
from IPython.kernel.zmq import serialize
serialize.pickle = pickle
DEFAULT_MEM_PER_CPU = 1000 # Mb
# ## Custom launchers
# Handles longer timeouts for startup and shutdown
# with pings between engine and controller.
# Update the ping period to 15 seconds instead of 3.
# Shutdown engines if they cannot be contacted for 3 minutes