def __init__(self,
recurrent=[],
recurrent_dim=[],
skip_list=[],
use_fast_fprop=0,
self_recurrent=1,
init_state_cons=0.,
init_U=InitCell('ortho'),
**kwargs):
super(RecurrentLayer, self).__init__(**kwargs)
self.recurrent = OrderedDict()
if self_recurrent:
self.recurrent[self.name] = self.nout
recurrent_dim = tolist(recurrent_dim)
for i, rec in enumerate(tolist(recurrent)):
if len(recurrent_dim) != 0:
self.recurrent[rec] = recurrent_dim[i]
else:
self.recurrent[rec] = None
self.init_U = init_U
self.init_states = OrderedDict()
self.init_state_cons = init_state_cons
self.use_fast_fprop = use_fast_fprop
self.skip_list = tolist(skip_list)
if len(self.skip_list) > 0:
if len(self.skip_list) != len(self.parent):
raise ValueError("length of parents and skip list should match")
def __init__(self,
recurrent=[],
recurrent_dim=[],
self_recurrent=1,
clip_gradient = True,
clip_bound = 5,
init_U=InitCell('ortho'),
**kwargs):
super(RecurrentLayer, self).__init__(**kwargs)
self.recurrent = OrderedDict()
if self_recurrent:
self.recurrent[self.name] = self.nout
recurrent_dim = tolist(recurrent_dim)
for i, rec in enumerate(tolist(recurrent)):
if len(recurrent_dim) != 0:
self.recurrent[rec] = recurrent_dim[i]
else:
self.recurrent[rec] = None
self.clip_gradient = clip_gradient
self.clip_bound = clip_bound
self.init_U = init_U
def __init__(self,
parent=[],
parent_dim=[],
nout=None,
init_W=InitCell('randn'),
init_b=InitCell('zeros'),
cons=0.,
name=None,
lr_scaler=None,
**kwargs):
super(StemCell, self).__init__(**kwargs)
if name is None:
name = self.__class__.name__.lower()
self.name = name
self.nout = nout
self.init_W = init_W
self.init_b = init_b
self.cons = cons
self.parent = OrderedDict()
parent_dim = tolist(parent_dim)
for i, par in enumerate(tolist(parent)):
if len(parent_dim) != 0 and len(parent) != 0:
if len(parent) != len(parent_dim):
raise AssertionError("You probably had a mistake providing,\
write number of values. It will end,\
up with a model containing a bug.")
self.parent[par] = parent_dim[i]
else:
self.parent[par] = None
self.params = OrderedDict()
self.lr_scaler = lr_scaler
def __init__(self,
parent=[],
parent_dim=[],
nout=None,
init_W=InitCell('randn'),
init_b=InitCell('zeros'),
cons=0.,
name=None,
lr_scaler=None,
**kwargs):
super(StemCell, self).__init__(**kwargs)
if name is None:
name = self.__class__.name__.lower()
self.name = name
self.nout = nout
self.init_W = init_W
self.init_b = init_b
self.cons = cons
self.parent = OrderedDict()
parent_dim = tolist(parent_dim)
for i, par in enumerate(tolist(parent)):
if len(parent_dim) != 0 and len(parent) != 0:
if len(parent) != len(parent_dim):
raise AssertionError(
"You probably had a mistake providing,\
write number of values. It will end,\
up with a model containing a bug.")
self.parent[par] = parent_dim[i]
else:
self.parent[par] = None
self.params = OrderedDict()
self.lr_scaler = lr_scaler
def __init__(self, is_vector=1, weight_norm=1.9365, keys=['W'], waivers=[]):
"""
.. todo::
WRITEME
"""
self.name = 'ext_regularize_post_grad'
self.weight_norm = weight_norm
self.keys = tolist(keys)
self.waivers = tolist(waivers)
self.is_vector = is_vector
def __init__(self,
is_vector=1,
weight_norm=1.9365,
keys=['W'],
waivers=[]):
"""
.. todo::
WRITEME
"""
self.name = 'ext_regularize_post_grad'
self.weight_norm = weight_norm
self.keys = tolist(keys)
self.waivers = tolist(waivers)
self.is_vector = is_vector
def __init__(self,
name,
data,
model,
optimizer,
cost,
outputs,
debug_print=0,
trainlog=None,
extension=None):
self.name = name
self.data = data
self.model = model
self.optimizer = optimizer
self.inputs = model.inputs
self.params = model.params
self.cost = cost
self.outputs = tolist(outputs)
self.updates = model.updates
self.extension = extension
self.debug_print = debug_print
t0 = time.time()
self.cost_fn = self.build_training_graph()
print "Elapsed compilation time: %f" % (time.time() - t0)
if self.debug_print:
from theano.printing import debugprint
debugprint(self.cost_fn)
if trainlog is None:
self.trainlog = TrainLog()
else:
self.trainlog = trainlog
self.endloop = 0
def __init__(self, lambd=0.0002, param_name=['W']):
"""
.. todo::
WRITEME
"""
self.name = 'ext_regularize_pre_grad'
self.lambd = lambd
self.param_name = tolist(param_name)
def build_recurrent_graph(self, n_steps=None, reverse=False, **kwargs):
self.nonseq_args = kwargs.pop('nonseq_args', None)
self.output_args = kwargs.pop('output_args', None)
self.context_args = kwargs.pop('context_args', None)
self.iterators = kwargs.pop('iterators', None)
self.nonseq_inputs = kwargs.pop('nonseq_inputs', None)
self.nNone = 0
inputs = self.inputs.values()
seqs = []
outputs = []
nonseqs = []
if self.nonseq_inputs is not None:
for i in self.nonseq_inputs:
nonseqs.append(inputs.pop(i))
self.input_args = OrderedDict()
self.recur_args = OrderedDict()
for name, node in self.nodes.items():
if hasattr(node, 'get_init_state'):
self.recur_args[name] = node
state = node.get_init_state()
outputs.append(state)
self.nrecur = len(self.recur_args)
# Substitutes initial hidden state into a context
if self.context_args is not None:
for i, (nname, node) in enumerate(self.output_args.items()):
for aname, arg in self.context_args.items():
if nname == aname:
outputs[i] = arg
if self.iterators is None:
seqs += inputs
elif self.iterators is not None:
seqs += inputs[len(self.iterators):]
outputs += inputs[:len(self.iterators)]
if self.output_args is not None:
self.nNone = len(self.output_args)
outputs = flatten(outputs + [None] * self.nNone)
if self.nonseq_args is not None:
for arg in self.nonseq_args:
nonseqs.append(arg)
self.nseqs = len(seqs)
self.noutputs = len(outputs)
self.nnonseqs = len(nonseqs)
result, updates = theano.scan(
fn=self.scan_fn,
sequences=seqs,
outputs_info=outputs,
non_sequences=nonseqs,
n_steps=n_steps,
go_backwards=reverse)
result = tolist(result)
if self.output_args is None and self.iterators is None:
return result
if len(updates) == 0:
return result[-self.nNone:]
for k, v in updates.iteritems():
k.default_update = v
return result[-self.nNone:], updates
def __init__(self, lambd=0.0002, keys=['W']):
"""
.. todo::
WRITEME
"""
self.name = 'ext_regularize_pre_grad'
self.lambd = lambd
self.keys = tolist(keys)
def __init__(self, lambd=0.0002, keys=['W']):
"""
.. todo::
WRITEME
"""
self.name = 'ext_regularize_pre_grad'
self.lambd = lambd
self.keys = tolist(keys)
def build_recurrent_graph(self, n_steps=None, reverse=False, **kwargs):
self.nonseq_args = kwargs.pop('nonseq_args', None)
self.output_args = kwargs.pop('output_args', None)
self.context_args = kwargs.pop('context_args', None)
self.iterators = kwargs.pop('iterators', None)
self.nonseq_inputs = kwargs.pop('nonseq_inputs', None)
self.nNone = 0
inputs = self.inputs.values()
seqs = []
outputs = []
nonseqs = []
if self.nonseq_inputs is not None:
for i in self.nonseq_inputs:
nonseqs.append(inputs.pop(i))
self.input_args = OrderedDict()
self.recur_args = OrderedDict()
for name, node in self.nodes.items():
if hasattr(node, 'get_init_state'):
self.recur_args[name] = node
state = node.get_init_state()
outputs.append(state)
self.nrecur = len(self.recur_args)
# Substitutes initial hidden state into a context
if self.context_args is not None:
for i, (nname, node) in enumerate(self.output_args.items()):
for aname, arg in self.context_args.items():
if nname == aname:
outputs[i] = arg
if self.iterators is None:
seqs += inputs
elif self.iterators is not None:
seqs += inputs[len(self.iterators):]
outputs += inputs[:len(self.iterators)]
if self.output_args is not None:
self.nNone = len(self.output_args)
outputs = flatten(outputs + [None] * self.nNone)
if self.nonseq_args is not None:
for arg in self.nonseq_args:
nonseqs.append(arg)
self.nseqs = len(seqs)
self.noutputs = len(outputs)
self.nnonseqs = len(nonseqs)
result, updates = theano.scan(
fn=self.scan_fn,
sequences=seqs,
outputs_info=outputs,
non_sequences=nonseqs,
n_steps=n_steps,
go_backwards=reverse)
result = tolist(result)
if self.output_args is None and self.iterators is None:
return result
if len(updates) == 0:
return result[-self.nNone:]
for k, v in updates.iteritems():
k.default_update = v
return result[-self.nNone:], updates
def __init__(self, lambd=0.0002, param_name=['W']):
"""
.. todo::
WRITEME
"""
self.name = 'ext_regularize_pre_grad'
self.lambd = lambd
self.param_name = tolist(param_name)
def __init__(self, is_vector=1, weight_norm=1.9365, param_name=['W']):
"""
.. todo::
WRITEME
"""
self.name = 'ext_regularize_post_grad'
self.weight_norm = weight_norm
self.param_name = tolist(param_name)
self.is_vector = is_vector
def __init__(self, is_vector=1, weight_norm=1.9365, param_name=['W']):
"""
.. todo::
WRITEME
"""
self.name = 'ext_regularize_post_grad'
self.weight_norm = weight_norm
self.param_name = tolist(param_name)
self.is_vector = is_vector
def set_graph(self):
self.graph = {}
for nname, node in self.nodes.items():
parent = node.parent
for par in tolist(parent.keys()):
if par in self.inputs.keys():
continue
if par in self.graph.keys():
self.graph[par] =\
tolist(self.graph[par]) + [node.name]
else:
self.graph[par] = node.name
sorted_nodes = topological_sort(self.graph)
if len(self.graph) > 0:
for i in xrange(len(self.nodes)):
self.sorted_nodes.append(sorted_nodes.popleft())
for node in self.nodes:
parent = self.nodes[node].parent
for par in tolist(parent.keys()):
try:
self.nodes[node].parent[par] = self.inputs_dim[par]
except:
if self.nodes[par].nout is not None:
# Assume this is FullyConnectedLayer
self.nodes[node].parent[par] = self.nodes[par].nout
else:
# Assume this is ConvLayer
try:
self.nodes[node].parent[par] = self.nodes[
par].outshape
except:
# Assume this is MaxPool2D
self.nodes[par].initialize()
self.nodes[node].parent[par] = self.nodes[
par].outshape
if hasattr(node, 'recurrent'):
recurrent = self.nodes[node].recurrent
for rec in tolist(recurrent.keys()):
self.nodes[node].recurrent[rec] = self.nodes[rec].nout
def __init__(self,
recurrent=[],
recurrent_dim=[],
self_recurrent=1,
init_U=InitCell('ortho'),
**kwargs):
super(RecurrentLayer, self).__init__(**kwargs)
self.recurrent = OrderedDict()
if self_recurrent:
self.recurrent[self.name] = self.nout
recurrent_dim = tolist(recurrent_dim)
for i, rec in enumerate(tolist(recurrent)):
if len(recurrent_dim) != 0:
self.recurrent[rec] = recurrent_dim[i]
else:
self.recurrent[rec] = None
self.init_U = init_U
def __init__(self,
batch_size,
recurrent=[],
recurrent_dim=[],
self_recurrent=1,
init_state_cons=0.,
init_U=InitCell('ortho'),
**kwargs):
super(RecurrentLayer, self).__init__(**kwargs)
self.recurrent = OrderedDict()
if self_recurrent:
self.recurrent[self.name] = self.nout
recurrent_dim = tolist(recurrent_dim)
for i, rec in enumerate(tolist(recurrent)):
if len(recurrent_dim) != 0:
self.recurrent[rec] = recurrent_dim[i]
else:
self.recurrent[rec] = None
self.batch_size = batch_size
self.init_U = init_U
self.init_states = OrderedDict()
self.init_state_cons = init_state_cons
def set_graph(self):
self.graph = {}
for nname, node in self.nodes.items():
parent = node.parent
for par in tolist(parent.keys()):
if par in self.inputs.keys():
continue
if par in self.graph.keys():
self.graph[par] =\
tolist(self.graph[par]) + [node.name]
else:
self.graph[par] = node.name
sorted_nodes = topological_sort(self.graph)
if len(self.graph) > 0:
for i in xrange(len(self.nodes)):
self.sorted_nodes.append(sorted_nodes.popleft())
for node in self.nodes:
parent = self.nodes[node].parent
for par in tolist(parent.keys()):
try:
self.nodes[node].parent[par] = self.inputs_dim[par]
except:
if self.nodes[par].nout is not None:
# Assume this is FullyConnectedLayer
self.nodes[node].parent[par] = self.nodes[par].nout
else:
# Assume this is ConvLayer
try:
self.nodes[node].parent[par] = self.nodes[par].outshape
except:
# Assume this is MaxPool2D
self.nodes[par].initialize()
self.nodes[node].parent[par] = self.nodes[par].outshape
if hasattr(node, 'recurrent'):
recurrent = self.nodes[node].recurrent
for rec in tolist(recurrent.keys()):
self.nodes[node].recurrent[rec] = self.nodes[rec].nout
def __init__(self,
name,
data,
model,
optimizer,
cost,
outputs,
n_steps,
debug_print=0,
trainlog=None,
extension=None,
lr_iterations=None,
decay_schedule=2,
k_speedOfconvergence=40):
#picklelized?
self.name = name # yes
self.data = data # no
self.model = model #yes
self.optimizer = optimizer #no
self.inputs = model.inputs #no
self.cost = cost #yes
self.outputs = tolist(outputs) #no
self.updates = OrderedDict() # no
self.updates.update(model.updates) #???
self.extension = extension #no
self.debug_print = debug_print #no
lr_scalers = OrderedDict() #yes
for node in self.model.nodes: #should
lr_scalers[node.name] = node.lr_scaler
self.optimizer.lr_scalers = lr_scalers #should
self.nBernoulli = np.ones((n_steps, )) #yes
t0 = time.time()
self.cost_fn = self.build_training_graph() # no but should
print "Elapsed compilation time: %f" % (time.time() - t0)
if self.debug_print: #no
from theano.printing import debugprint
debugprint(self.cost_fn)
if trainlog is None: #yes
self.trainlog = TrainLog()
else:
self.trainlog = trainlog
self.endloop = 0 #no
self.lr_iterations = lr_iterations #yes
self.lastBatchlastPoch = 0 #yes
self.decay_schedule = decay_schedule #yes
self.k = k_speedOfconvergence #yes
self.schedRate = 1 #yes
self.n_steps = n_steps #yes
def restore(self,
data,
optimizer,
cost,
outputs,
n_steps,
debug_print=0,
trainlog=None,
extension=None,
lr_iterations=None,
decay_schedule=2,
k_speedOfconvergence=40):
self.data = data
self.optimizer = optimizer
self.inputs = self.model.inputs
self.cost = cost
self.outputs = tolist(outputs)
#self.updates = OrderedDict()
#self.updates.update(self.model.updates)
self.updates = self.model.updates
self.extension = extension
self.debug_print = debug_print
lr_scalers = OrderedDict()
for node in self.model.nodes:
lr_scalers[node.name] = node.lr_scaler
self.optimizer.lr_scalers = lr_scalers
self.nBernoulli = np.ones((n_steps, ))
t0 = time.time()
self.cost_fn = self.build_training_graph()
print "Elapsed compilation time: %f" % (time.time() - t0)
if self.debug_print:
from theano.printing import debugprint
debugprint(self.cost_fn)
if trainlog is None:
self.trainlog = TrainLog()
else:
self.trainlog = trainlog
self.endloop = 0
self.lr_iterations = lr_iterations
self.lastBatchlastPoch = 0
self.decay_schedule = decay_schedule
self.k = k_speedOfconvergence
self.schedRate = 1
self.n_steps = n_steps
def __init__(self,
name,
data,
model,
optimizer,
cost,
outputs,
extension=None):
self.name = name
self.data = data
self.model = model
self.optimizer = optimizer
self.inputs = model.inputs
self.params = model.params
self.cost = cost
self.outputs = tolist(outputs)
self.updates = model.updates
self.extension = extension
self.cost_fn = self.build_training_graph()
self.trainlog = TrainLog()
self.endloop = 0
def __init__(self,
parshape=[],
outshape=None,
filtershape=None,
tied_bias=True,
step_size=(1, 1),
border_mode='valid',
**kwargs):
super(Conv2DLayer, self).__init__(**kwargs)
# Shape should be (batch_size, num_channels, x, y)
if (outshape is None and filtershape is None) or\
(outshape is not None and filtershape is not None):
raise ValueError("Either outshape or filtershape should be given,\
but don't provide both of them.")
self.outshape = outshape
self.filtershape = filtershape
self.tied_bias = tied_bias
self.step_size = totuple(step_size)
self.border_mode = border_mode
for i, par in enumerate(tolist(self.parent.keys())):
if len(parshape) != 0:
self.parent[par] = parshape[i]
def __init__(self,
parshape=[],
outshape=None,
filtershape=None,
tied_bias=True,
step_size=(1, 1),
border_mode='valid',
**kwargs):
super(Conv2DLayer, self).__init__(**kwargs)
# Shape should be (batch_size, num_channels, x, y)
if (outshape is None and filtershape is None) or\
(outshape is not None and filtershape is not None):
raise ValueError("Either outshape or filtershape should be given,\
but don't provide both of them.")
self.outshape = outshape
self.filtershape = filtershape
self.tied_bias = tied_bias
self.step_size = totuple(step_size)
self.border_mode = border_mode
for i, par in enumerate(tolist(self.parent.keys())):
if len(parshape) != 0:
self.parent[par] = parshape[i]
def get_params(self):
params = []
for graph in tolist(self.graphs):
params += graph.params
return params
def slices(self, start, end):
batches = [mat[start:end] for mat in self.data]
mask = tolist(self.create_mask(batches[0].swapaxes(0, 1)))
batches = [self.zero_pad(batch) for batch in batches]
return totuple(batches + mask)
def add_node(self, nodes):
for node in tolist(nodes):
self.nodes[node.name] = node
self.params = self.get_params()
def scan_fn(self, *args):
next_recurrence = []
sorted_nodes = topological_sort(self.graph)
inputs = tolist(args[:self.nseqs])
recurrence = tolist(args[self.nseqs:self.nseqs+self.nrecur])
inputs += tolist(args[self.nseqs+self.nrecur:self.nseqs+self.noutputs])
nonseqs = tolist(args[self.nseqs+self.noutputs:])
for nname, node in self.nodes.items():
for i, (aname, arg) in enumerate(self.recur_args.items()):
if node is arg:
node.rec_out = recurrence[i]
if len(sorted_nodes) != 0:
for node in self.sorted_nodes:
inp = []
parent = self.nodes[node].parent
for par in parent:
tok = 1
for inp2 in inputs:
if par in inp2.name:
inp.append(inp2)
tok = 0
break
if tok:
inp.append(self.nodes[par].out)
if self.nodes[node] in self.recur_args.values():
rec_inp = []
recurrent = self.nodes[node].recurrent
for rec in recurrent:
rec_inp.append(self.nodes[rec].rec_out)
inp = [inp, rec_inp]
self.nodes[node].out = self.nodes[node].fprop(inp)
next_recurrence.append(self.nodes[node].out)
else:
self.nodes[node].out = self.nodes[node].fprop(inp)
else:
# Assume that you have only single depth (parallel) graph
# Instead of Queue use for-loop to forcibly run the operation
for node in self.nodes:
inp = []
parent = self.nodes[node].parent
for par in parent:
tok = 1
for inp2 in inputs:
if par in inp2.name:
inp.append(inp2)
tok = 0
break
if tok:
inp.append(self.nodes[par].out)
if self.nodes[node] in self.recur_args.values():
rec_inp = []
recurrent = self.nodes[node].recurrent
for rec in recurrent:
rec_inp.append(self.nodes[rec].rec_out)
inp = [inp, rec_inp]
self.nodes[node].out = self.nodes[node].fprop(inp)
next_recurrence.append(self.nodes[node].out)
else:
self.nodes[node].out = self.nodes[node].fprop(inp)
required_outputs = []
if self.iterators is not None:
for arg in self.iterators:
for node in self.nodes.values():
if node is arg:
required_outputs.append(node.out)
if self.output_args is not None:
for arg in self.output_args:
for node in self.nodes.values():
if node is arg:
required_outputs.append(node.out)
return next_recurrence + required_outputs
def scan_fn(self, *args):
next_recurrence = []
sorted_nodes = topological_sort(self.graph)
inputs = tolist(args[:self.nseqs])
recurrence = tolist(args[self.nseqs:self.nseqs+self.nrecur])
inputs += tolist(args[self.nseqs+self.nrecur:self.nseqs+self.noutputs])
nonseqs = tolist(args[self.nseqs+self.noutputs:])
for nname, node in self.nodes.items():
for i, (aname, arg) in enumerate(self.recur_args.items()):
if node is arg:
node.rec_out = recurrence[i]
if len(sorted_nodes) != 0:
for node in self.sorted_nodes:
inp = []
parent = self.nodes[node].parent
for par in parent:
tok = 1
for inp2 in inputs:
if par in inp2.name:
inp.append(inp2)
tok = 0
break
if tok:
inp.append(self.nodes[par].out)
if self.nodes[node] in self.recur_args.values():
rec_inp = []
recurrent = self.nodes[node].recurrent
for rec in recurrent:
rec_inp.append(self.nodes[rec].rec_out)
inp = [inp, rec_inp]
self.nodes[node].out = self.nodes[node].fprop(inp)
next_recurrence.append(self.nodes[node].out)
else:
self.nodes[node].out = self.nodes[node].fprop(inp)
else:
# Assume that you have only single depth (parallel) graph
# Instead of Queue use for-loop to forcibly run the operation
for node in self.nodes:
inp = []
parent = self.nodes[node].parent
for par in parent:
tok = 1
for inp2 in inputs:
if par in inp2.name:
inp.append(inp2)
tok = 0
break
if tok:
inp.append(self.nodes[par].out)
if self.nodes[node] in self.recur_args.values():
rec_inp = []
recurrent = self.nodes[node].recurrent
for rec in recurrent:
rec_inp.append(self.nodes[rec].rec_out)
inp = [inp, rec_inp]
self.nodes[node].out = self.nodes[node].fprop(inp)
next_recurrence.append(self.nodes[node].out)
else:
self.nodes[node].out = self.nodes[node].fprop(inp)
required_outputs = []
if self.iterators is not None:
for arg in self.iterators:
for node in self.nodes.values():
if node is arg:
required_outputs.append(node.out)
if self.output_args is not None:
for arg in self.output_args:
for node in self.nodes.values():
if node is arg:
required_outputs.append(node.out)
return next_recurrence + required_outputs
def add_node(self, nodes):
for node in tolist(nodes):
self.nodes[node.name] = node
self.params = self.get_params()
def slices(self, start, end):
batches = [mat[start:end] for mat in self.data]
mask = tolist(self.create_mask(batches[0].swapaxes(0, 1)))
batches = [self.zero_pad(batch) for batch in batches]
return totuple(batches + mask)
def get_params(self):
params = []
for graph in tolist(self.graphs):
params += graph.params
return params
def load(self, data_path):
dataset = 'audio.tar.gz'
datafile = os.path.join(data_path, dataset)
if not os.path.isfile(datafile):
try:
import urllib
urllib.urlretrieve('http://google.com')
url =\
'https://dl.dropboxusercontent.com/u/15378192/audio.tar.gz'
except AttributeError:
import urllib.request as urllib
url =\
'https://dl.dropboxusercontent.com/u/15378192/audio.tar.gz'
print("Downloading data from %s" % url)
urllib.urlretrieve(url, datafile)
if not os.path.exists(os.path.join(data_path, "audio")):
tar = tarfile.open(datafile)
os.chdir(data_path)
tar.extractall()
tar.close()
h5_file_path = os.path.join(data_path, "saved_fruit.h5")
if not os.path.exists(h5_file_path):
data_path = os.path.join(data_path, "audio")
audio_matches = []
for root, dirnames, filenames in os.walk(data_path):
for filename in fnmatch.filter(filenames, '*.wav'):
audio_matches.append(os.path.join(root, filename))
random.seed(1999)
random.shuffle(audio_matches)
# http://mail.scipy.org/pipermail/numpy-discussion/2011-March/055219.html
h5_file = tables.openFile(h5_file_path, mode='w')
data_x = h5_file.createVLArray(h5_file.root, 'data_x',
tables.Float32Atom(shape=()),
filters=tables.Filters(1))
data_y = h5_file.createVLArray(h5_file.root, 'data_y',
tables.Int32Atom(shape=()),
filters=tables.Filters(1))
for wav_path in audio_matches:
# Convert chars to int classes
word = wav_path.split(os.sep)[-1][:6]
chars = [ord(c) - 97 for c in word]
data_y.append(np.array(chars, dtype='int32'))
fs, d = wavfile.read(wav_path)
data_x.append(d.astype(theano.config.floatX))
h5_file.close()
h5_file = tables.openFile(h5_file_path, mode='r')
raw_X = np.array([np.asarray(x) for x in h5_file.root.data_x])
cls = np.array([''.join([chr(y+97) for y in Y]) for Y in h5_file.root.data_y])
if self.name != 'all':
fruit_list = []
if len(self.name) > 1:
for i, fruit_name in enumerate(cls):
for name in self.name:
if name in fruit_name:
fruit_list.append(i)
else:
for i, fruit_name in enumerate(cls):
if self.name in fruit_name:
fruit_list.append(i)
else:
fruit_list = tolist(np.arange(len(raw_X)))
raw_X = raw_X[fruit_list]
if self.prep == 'normalize':
pre_X, self.X_mean, self.X_std = self.global_normalize(raw_X)
elif self.prep == 'standardize':
pre_X, self.X_max, self.X_min = self.standardize(raw_X)
X = np.array([segment_axis(x, self.frame_size, 0) for x in pre_X])
return [X]
