def _classify(params):
ret = { }
output_dim = {}
hash = hashlib.new('ripemd160')
hash.update(json.dumps(params))
hash = hash.hexdigest()
for k in params:
try:
params[k] = numpy.asarray(params[k], dtype='float32')
if k != 'data':
output_dim[k] = network.n_out[k] # = [network.n_in,2] if k == 'data' else network.n_out[k]
except Exception:
if k != 'data' and not k in network.n_out:
ret['error'] = 'unknown target: %s' % k
else:
ret['error'] = 'unable to convert %s to an array from value %s' % (k,str(params[k]))
break
if not 'error' in ret:
data = StaticDataset(data=[params], output_dim=output_dim)
data.init_seq_order()
try:
data = StaticDataset(data=[params], output_dim=output_dim)
data.init_seq_order()
except Exception:
ret['error'] = "invalid data: %s" % params
else:
batches = data.generate_batches(recurrent_net=network.recurrent,
batch_size=sys.maxsize, max_seqs=1)
if not hash in workers:
workers[hash] = ClassificationTaskThread(network, devices, data, batches)
workers[hash].json_params = params
print("worker started:", hash, file=log.v3)
ret['result'] = { 'hash' : hash }
return ret
def _classify(params):
ret = { }
output_dim = {}
hash = hashlib.new('ripemd160')
hash.update(json.dumps(params))
hash = hash.hexdigest()
for k in params:
try:
params[k] = numpy.asarray(params[k], dtype='float32')
if k != 'data':
output_dim[k] = network.n_out[k] # = [network.n_in,2] if k == 'data' else network.n_out[k]
except Exception:
if k != 'data' and not k in network.n_out:
ret['error'] = 'unknown target: %s' % k
else:
ret['error'] = 'unable to convert %s to an array from value %s' % (k,str(params[k]))
break
if not 'error' in ret:
data = StaticDataset(data=[params], output_dim=output_dim)
data.init_seq_order()
try:
data = StaticDataset(data=[params], output_dim=output_dim)
data.init_seq_order()
except Exception:
ret['error'] = "invalid data: %s" % params
else:
batches = data.generate_batches(recurrent_net=network.recurrent,
batch_size=sys.maxint, max_seqs=1)
if not hash in classifiers:
classifiers[hash] = ClassificationTaskThread(network, devices, data, batches)
classifiers[hash].json_params = params
print >> log.v3, "classifier started:", hash
ret['result'] = { 'hash' : hash }
return ret
def classify(self, data, output_file):
out = open(output_file, 'w')
batches = data.generate_batches(recurrent_net=self.network.recurrent,
batch_size=data.num_timesteps, max_seqs=1)
forwarder = ClassificationTaskThread(self.network, self.devices, data, batches)
forwarder.join()
print >> out, forwarder.output
out.close()
def classify(self, data, output_file):
out = open(output_file, 'w')
batches = data.generate_batches(recurrent_net=self.network.recurrent,
batch_size=data.num_timesteps, max_seqs=1)
forwarder = ClassificationTaskThread(self.network, self.devices, data, batches)
forwarder.join()
print(forwarder.output, file=out)
out.close()
def _classification_task(self, network, devices, data, batches):
"""
Runs a classification task in an executor (async).
:param network: Engine network.
:param devices: Devices assigned to this engine.
:param data: The data on which to classify.
:param batches: Generated batches.
:return:
"""
# This will be executed in `executor` pool
td = ClassificationTaskThread(network, devices, data, batches)
td.join()
return td
def forward_single(self, dataset, seq_idx, output_layer_name=None):
"""
Forwards a single sequence.
If you want to perform search, and get a number of hyps out, use :func:`search_single`.
:param Dataset.Dataset dataset:
:param int seq_idx:
:param str|None output_layer_name: e.g. "output". if not set, will read from config "forward_output_layer"
:return: numpy array, output in time major format (time,dim)
:rtype: numpy.ndarray
"""
batch = Batch()
batch.init_with_one_full_sequence(seq_idx=seq_idx, dataset=dataset)
batch_generator = iter([batch])
batches = BatchSetGenerator(dataset, generator=batch_generator)
forwarder = ClassificationTaskThread(self.network, self.devices, dataset, batches)
forwarder.join()
assert forwarder.output.shape[1] == 1
return forwarder.output[:, 0]
def forward_single(self, dataset, seq_idx, output_layer_name=None):
"""
Forwards a single sequence.
If you want to perform search, and get a number of hyps out, use :func:`search_single`.
:param Dataset.Dataset dataset:
:param int seq_idx:
:param str|None output_layer_name: e.g. "output". if not set, will read from config "forward_output_layer"
:return: numpy array, output in time major format (time,dim)
:rtype: numpy.ndarray
"""
from EngineBatch import Batch, BatchSetGenerator
batch = Batch()
batch.init_with_one_full_sequence(seq_idx=seq_idx, dataset=dataset)
batch_generator = iter([batch])
batches = BatchSetGenerator(dataset, generator=batch_generator)
forwarder = ClassificationTaskThread(self.network, self.devices, dataset, batches)
forwarder.join()
assert forwarder.output.shape[1] == 1
return forwarder.output[:, 0]
def classification_task(self, network, devices, data, batches):
#This will be executed in `executor` pool
td = ClassificationTaskThread(network, devices, data, batches)
td.join()
return td