def _classify(params): ret = { } output_dim = {} hash = hashlib.new('ripemd160') hash.update(json.dumps(params)) hash = hash.hexdigest() for k in params: try: params[k] = numpy.asarray(params[k], dtype='float32') if k != 'data': output_dim[k] = network.n_out[k] # = [network.n_in,2] if k == 'data' else network.n_out[k] except Exception: if k != 'data' and not k in network.n_out: ret['error'] = 'unknown target: %s' % k else: ret['error'] = 'unable to convert %s to an array from value %s' % (k,str(params[k])) break if not 'error' in ret: data = StaticDataset(data=[params], output_dim=output_dim) data.init_seq_order() try: data = StaticDataset(data=[params], output_dim=output_dim) data.init_seq_order() except Exception: ret['error'] = "invalid data: %s" % params else: batches = data.generate_batches(recurrent_net=network.recurrent, batch_size=sys.maxsize, max_seqs=1) if not hash in workers: workers[hash] = ClassificationTaskThread(network, devices, data, batches) workers[hash].json_params = params print("worker started:", hash, file=log.v3) ret['result'] = { 'hash' : hash } return ret
def _classify(params): ret = { } output_dim = {} hash = hashlib.new('ripemd160') hash.update(json.dumps(params)) hash = hash.hexdigest() for k in params: try: params[k] = numpy.asarray(params[k], dtype='float32') if k != 'data': output_dim[k] = network.n_out[k] # = [network.n_in,2] if k == 'data' else network.n_out[k] except Exception: if k != 'data' and not k in network.n_out: ret['error'] = 'unknown target: %s' % k else: ret['error'] = 'unable to convert %s to an array from value %s' % (k,str(params[k])) break if not 'error' in ret: data = StaticDataset(data=[params], output_dim=output_dim) data.init_seq_order() try: data = StaticDataset(data=[params], output_dim=output_dim) data.init_seq_order() except Exception: ret['error'] = "invalid data: %s" % params else: batches = data.generate_batches(recurrent_net=network.recurrent, batch_size=sys.maxint, max_seqs=1) if not hash in classifiers: classifiers[hash] = ClassificationTaskThread(network, devices, data, batches) classifiers[hash].json_params = params print >> log.v3, "classifier started:", hash ret['result'] = { 'hash' : hash } return ret
def classify(self, data, output_file): out = open(output_file, 'w') batches = data.generate_batches(recurrent_net=self.network.recurrent, batch_size=data.num_timesteps, max_seqs=1) forwarder = ClassificationTaskThread(self.network, self.devices, data, batches) forwarder.join() print >> out, forwarder.output out.close()
def classify(self, data, output_file): out = open(output_file, 'w') batches = data.generate_batches(recurrent_net=self.network.recurrent, batch_size=data.num_timesteps, max_seqs=1) forwarder = ClassificationTaskThread(self.network, self.devices, data, batches) forwarder.join() print(forwarder.output, file=out) out.close()
def _classification_task(self, network, devices, data, batches): """ Runs a classification task in an executor (async). :param network: Engine network. :param devices: Devices assigned to this engine. :param data: The data on which to classify. :param batches: Generated batches. :return: """ # This will be executed in `executor` pool td = ClassificationTaskThread(network, devices, data, batches) td.join() return td
def forward_single(self, dataset, seq_idx, output_layer_name=None): """ Forwards a single sequence. If you want to perform search, and get a number of hyps out, use :func:`search_single`. :param Dataset.Dataset dataset: :param int seq_idx: :param str|None output_layer_name: e.g. "output". if not set, will read from config "forward_output_layer" :return: numpy array, output in time major format (time,dim) :rtype: numpy.ndarray """ batch = Batch() batch.init_with_one_full_sequence(seq_idx=seq_idx, dataset=dataset) batch_generator = iter([batch]) batches = BatchSetGenerator(dataset, generator=batch_generator) forwarder = ClassificationTaskThread(self.network, self.devices, dataset, batches) forwarder.join() assert forwarder.output.shape[1] == 1 return forwarder.output[:, 0]
def forward_single(self, dataset, seq_idx, output_layer_name=None): """ Forwards a single sequence. If you want to perform search, and get a number of hyps out, use :func:`search_single`. :param Dataset.Dataset dataset: :param int seq_idx: :param str|None output_layer_name: e.g. "output". if not set, will read from config "forward_output_layer" :return: numpy array, output in time major format (time,dim) :rtype: numpy.ndarray """ from EngineBatch import Batch, BatchSetGenerator batch = Batch() batch.init_with_one_full_sequence(seq_idx=seq_idx, dataset=dataset) batch_generator = iter([batch]) batches = BatchSetGenerator(dataset, generator=batch_generator) forwarder = ClassificationTaskThread(self.network, self.devices, dataset, batches) forwarder.join() assert forwarder.output.shape[1] == 1 return forwarder.output[:, 0]
def classification_task(self, network, devices, data, batches): #This will be executed in `executor` pool td = ClassificationTaskThread(network, devices, data, batches) td.join() return td