def predict_ids(model_name, vg, vd): workspace.ResetWorkspace() # preproc the input vg = vg.astype(np.float32) vd = vd.astype(np.float32) #if len(self.preproc_param) == 0: preproc_param = pickle.load(open(model_name + '_preproc_param.p', "rb")) dummy_ids = np.zeros(len(vg)) preproc_data_arrays = preproc.dc_iv_preproc( vg, vd, dummy_ids, preproc_param['scale'], preproc_param['vg_shift'], ) # print(preproc_data_arrays[0].shape) _preproc_data_arrays = [ np.expand_dims(x, axis=1) if len(x.shape) < 2 else x for x in preproc_data_arrays ] workspace.FeedBlob('DBInput_train/sig_input', _preproc_data_arrays[0]) workspace.FeedBlob('DBInput_train/tanh_input', _preproc_data_arrays[1]) pred_net = exporter.load_net(model_name + '_init', model_name + '_predict') workspace.RunNet(pred_net) _ids = np.squeeze(workspace.FetchBlob('prediction')) restore_id_func, get_restore_id_grad_func = preproc.get_restore_id_func( preproc_param['scale']) ids = restore_id_func(_ids) return ids
def predict_ids(self, vg, vd): # preproc the input vg = vg.astype(np.float32) vd = vd.astype(np.float32) if len(self.preproc_param) == 0: self.preproc_param = pickle.load(open(self.pickle_file_name, "rb")) dummy_ids = np.zeros(len(vg)) preproc_data_arrays = preproc.dc_iv_preproc( vg, vd, dummy_ids, self.preproc_param['scale'], self.preproc_param['vg_shift'], ) _preproc_data_arrays = [ np.expand_dims(x, axis=1) for x in preproc_data_arrays ] workspace.FeedBlob('DBInput_train/sig_input', _preproc_data_arrays[0]) workspace.FeedBlob('DBInput_train/tanh_input', _preproc_data_arrays[1]) pred_net = self.net_store['pred_net'] workspace.RunNet(pred_net) _ids = np.squeeze(schema.FetchRecord(self.pred).get()) restore_id_func, _ = preproc.get_restore_id_func( self.preproc_param['scale'], self.preproc_param['vg_shift'], ) ids = restore_id_func(_ids) return _ids, ids
def add_data( self, data_tag, data_arrays, preproc_param, override=True, ): ''' data_arrays are in the order of sig_input, tanh_input, and label ''' assert len(data_arrays) == 3, 'Incorrect number of input data' # number of examples and same length assertion num_example = len(data_arrays[0]) for data in data_arrays[1:]: assert len(data) == num_example, 'Mismatch dimensions' # set default values in preproc_param if not set preproc_param.setdefault('preproc_slope_vg', -1.0) preproc_param.setdefault('preproc_threshold_vg', 0.0) preproc_param.setdefault('preproc_slope_vd', -1.0) preproc_param.setdefault('preproc_threshold_vd', 0.0) self.preproc_param = preproc_param self.pickle_file_name = self.model_name + '_preproc_param' + '.p' db_name = self.model_name + '_' + data_tag + '.minidb' if os.path.isfile(db_name): if override: print("XXX Delete the old database...") os.remove(db_name) os.remove(self.pickle_file_name) else: raise Exception( 'Encounter database with the same name. ' + 'Choose the other model name or set override to True.') print("+++ Create a new database...") pickle.dump(self.preproc_param, open(self.pickle_file_name, 'wb')) preproc_data_arrays = preproc.dc_iv_preproc( data_arrays[0], data_arrays[1], data_arrays[2], self.preproc_param['scale'], self.preproc_param['vg_shift'], slope_vg=self.preproc_param['preproc_slope_vg'], thre_vg=self.preproc_param['preproc_threshold_vg'], slope_vd=self.preproc_param['preproc_slope_vd'], thre_vd=self.preproc_param['preproc_threshold_vd'], ) self.preproc_data_arrays = preproc_data_arrays # Only expand the dim if the number of dimension is 1 preproc_data_arrays = [ np.expand_dims(x, axis=1) if x.ndim == 1 else x for x in preproc_data_arrays ] # Write to database data_reader.write_db('minidb', db_name, preproc_data_arrays) self.input_data_store[data_tag] = [db_name, num_example]
def predict_ids(self, vg, vd): # preproc the input vg = vg.astype(np.float32) vd = vd.astype(np.float32) if len(self.preproc_param) == 0: self.preproc_param = pickle.load(open(self.pickle_file_name, "rb")) dummy_ids = np.zeros(len(vg)) preproc_data_arrays = [vg, vd, dummy_ids] if self.train_target == TrainTarget.ORIGIN: preproc_data_arrays = preproc.dc_iv_preproc( vg, vd, dummy_ids, self.preproc_param['scale'], self.preproc_param['vg_shift'], ) _preproc_data_arrays = [ np.expand_dims(x, axis=1) if x.ndim == 1 else x for x in preproc_data_arrays ] workspace.FeedBlob('DBInput_train/sig_input', _preproc_data_arrays[0]) workspace.FeedBlob('DBInput_train/tanh_input', _preproc_data_arrays[1]) if self.train_target == TrainTarget.ADJOINT: adjoint_input = np.ones((vg.shape[0], 1)) workspace.FeedBlob('DBInput_train/adjoint_input', adjoint_input) pred_net = self.net_store['pred_net'] workspace.RunNet(pred_net) _ids = np.squeeze(schema.FetchRecord(self.pred).get()) ids = _ids if self.train_target == TrainTarget.ORIGIN: restore_id_func, _ = preproc.get_restore_id_func( self.preproc_param['scale'], self.preproc_param['vg_shift'], ) ids = restore_id_func(_ids) return _ids, ids
def predict_ids_grads(model_name, vg, vd): workspace.ResetWorkspace() # preproc the input vg = vg.astype(np.float32) vd = vd.astype(np.float32) #if len(self.preproc_param) == 0: preproc_param = pickle.load(open(model_name + '_preproc_param.p', "rb")) dummy_ids = np.zeros(len(vg)) preproc_data_arrays = preproc.dc_iv_preproc( vg, vd, dummy_ids, preproc_param['scale'], preproc_param['vg_shift'], ) _preproc_data_arrays = [ np.expand_dims(x, axis=1) for x in preproc_data_arrays ] workspace.FeedBlob('DBInput_train/sig_input', _preproc_data_arrays[0]) workspace.FeedBlob('DBInput_train/tanh_input', _preproc_data_arrays[1]) adjoint_input = np.ones((_preproc_data_arrays[0].shape[0], 1)) workspace.FeedBlob('adjoint_input', adjoint_input) pred_net = exporter.load_net(model_name + '_init', model_name + '_predict') workspace.RunNet(pred_net) _ids = np.squeeze(workspace.FetchBlob('origin/Mul/origin_pred')) _sig_grad = np.squeeze( workspace.FetchBlob('adjoint/sig_fc_layer_0/output')) _tanh_grad = np.squeeze( workspace.FetchBlob('adjoint/tanh_fc_layer_0/output')) restore_id_func, get_restore_id_grad_func = preproc.get_restore_id_func( preproc_param['scale']) ids = restore_id_func(_ids) sig_grad, tanh_grad = get_restore_id_grad_func(_sig_grad, _tanh_grad) return ids, sig_grad, tanh_grad
def add_data( self, data_tag, data_arrays, preproc_param, override=True, ): ''' data_arrays are in the order of 1) for train origin: sig_input, tanh_input, and label 2) for train adjoint: sig_input, tanh_input, sig_adjoint_label and tanh_adjoint_label ''' assert ((len(data_arrays) == 3 and self.train_target == TrainTarget.ORIGIN) or (len(data_arrays) == 4 and self.train_target == TrainTarget.ADJOINT)), 'Incorrect number of input data' # number of examples and same length assertion num_example = len(data_arrays[0]) for data in data_arrays[1:]: assert len(data) == num_example, 'Mismatch dimensions' self.preproc_param = preproc_param self.pickle_file_name = self.model_name + '_preproc_param' + '.p' db_name = self.model_name + '_' + data_tag + '.minidb' if os.path.isfile(db_name): if override: print("XXX Delete the old database...") os.remove(db_name) os.remove(self.pickle_file_name) else: raise Exception( 'Encounter database with the same name. ' + 'Choose the other model name or set override to True.') print("+++ Create a new database...") pickle.dump(self.preproc_param, open(self.pickle_file_name, 'wb')) if self.train_target == TrainTarget.ORIGIN: preproc_data_arrays = preproc.dc_iv_preproc( data_arrays[0], data_arrays[1], data_arrays[2], self.preproc_param['scale'], self.preproc_param['vg_shift']) if self.train_target == TrainTarget.ADJOINT: adjoint_input = np.ones((origin_input.shape[0], 1)) raise Exception('Not Implemented') self.preproc_data_arrays = preproc_data_arrays # Only expand the dim if the number of dimension is 1 preproc_data_arrays = [ np.expand_dims(x, axis=1) if x.ndim == 1 else x for x in preproc_data_arrays ] # Write to database data_reader.write_db( 'minidb', db_name, preproc_data_arrays, ) self.input_data_store[data_tag] = [db_name, num_example]
print(data_arrays_train[0].shape, data_arrays_train[1].shape, data_arrays_train[2].shape) print(data_arrays_eval[0].shape, data_arrays_eval[1].shape, data_arrays_eval[2].shape) scale, vg_shift = preproc.compute_dc_meta(*data_arrays_train) preproc_param = { 'scale': scale, 'vg_shift': vg_shift, } print(preproc_param) ## Saving the preproc param preproc_data_arrays_train = preproc.dc_iv_preproc(data_arrays_train[0], data_arrays_train[1], data_arrays_train[2], preproc_param['scale'], preproc_param['vg_shift']) preproc_data_arrays_eval = preproc.dc_iv_preproc(data_arrays_eval[0], data_arrays_eval[1], data_arrays_eval[2], preproc_param['scale'], preproc_param['vg_shift']) # Only expand the dim if the number of dimension is 1 preproc_data_arrays_train = [ np.expand_dims(x, axis=1) if x.ndim == 1 else x for x in preproc_data_arrays_train ] preproc_data_arrays_eval = [ np.expand_dims(x, axis=1) if x.ndim == 1 else x