def _time_performance_run_unifed_lstm_gpu( self, test_config, x_train, y_train): # Get performance number for Unified_LSTM with grappler swap the impl input_shape = test_config['input_shape'] rnn_state_size = test_config['rnn_state_size'] timestep = test_config['timestep'] epoch = test_config['epoch'] warmup_epoch = test_config['warmup_epoch'] ops.reset_default_graph() K.set_session(session.Session(config=self.config)) layer = UnifiedLSTM(rnn_state_size) inputs = keras.layers.Input( shape=[timestep, input_shape], dtype=dtypes.float32) outputs, _ = layer(inputs) model = keras.models.Model(inputs, outputs) model.compile('sgd', 'mse') total_duration = 0 for i in range(epoch): start_time = time.time() model.fit(x_train, y_train) end_time = time.time() if i >= warmup_epoch: duration_per_epoch = end_time - start_time total_duration += duration_per_epoch logging.vlog(2, '%s: Time consumed for epoch %d is: %s', 'Unified LSTM', i, duration_per_epoch) logging.info('Average performance for %s per epoch is: %s', 'Unified LSTM', (total_duration / epoch)) return total_duration / epoch
def test_keras_model_with_lstm(self): input_shape = 10 rnn_state_size = 8 output_shape = 8 timestep = 4 batch = 100 epoch = 10 (x_train, y_train), _ = testing_utils.get_test_data( train_samples=batch, test_samples=0, input_shape=(timestep, input_shape), num_classes=output_shape) y_train = keras.utils.to_categorical(y_train, output_shape) K.set_session(session.Session(config=self.config)) layer = UnifiedLSTM(rnn_state_size) inputs = keras.layers.Input( shape=[timestep, input_shape], dtype=dtypes.float32) outputs, unused_runtime = layer(inputs) model = keras.models.Model(inputs, outputs) model.compile('rmsprop', loss='mse') model.fit(x_train, y_train, epochs=epoch)
def setup_tpu_session(master): """Initializes and returns a Keras/TF session connected the TPU `master`.""" session = tf_session.Session( target=master, config=config_pb2.ConfigProto(isolate_session_state=True)) K.set_session(session) K.get_session().run(tpu.initialize_system()) return session
def load(path, opts, vars): try: print('create session and graph') server = tf.train.Server.create_local_server() sess = tf.Session(server.target) graph = tf.get_default_graph() backend.set_session(sess) print('\nload model from ' + path) trainerPath = path trainerPath = trainerPath.replace("trainer.PythonModel.model", "trainer") print('tp: {}'.format(trainerPath)) # parsing trainer to retreive input output xmldoc = minidom.parse(trainerPath) streams = xmldoc.getElementsByTagName('streams') streamItem = streams[0].getElementsByTagName('item') n_input = streamItem[0].attributes['dim'].value print("#input: " + str(n_input)) classes = xmldoc.getElementsByTagName('classes') n_output = len(classes[0].getElementsByTagName('item')) print("#output: " + str(n_output)) # copy unique network file network_tmp = os.path.dirname(path) + '\\' + opts['network'] + '.py' shutil.copy(path + '.' + opts['network'] + '.py', network_tmp) print ('\nload training configuration from ' + network_tmp) # reload sys path and import network file importlib.reload(site) # loading network specific options module = __import__(opts['network']) set_opts_from_config(opts, module.conf) # load weights weigth_path = path + '.' + opts['network'] + '_weights.h5' print('\nModelpath {}'.format(weigth_path)) model = module.getModel(int(n_input), int(n_output)) model.load_weights(weigth_path) # creating the predict function in order to avoid graph-scope errors later print('create prediction function') model._make_predict_function() #with graph.as_default(): # with sess.as_default(): # n_input = model.inputs[0].shape[1] # model.predict(np.zeros((1,n_input), dtype=np.float32)) vars['graph'] = graph vars['session'] = sess vars['model'] = model except Exception as e: print_exception(e, 'load') exit()
def configure_and_create_session(distribution_strategy): """Configure session config and create a session with it.""" # TODO(priyag): Throw error if a session already exists. session_config = K.get_default_session_config() if is_tpu_strategy(distribution_strategy): # TODO(priyag, yuefengz): Remove this workaround when Distribute # Coordinator is integrated with keras and we can create a session from # there. distribution_strategy.configure(session_config) master = distribution_strategy.extended._tpu_cluster_resolver.master() # pylint: disable=protected-access session = session_module.Session(config=session_config, target=master) else: worker_context = dc_context.get_current_worker_context() if worker_context: dc_session_config = worker_context.session_config # Merge the default session config to the one from distribute coordinator, # which is fine for now since they don't have conflicting configurations. dc_session_config.MergeFrom(session_config) session = session_module.Session( config=dc_session_config, target=worker_context.master_target) else: session = session_module.Session(config=session_config) K.set_session(session)
def inception_feature_activations(hdf5s, input_shape, batch_size, checkpoint_path=None): # Inception Network. # Pool_3 layer output, features for FID. if checkpoint_path is None: print('InceptionV3: ImageNet pre-trained network.') images_input = tf.placeholder(dtype=tf.float32, shape=[None] + input_shape, name='images') images = 2*images_input images -= 1 images = tf.image.resize_bilinear(images, [299, 299]) inception_v3_no_top = inception_v3.InceptionV3(include_top=False, weights='imagenet', input_shape=[299, 299, 3], input_tensor=images) else: print('InceptionV3: Fine-tuned trained network.') images_input = tf.placeholder(dtype=tf.float32, shape=[None] + input_shape, name='images') images = 2*images_input images -= 1 images = tf.image.resize_bilinear(images, [299, 299]) inception_v3_no_top = inception_v3.InceptionV3(include_top=False, weights=None, input_shape=[299, 299, 3], input_tensor=images) net = GlobalAveragePooling2D()(inception_v3_no_top.output) inception_v3_model = Model(inputs=inception_v3_no_top.input, outputs=net) print('Pool_3 InceptionV3 Features:', net.shape.as_list()[-1]) saver = tf.train.Saver() with tf.keras.backend.get_session() as sess: K.set_session(sess) sess.run(tf.global_variables_initializer()) if checkpoint_path is not None: saver.restore(sess, checkpoint_path) print('Restored Model: %s' % checkpoint_path) for hdf5_path in hdf5s: hdf5_img_file = h5py.File(hdf5_path, mode='r') if 'images' not in hdf5_img_file: images_storage = hdf5_img_file['generated_images'] else: images_storage = hdf5_img_file['images'] num_samples = images_storage.shape[0] batches = int(num_samples/batch_size) features_shape = (num_samples, net.shape.as_list()[-1]) hdf5_feature_path = hdf5_path.replace('_images_','_features_') if os.path.isfile(hdf5_feature_path): os.remove(hdf5_feature_path) hdf5_features_file = h5py.File(hdf5_feature_path, mode='w') features_storage = hdf5_features_file.create_dataset(name='features', shape=features_shape, dtype=np.float32) print('Starting features extraction...') print('\tImage File:', hdf5_path) ind = 0 for batch_num in range(batches): batch_images = images_storage[batch_num*batch_size:(batch_num+1)*batch_size] feed_dict = {images_input:batch_images} activations = sess.run(inception_v3_model.outputs, feed_dict) features_storage[batch_num*batch_size:(batch_num+1)*batch_size] = activations[0] ind += batch_size print('\tFeature File:', hdf5_feature_path) print('\tNumber of samples:', ind)
def tpu_session(self): """Yields a TPU session and sets it as the default Keras session.""" with self._graph.as_default(): default_session = K.get_session() # N.B. We have to call `K.set_session()` AND set our session as the # TF default. `K.get_session()` surprisingly does not return the value # supplied by K.set_session otherwise. K.set_session(self._session) with self._session.as_default(): yield self._session K.set_session(default_session)
def configure_and_create_session(distribution_strategy): """Configure session config and create a session with it.""" # TODO(priyag): Throw error if a session already exists. session_config = K.get_default_session_config() distribution_strategy.configure(session_config) if distribution_strategy.__class__.__name__ == 'TPUStrategy': # TODO(priyag): Remove this workaround when Distributed Coordinator is # integrated with keras and we can create a session from there. master = distribution_strategy._tpu_cluster_resolver.master() # pylint: disable=protected-access session = session_module.Session(config=session_config, target=master) else: session = session_module.Session(config=session_config) K.set_session(session)
def setup_tpu_session(tpu_name_or_address): """Initializes and returns a Keras/TF session connected the TPU `master`. Args: tpu_name_or_address: A string that is either the name of the Cloud TPU, the grpc address of the Cloud TPU, or (Googlers only) the BNS name of the Cloud TPU. If tpu_name_or_address is None, the TPUClusterResolver will examine the environment to determine a potential Cloud TPU to use. Returns: A `tf.Session`. """ cluster_resolver = tpu_cluster_resolver.TPUClusterResolver( tpu_name_or_address) cluster_spec = cluster_resolver.cluster_spec() session = tf_session.Session( target=cluster_resolver.master(), config=config_pb2.ConfigProto( isolate_session_state=True)) if cluster_spec: session.cluster_def.CopyFrom(cluster_spec.as_cluster_def()) K.set_session(session) K.get_session().run(tpu.initialize_system()) return session
from tensorflow.python.keras.layers import Input from tensorflow.python.keras.models import Model from tensorflow.python.keras.optimizers import SGD, Adam from tensorflow.python.keras.regularizers import l1 from tensorflow.python.keras import backend as K from tqdm import tqdm from zamba.models.cnnensemble.src import config from zamba.models.cnnensemble.src import utils # turn off noisy CPP compilation notifications os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True # tf_config.gpu_options.visible_device_list = "0" K.set_session(tf.Session(config=tf_config)) INPUT_ROWS = 404 INPUT_COLS = 720 INPUT_CHANNELS = 3 INPUT_SHAPE = (INPUT_ROWS, INPUT_COLS, INPUT_CHANNELS) VIDEO_FPS = 24 PREDICT_FRAMES = [2, 8, 12, 18 ] + [i * VIDEO_FPS // 2 + 24 for i in range(14 * 2)] from .config import CLASSES NB_CLASSES = len(CLASSES) cnnensemble_path = config.MODEL_DIR
def predictFromArray(arr): with graph.as_default(): set_session(sess) result = classifier.predict(arr) index = np.argmax(result[0]) return (labels[index])
def predict_emotion(self, img): global session set_session(session) self.preds = self.loaded_model.predict(img) return Expressions.emotions[np.argmax(self.preds)]
import os import sys import tensorflow as tf from tensorflow.python.keras import backend as K from tensorflow.python.keras.models import Model, load_model # https://github.com/tensorflow/tensorflow/issues/29161 # https://github.com/keras-team/keras/issues/10340 session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto()) K.set_session(session) flags = tf.compat.v1.app.flags flags.DEFINE_string("model_file_path", "model.h5", "File path of the trained model.") flags.DEFINE_string("suffix", "unspecified", "Suffix in the file name.") FLAGS = flags.FLAGS def main(_): print("Getting hyperparameters ...") print("Using command {}".format(" ".join(sys.argv))) flag_values_dict = FLAGS.flag_values_dict() for flag_name in sorted(flag_values_dict.keys()): flag_value = flag_values_dict[flag_name] print(flag_name, flag_value) model_file_path = FLAGS.model_file_path suffix = FLAGS.suffix print("Loading the model from training ...") model = load_model(model_file_path,
try: os.mkdir(dirname) except OSError as exc: if exc.errno != errno.EEXIST: raise pass # ------------------------------------------------------------------------------ # script initialization # ------------------------------------------------------------------------------ # set tensorflow environment to limit GPU memory usage, and select GPU config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True set_session(tf.compat.v1.Session(config=config)) # intialize hardcoded variables kernel_size = [3, 3] output_size = 64 num_layers = 27 patch_size = 51 nepochs = 40 nbatch = 100 learning_rate = .0001 #for Adam optimizer # Setting up paths exp_name = r'_n100_lr0001_' #bbatch & learning rate main_path = r'/cluster/projects/uludag/Brian'
def test_elastic_state(self): with self.test_session(config=self.config) as sess: K.set_session(sess) v = 1.0 if hvd.rank() == 0 else 2.0 model1 = tf.keras.Sequential( [tf.keras.layers.Dense(2, activation='softmax')]) model1.build((2, 2)) model1.set_weights([ np.array([[v, v], [v, v]], dtype=np.float32), np.array([v, v], dtype=np.float32) ]) model2 = tf.keras.Sequential( [tf.keras.layers.Dense(2, activation='softmax')]) model2.build((2, 2)) model2.set_weights([ np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32), np.array([0.0, 0.0], dtype=np.float32) ]) optimizer = tf.keras.optimizers.Adam(0.001 * hvd.size()) state = hvd.elastic.KerasState(model1, optimizer, batch=20 + hvd.rank(), epoch=10 + hvd.rank()) state.sync() model1_weights = model1.get_weights() model2_weights = model2.get_weights() # After sync, all values should match the root rank for w in state.model.get_weights(): self.assertAllClose(w, np.ones_like(w)) assert state.batch == 20 assert state.epoch == 10 # Partially modify then restore model1.set_weights(model2_weights) state.batch = 21 state.epoch = 11 state.restore() for w1, w2 in zip(model1.get_weights(), model1_weights): self.assertAllClose(w1, w2) assert state.batch == 20 assert state.epoch == 10 # Partially modify then commit model1.set_weights(model2_weights) state.batch = 21 state.epoch = 11 state.commit() state.restore() for w1, w2 in zip(model1.get_weights(), model2_weights): self.assertAllClose(w1, w2) assert state.batch == 21 assert state.epoch == 11
def initialize_vars(self, sess): sess.run(tf.compat.v1.local_variables_initializer()) sess.run(tf.compat.v1.global_variables_initializer()) K.set_session(sess)
def export_model(keras_model, export_path, model_version=0, weights_path=None): """Export a model for use with tensorflow-serving. Args: keras_model (tensorflow.keras.Model): instantiated Keras model to export export_path (str): destination to save the exported model files model_version (int): integer version of the model weights_path (str): path to a .h5 or .tf weights file """ # Start the tensorflow session gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8, allow_growth=False) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) K.set_session(sess) K._LEARNING_PHASE = tf.constant(0) K.set_learning_phase(0) # Create export path if it doesn't exist export_path = os.path.join(export_path, str(model_version)) builder = SavedModelBuilder(export_path) # legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') # Initialize global variables and the model init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) # Load the model and the weights if weights_path is not None: keras_model.load_weights(weights_path) # Export for tracking if isinstance(keras_model.input, list): input_map = { "input{}".format(i): input_tensor for i, input_tensor in enumerate(keras_model.input) } output_map = {'output': keras_model.output} # Export for panoptic elif isinstance(keras_model.output, list): input_map = {'image': keras_model.input} output_map = { 'prediction{}'.format(i): tensor for i, tensor in enumerate(keras_model.output) } # Export for normal model architectures else: input_map = {"image": keras_model.input} output_map = {"prediction": keras_model.output} prediction_signature = tf.saved_model.signature_def_utils.predict_signature_def( input_map, output_map) # Add the meta_graph and the variables to the builder builder.add_meta_graph_and_variables( sess, [tag_constants.SERVING], signature_def_map={ signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature }) # Save the graph builder.save()
def main_loop(name, n_episode=100000, max_steps=1000, epsilon=0.2, controler_layer_size=128, sub_policy_layer_size=256): data = pd.DataFrame(columns=["meta_z", "meta_rewards", "rat", "utility"]) config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True session = tf.compat.v1.Session(config=config) KTF.set_session(session) T = 500 totalTime = 0 GAMMA = 0.98 # n_episode = 100000 # max_steps = 10000 i_episode = 0 n_actions = 5 n_signal = 4 render = False meta_Pi = [] meta_V = [] for i in range(n_agent): meta_Pi.append( PPOPolicyNetwork(num_features=32, num_actions=n_signal, layer_size=controler_layer_size, epsilon=epsilon, learning_rate=0.0003)) meta_V.append( ValueNetwork(num_features=32, hidden_size=256, learning_rate=0.001)) Pi = [[] for _ in range(n_agent)] V = [[] for _ in range(n_agent)] for i in range(n_agent): for j in range(n_signal): Pi[i].append( PPOPolicyNetwork(num_features=30, num_actions=n_actions, layer_size=sub_policy_layer_size, epsilon=epsilon, learning_rate=0.0003)) V[i].append( ValueNetwork(num_features=30, hidden_size=256, learning_rate=0.001)) while i_episode < n_episode: i_episode += 1 number = 0 avg = [0] * n_agent u_bar = [0] * n_agent utili = [0] * n_agent u = [[] for _ in range(n_agent)] max_u = 0.003 ep_actions = [[] for _ in range(n_agent)] ep_rewards = [[] for _ in range(n_agent)] ep_states = [[] for _ in range(n_agent)] meta_z = [[] for _ in range(n_agent)] meta_rewards = [[] for _ in range(n_agent)] meta_states = [[] for _ in range(n_agent)] signal = [0] * n_agent rat = [0.0] * n_agent score = 0 steps = 0 env = np.zeros((12, 12)) resource = [] resource_type = [] ant = [] possession = [[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]] for i in range(n_agent): ant.append(np.random.randint(2, 10, 2)) env[ant[i][0]][ant[i][1]] = 1 for i in range(n_resource): resource.append(np.random.randint(3, 9, 2)) resource_type.append(np.random.randint(3)) su = [0] * n_agent ac = [0] * n_actions su = np.array(su) obs = get_obs(ant, resource, resource_type, env, possession, requirement) while steps < max_steps: if number > 700: break if steps % T == 0: for i in range(n_agent): h = copy.deepcopy(obs[i]) h.append(rat[i]) h.append(utili[i]) p_z = meta_Pi[i].get_dist(np.array([h]))[0] z = np.random.choice(range(n_signal), p=p_z) signal[i] = z meta_z[i].append(to_categorical(z, n_signal)) meta_states[i].append(h) steps += 1 action = [] for i in range(n_agent): h = copy.deepcopy(obs[i]) p = Pi[i][signal[i]].get_dist(np.array([h]))[0] action.append(np.random.choice(range(n_actions), p=p)) ep_states[i].append(h) ep_actions[i].append(to_categorical(action[i], n_actions)) env, ant, resource, resource_type, possession, rewards = step( env, ant, action, resource, resource_type, possession, requirement) su += np.array(rewards, dtype=np.int) score += sum(rewards) obs = get_obs(ant, resource, resource_type, env, possession, requirement) for i in range(n_agent): u[i].append(rewards[i]) u_bar[i] = sum(u[i]) / len(u[i]) for i in range(n_agent): avg[i] = sum(u_bar) / len(u_bar) if avg[i] != 0: rat[i] = (u_bar[i] - avg[i]) / avg[i] else: rat[i] = 0 utili[i] = min(1, avg[i] / max_u) for i in range(n_agent): if signal[i] == 0: ep_rewards[i].append(rewards[i]) else: h = copy.deepcopy(obs[i]) h.append(rat[i]) h.append(utili[i]) p_z = meta_Pi[i].get_dist(np.array([h]))[0] r_p = p_z[signal[i]] ep_rewards[i].append(r_p) if steps % T == 0: for i in range(n_agent): meta_rewards[i].append(utili[i] / (0.1 + abs(rat[i]))) if signal[i] == 0: continue ep_actions[i] = np.array(ep_actions[i]) ep_rewards[i] = np.array(ep_rewards[i], dtype=np.float_) ep_states[i] = np.array(ep_states[i]) targets = discount_rewards(ep_rewards[i], GAMMA) V[i][signal[i]].update(ep_states[i], targets) vs = V[i][signal[i]].get(ep_states[i]) ep_advantages = targets - vs ep_advantages = (ep_advantages - np.mean(ep_advantages) ) / (np.std(ep_advantages) + 0.0000000001) Pi[i][signal[i]].update(ep_states[i], ep_actions[i], ep_advantages) ep_actions = [[] for _ in range(n_agent)] ep_rewards = [[] for _ in range(n_agent)] ep_states = [[] for _ in range(n_agent)] for i in range(n_agent): if len(meta_rewards[i]) == 0: continue meta_z[i] = np.array(meta_z[i]) meta_rewards[i] = np.array(meta_rewards[i]) meta_states[i] = np.array(meta_states[i]) meta_V[i].update(meta_states[i], meta_rewards[i]) meta_advantages = meta_rewards[i] - meta_V[i].get(meta_states[i]) meta_Pi[i].update(meta_states[i], meta_z[i], meta_advantages) id_string = "n_episode={}_max_steps={}_epsilon={}_controler_layer_size={}_sub_policy_layer_size={}"\ .format(n_episode, max_steps, epsilon, controler_layer_size, sub_policy_layer_size).replace("=", "_") print("{}/{} {} {}".format(i_episode, n_episode, name, id_string)) data.loc[i_episode] = [ np.array(meta_z), np.array(meta_rewards), rat, np.array(su) / max_steps ] # print("resource utilization", score/max_steps) # print("total score", su) # Save data (at every episode, just to be sure if there is a problem) if not os.path.exists("data/{}_{}".format( os.path.basename(__file__)[:-3], id_string)): os.mkdir("data/{}_{}".format( os.path.basename(__file__)[:-3], id_string)) data.to_pickle("data/{}_{}/{}".format( os.path.basename(__file__)[:-3], id_string, name))
def ai(): ''' Main AI flask server ''' if request.method == 'POST': # for tensorflow static graph global graph, sess sess = tf.Session() graph = tf.get_default_graph() set_session(sess) # init the response dictionary response = {} # get values and files from request values = request.values.to_dict() files = request.files.to_dict() # input dataframes pred_df = values.get('prediction_dataframe') vis_df = values.get('visualize_dataframe') # modes to serve modes = values.get('modes').replace(' ', '').split(',') mode_pred = True if (('prediction' in modes) and pred_df) else False mode_heatmap = True if (('heatmap' in modes) and pred_df) else False mode_vis = True if (('visualize' in modes) and vis_df) else False mode_summary = True if (('summary' in modes) and vis_df) else False print(mode_pred, mode_heatmap, mode_vis, mode_summary) # summary if mode_summary: print('summary mode') response['Summary'] = {} summary_df = summary(vis_df) return Response(summary_df.to_json(orient="records"), mimetype='application/json') # device_id assertion device_id = values.get('device_id') if device_id == None: return '設備IDを指示してください。' # init models sklearn_nos, sklearn_std, tf_nos, tf_std = False, False, False, False sklearn_nos_model, sklearn_std_model, tf_nos_model, tf_std_model = None, None, None, None available_sk_models = [m.name for m in list((sklearn_path/'models'/device_id).rglob('*.pickle')) if 'scaler' not in m.name] available_tf_models = [m.name for m in list((tf_path/'models'/device_id).rglob('*.hdf5'))] available_models = available_sk_models + available_tf_models if values.get('models'): model_name = values['models'] print(f'Model: {model_name}') # available model assertion if model_name not in available_models: print('not available model') return f'Not available models. Current available models for device_id {device_id}: {available_models}.' # standard or not if 'nos' in model_name: model_mode = 'nos' elif 'std' in model_name: model_mode = 'std' # else: return f'' print(model_mode) # sklearn or tensorflow if 'sk' in model_name: module = 'sklearn' sklearn_model_name = model_name sklearn_model = pickle.load(open(sklearn_path/'models'/device_id/model_name, 'rb')) elif 'tf' in model_name: module = 'tensorflow' tf_model_name = model_name tf_model = load_model(tf_path/'models'/device_id/model_name) # else: return f'' print(f'module: {module}') print(f'Loaded model: {model_name}') # else: return f'' # init scalers sklearn_scaler, tf_scaler = None, None sklearn_scaler_name, tf_scaler_name = None, None if model_mode == 'std': if module == 'sklearn': sklearn_scaler_name = f'scaler_of_{sklearn_model_name.split(".")[0]}.pickle' sklearn_scaler_path = sklearn_path/'models'/device_id/sklearn_scaler_name if not os.path.isfile(sklearn_scaler_path): return f'{sklearn_scaler_name} not found.' sklearn_scaler = pickle.load(open(sklearn_scaler_path, 'rb')) print(f'Loaded scaler: {sklearn_scaler_path}') elif module == 'tensorflow': tf_scaler_name = f'scaler_of_{tf_model_name.split(".")[0]}.pickle' tf_scaler_path = tf_path/'models'/device_id/tf_scaler_name if not os.path.isfile(tf_scaler_path): return f'{tf_scaler_name} not found.' tf_scaler = pickle.load(open(tf_scaler_path, 'rb')) print(f'Loaded scaler: {tf_scaler_path}') if module == 'sklearn': response['Scikit-learn'] = {model_mode: {'Model': sklearn_model_name}} if mode_pred: try: pred_res = sk_pred(pred_df, sklearn_model, sklearn_scaler) except Exception as e: tb = traceback.format_exc() print(tb) tb = tb.split('\n')[-2] return jsonify(Error='インポートしたCSVファイルに誤りがあります。') response['Scikit-learn'][model_mode]['Prediction'] = f'{pred_res[0]:.2f}' if mode_vis: try: vis_res = sk_vis(vis_df, sklearn_model, sklearn_scaler) except Exception as e: tb = traceback.format_exc() print(tb) tb = tb.split('\n')[-2] return jsonify(Error='インポートしたCSVファイルに誤りがあります。') response['Scikit-learn'][model_mode]['Visualization'] = {'Sorted predictions' : vis_res['sorted_pred'], 'R2' : vis_res['r2'], 'MAE1' : vis_res['mae1'], 'MAE2' : vis_res['mae2'], 'MSE' : vis_res['mse'], 'RMSE' : vis_res['rmse']} if mode_heatmap: response['Scikit-learn'][model_mode].update({'Heatmap': {}, 'Download': {}}) range1 = values.get('range1').replace(' ', '').split(',') range2 = values.get('range2').replace(' ', '').split(',') sim_input, sim_name1, sim_name2 = get_sim_input(pred_df, range1, range2) sim_df, download_df = simulation(sim_input, sklearn_model, sklearn_scaler, sim_name1, sim_name2) for col in list(sim_df.columns): response['Scikit-learn'][model_mode]['Heatmap'][col] = sim_df[col].to_numpy().tolist() for col in list(download_df.columns): response['Scikit-learn'][model_mode]['Download'][col] = download_df[col].to_numpy().tolist() print(sim_df.head()) return jsonify(response) elif module == 'tensorflow': response['Tensorflow'] = {model_mode: {'Model': tf_model_name}} with graph.as_default(): set_session(sess) if mode_pred: try: pred_res = tf_pred(pred_df, tf_model, tf_scaler) except Exception as e: tb = traceback.format_exc() print(tb) tb = tb.split('\n')[-2] return jsonify(Error='インポートしたCSVファイルに誤りがあります。') response['Tensorflow'][model_mode]['Prediction'] = f'{pred_res[0][0]:.2f}' if mode_vis: try: vis_res = tf_vis(vis_df, tf_model, tf_scaler) except Exception as e: tb = traceback.format_exc() print(tb) tb = tb.split('\n')[-2] return jsonify(Error='インポートしたCSVファイルに誤りがあります。') response['Tensorflow'][model_mode]['Visualization'] = {'Sorted predictions' : vis_res['sorted_pred'], 'R2' : vis_res['r2'], 'MAE1' : vis_res['mae1'], 'MAE2' : vis_res['mae2'], 'MSE' : vis_res['mse'], 'RMSE' : vis_res['rmse']} if mode_heatmap: response['Tensorflow'][model_mode].update({'Heatmap': {}, 'Download': {}}) range1 = values.get('range1').replace(' ', '').split(',') range2 = values.get('range2').replace(' ', '').split(',') sim_input, sim_name1, sim_name2 = get_sim_input(pred_df, range1, range2) sim_df, download_df = simulation(sim_input, tf_model, tf_scaler, sim_name1, sim_name2) for col in list(sim_df.columns): response['Tensorflow'][model_mode]['Heatmap'][col] = sim_df[col].to_numpy().tolist() for col in list(download_df.columns): response['Tensorflow'][model_mode]['Download'][col] = download_df[col].to_numpy().tolist() print(sim_df.head()) return jsonify(response) else: return 'Not allowed method.'
from sklearn.model_selection import train_test_split from sklearn.utils import class_weight # =================================================== # STEP: Globals # =================================================== # Directories dir_medseg = './data/MedSeg/' dir_sandstone = 'C:/Users/elite/Desktop/sandstone_data_for_ML/full_labels_for_deep_learning/128_patches/' dir_models = 'seg_models/' dir_metrics = './metrics/' # Other global variables DEVICE = '/physical_device:GPU:0' config = tf_v1.ConfigProto(device_count={'GPU': 1, 'CPU': 8}) K.set_session(tf_v1.Session(config=config)) # ============================================================= # STEP: Model Parameters # ============================================================= TRAIN_IMAGS = [] TRAIN_MASKS = [] # dir_ims = "images/" # dir_masks = "masks/" dir_ims = "images1/" dir_masks = "masks1/" # Read from TIFF images (MedSeg). DATASET = "MedSeg"
end = (i + 1) * batch_size if end > n_samples: end = n_samples batch_xs = nextBatch(train_x[start:end, :], n_outputs) batch_ys = np.reshape(train_y[start:end], (int(batch_size / n_outputs), n_outputs)) yield batch_xs, batch_ys start = datetime.datetime.now() config = tf.ConfigProto() config.gpu_options.allow_growth = True session = tf.Session(config=config) KTF.set_session(session) # mlp.compile(optimizer=Adam(learning_rate=lr), mlp.compile( optimizer=Adam(lr=lr), loss=my_loss, metrics=[my_metric], ) # file_name_trained = '/home/yezhizi/Documents/TianchiMetro/code/ckpt/lr: 0.0001-batch_size: 93312-l2_param: 0.01-dropout: 0.8-training_epochs: 10000-n_inputs: 239-n_outputs: 3-n_hidden: []-n_mlp: [400, 400, 400]-n_samples2146176-weights[10000, 100, 1]/mlp-ep7600-loss88.766-val_loss98.917-lr: 0.0001-batch_size: 93312-l2_param: 0.01-dropout: 0.8-training_epochs: 10000-n_inputs: 239-n_outputs: 3-n_hidden: []-n_mlp: [400, 400, 400]-n_samples2146176-weights[10000, 100, 1].h5' # mlp.load_weights(file_name_trained) if mode == constants.TRAIN: steps_per_epoch = int(np.ceil(n_samples / batch_size)) check_point = ModelCheckpoint(file_name,
def process_in(self): global sess global graph # while True: while self.state in [self.STATE_EOS_RECEIVED, self.STATE_CONNECTED, self.STATE_INITIALIZED, self.STATE_PROCESSING]: if self.image_in_buf: if self.image_in_buf[0] == 'EOS': # self.image_out_buf.append('EOS') del self.image_in_buf[0] self.feedback_end_to_master() # self.finish_request() continue # try: # self._increment_num_processing(1) #1 else: gray_img = self.img_pre_processing(self.image_in_buf[0]) faces = self.detect_faces(gray_img) # print(faces) if len(faces): #face detected! for face in faces: # print(face) gray_face = self.face_pre_processing(gray_img, face) if gray_face is None: continue with graph.as_default(): set_session(sess) # print(gray_face.shape) emo = self.emotion_classifier.predict(gray_face) self.last_decoder_message = time.time() event = dict(status=common.STATUS_SUCCESS, result=dict(hypotheses=[dict(transcript='return result',\ emo=emo.tolist(),\ face=face.tolist())], final=False,count=self.ccount)) self.image_out_buf.append(event) # try: # self.ws.write_message(json.dumps(event)) # del event # except: # e = sys.exc_info()[1] # print('ERROR_face_respond') # print(e) # logger.warning("Failed to send event to master: %s" % e) else: #no face detected! event = dict(status=common.STATUS_SUCCESS, result=dict(hypotheses=[dict(transcript='return result',\ emo=[],\ face=[])], final=False,count=self.ccount)) self.image_out_buf.append(event) # try: # self.ws.write_message(json.dumps(event)) # del event # except: # e = sys.exc_info()[1] # print('ERROR_noface_respond') # print(e) # logger.warning("Failed to send event to master: %s" % e) del self.image_in_buf[0] self.ccount+=1 time.sleep(0.001) # self.last_decoder_message = time.time() else: pass
def run_killoran(killoran=True): """Runs the GFP comparative tests on the Killoran (aka AM-VAE) optimization algorithm""" TRAIN_SIZE = 5000 train_size_str = "%ik" % (TRAIN_SIZE / 1000) for i in range(3): RANDOM_STATE = i + 1 print(RANDOM_STATE) num_models = [1, 5, 20][i] X_train, _, _ = util.get_experimental_X_y(random_state=RANDOM_STATE, train_size=TRAIN_SIZE) LD = 20 L = X_train.shape[1] vae_suffix = '_%s_%i' % (train_size_str, RANDOM_STATE) ground_truth = gfp_gp.SequenceGP(load=True, load_prefix="data/gfp_gp") loss = losses.neg_log_likelihood keras.utils.get_custom_objects().update({"neg_log_likelihood": loss}) oracle_suffix = '_%s_%i_%i' % (train_size_str, num_models, RANDOM_STATE) sess = tf.Session(graph=tf.get_default_graph()) K.set_session(sess) vae = util.build_vae(latent_dim=20, n_tokens=20, seq_length=X_train.shape[1], enc1_units=50) vae.encoder_.load_weights("../models/vae_0_encoder_weights%s.h5" % vae_suffix) vae.decoder_.load_weights("../models/vae_0_decoder_weights%s.h5" % vae_suffix) vae.vae_.load_weights("../models/vae_0_vae_weights%s.h5" % vae_suffix) oracles = [ keras.models.load_model("../models/oracle_%i%s.h5" % (i, oracle_suffix)) for i in range(num_models) ] if not killoran: results, test_max = optimization_algs.killoran_opt(X_train, vae, oracles, ground_truth, steps=30000, epsilon1=1e-5, epsilon2=1., noise_std=1e-5, LD=20, verbose=False, adam=False) np.save( "../results/mala_results_%s_%i.npy" % (train_size_str, RANDOM_STATE), results) suffix = "_%s_%i" % (train_size_str, RANDOM_STATE) with open('results/%s_max%s.json' % ('mala', suffix), 'w') as outfile: json.dump(test_max, outfile) else: results, test_max = optimization_algs.killoran_opt(X_train, vae, oracles, ground_truth, steps=10000, epsilon1=0., epsilon2=0.1, noise_std=1e-6, LD=20, verbose=False, adam=True) np.save( "../results/killoran_may_results_%s_%i.npy" % (train_size_str, RANDOM_STATE), results) suffix = "_%s_%i" % (train_size_str, RANDOM_STATE) with open('../results/%s_max%s.json' % ('killoran', suffix), 'w') as outfile: json.dump(test_max, outfile)
def model_to_estimator(keras_model=None, keras_model_path=None, custom_objects=None, model_dir=None, config=None): """Constructs an `Estimator` instance from given keras model. For usage example, please see @{$programmers_guide/estimators$creating_estimators_from_keras_models}. Args: keras_model: Keras model in memory. keras_model_path: Directory to a keras model on disk. custom_objects: Dictionary for custom objects. model_dir: Directory to save Estimator model parameters, graph and etc. config: Configuration object. Returns: An Estimator from given keras model. Raises: ValueError: if neither keras_model nor keras_model_path was given. ValueError: if both keras_model and keras_model_path was given. ValueError: if the keras_model_path is a GCS URI. ValueError: if keras_model has not been compiled. """ if (not keras_model) and (not keras_model_path): raise ValueError( 'Either `keras_model` or `keras_model_path` needs to be provided.') if keras_model and keras_model_path: raise ValueError( 'Please specity either `keras_model` or `keras_model_path`, ' 'but not both.') if not keras_model: if keras_model_path.startswith( 'gs://') or 'storage.googleapis.com' in keras_model_path: raise ValueError( '%s is not a local path. Please copy the model locally first.' % keras_model_path) logging.info('Loading models from %s', keras_model_path) keras_model = models.load_model(keras_model_path) else: logging.info('Using the Keras model provided.') keras_model = keras_model if not hasattr(keras_model, 'optimizer') or not keras_model.optimizer: raise ValueError( 'The given keras model has not been compiled yet. Please compile first ' 'before calling `model_to_estimator`.') if isinstance(config, dict): config = run_config_lib.RunConfig(**config) keras_model_fn = _create_keras_model_fn(keras_model, custom_objects) estimator = estimator_lib.Estimator( keras_model_fn, model_dir=model_dir, config=config) # Check if we need to call get_weights: if _any_variable_initialized(): keras_weights = keras_model.get_weights() # Warn if config passed to estimator tries to update GPUOptions. If a # session has already been created, the GPUOptions passed to the first # session sticks. if estimator._session_config.HasField('gpu_options'): logging.warning( 'The Keras backend session has already been set. ' 'The _session_config passed to model_to_estimator will not be used.') else: # Pass the config into keras backend's default session. sess = session.Session(config=estimator._session_config) K.set_session(sess) keras_weights = None if keras_model._is_graph_network: # TODO(yifeif): move checkpoint initialization to scaffold.init_fn _save_first_checkpoint(keras_model, estimator, custom_objects, keras_weights) elif keras_model.built: logging.warning('You are creating an Estimator from a Keras model ' 'manually subclassed from `Model`, that was ' 'already called on some inputs (and thus already had ' 'weights). We are currently unable to preserve ' 'the model\'s state (its weights) ' 'as part of the estimator ' 'in this case. Be warned that the estimator ' 'has been created using ' 'a freshly initialized version of your model.\n' 'Note that this doesn\'t affect the state of the ' 'model instance you passed as `keras_model` argument.') return estimator
def index(): if request.method=='GET': return render_template('donghwi.html') if request.method=='POST': real = time.strftime('%m-%d', time.localtime(time.time())) file = request.files['image_uploads'] hit=float(request.form['heat']) files = '{}'.format(file) l = [] m = [] for i in range(len(files)): l.append(files[i]) for j in range(len(l)): if l[j] == '\'': aa = l.index('\'') l.remove('\'') m.append(aa) elif l[j] == '.': bb = l.index('.') l.remove('.') m.append(bb) original_name = files[m[0] + 1:m[1] + 1] print(original_name) original_Image=real+original_name+'.jpg' img = image.load_img(file, target_size=(model.input.shape[1], model.input.shape[2])) img = np.array(img) asdf=os.getcwd() cv2.imwrite(asdf+'/static/save_img/{}'.format(original_Image), img) original_Path='/static/save_img/'+original_Image img_tensor = img.reshape((1, model.input.shape[1], model.input.shape[2], 3)) img_tensor = img_tensor / 255 global sess global graph with graph.as_default(): set_session(sess) prediction = model.predict(img_tensor) y_pred = np.argmax(prediction) # Get_Image_Predic hypothesis = model.output[:, y_pred] Last_conv_Layer = model.get_layer('conv5_block16_2_conv') # Get Last Layer Gradients = K.gradients(hypothesis, Last_conv_Layer.output)[0] # Get y_predic gradi from Last_Layer pooled_grad = K.mean(Gradients, axis=(0, 1, 2)) # Get gradients Average with graph.as_default(): set_session(sess) iterate = K.function([model.input], [pooled_grad, Last_conv_Layer.output[0]]) # Get output pooled_grad_value, conv_layer_value = iterate([img_tensor]) # input to original image-> get 2 numpy for i in range(pooled_grad.shape[0]): conv_layer_value[:, :, i] *= pooled_grad_value[i] # important feature square heatmap = np.mean(conv_layer_value, axis=-1) # average heatmap = np.maximum(heatmap, 0) heatmap /= np.max(heatmap) imag = image.load_img(file, target_size=(model.input.shape[1], model.input.shape[2])) imag=np.array(imag) heatmap = cv2.resize(heatmap, (imag.shape[1], imag.shape[0])) heatmap = np.uint8(255 * heatmap) heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) Finist = heatmap * hit + imag heat_name='Heat'+real+original_Image heat_path='/static/save_img/'+heat_name cv2.imwrite(asdf+'/static/save_img/{}'.format(heat_name), Finist) if y_pred==0: result='CNV' elif y_pred==1: result='DME' elif y_pred==2: result='DRUSEN' elif y_pred==3: result='NORMAL' cnv=round(prediction[0][0],4) dme=round(prediction[0][1],4) dru=round(prediction[0][2],4) nor=round(prediction[0][3],4) return render_template('donghwi.html',send=heat_path,gor=original_Path,price=result,cnv=cnv,dme=dme,dru=dru,nor=nor)
def crop_and_classify_subimages(): if request.method == 'POST': f = request.files['file'] basepath = os.path.dirname(__file__) file_path = os.path.join(basepath, 'uploads', secure_filename(f.filename)) f.save(file_path) # Preprocess the image first im_gray = cv2.cvtColor(f, cv2.COLOR_BGR2GRAY) # check that f is the image!!!! im_blur = cv2.GaussianBlur(im_gray, (5, 5), 0) # Calculate median third median_array = image.img_to_array(im_blur) median_array = median_array.ravel() median = np.median(median_array) median_third = median * 0.66 # Threshold the image ret, im_th = cv2.threshold(im_gray, median_third, 255, cv2.THRESH_BINARY_INV) image, ctrs, hier = cv2.findContours(im_th.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # Get rectangles contains each contour rects = [cv2.boundinRect(ctr) for ctr in ctrs] indiv_boxes = [] count = 0 for rect in rects: number = [] if rect[2] + rect[3] < 80: continue # rect object gives tuple with x1, y1, height, width count = count + 1 area = rect[2] * rect[3] border = b_size(area) # Draw the rectangles cv2.rectangle(im, (rect[0], rect[1]), (rect[0] + rect[2], rect[1] + rect[3]), (0, 255, 0), 3) # Cropping each bounding box and storing the image inside into roi roi = im_th[int(rect[1]):int(rect[1] + rect[3]), int(rect[0]):int(rect[0] + rect[2])] # Make number thicker so it becomes easier to recognize kernel = np.ones((5, 5), np.uint8) dilated_im = cv2.dilate(roi, kernel, iterations=1) # Making roi into an image so that we can resize img_jpg = Image.fromarray(dilated_im) # Adding a border to the image img_with_border = ImageOps.expand(img_jpg, border=border, fill=0) #15 # Setting the size for resizing size = (28, 28) # Padding and resizing the image (which is one rectange) new_img = pad_and_resize(img_with_border, size=size) # Turning the resized image back into an array value = img_to_array(new_img) # print(list(value)) y = np.expand_dims(value, axis=0) ################## with g.as_default(): set_session(sess) predictions = model.predict(y) number.append(np.argmax(predictions)) number.append(rect) #print('prediction: {}'.format(np.argmax(predictions))) indiv_boxes.append(number) return (indiv_boxes)
def predict(): print("STRATING PreDICT") start_date = date(2017, 1, 1) end_date = date(2018, 1, 1) features_list = ['open', 'high', 'low', 'close'] tickers_list = [ 'AAPL', 'ATVI', 'CMCSA', 'COST', 'CSX', 'DISH', 'EA', 'EBAY', 'FB', 'GOOGL', 'HAS', 'ILMN', 'INTC', 'MAR', 'REGN', 'SBUX' ] predictor_list = ['lstm', 'cnn', 'dense'] batch_norm_list = [False, True] window_length_list = [5, 10, 15, 20] for predictor_type in predictor_list: for use_batch_norm in batch_norm_list: for window_length in window_length_list: # setup environment num_training_time = 200 env = PortfolioEnv(start_date, end_date, window_length, tickers_list, features_list, trading_cost=0.00, batch_size=num_training_time) # variable_scope ="ddpg" action_dim = env.action_space.shape[0] actor_noise = OrnsteinUhlenbeckActionNoise( mu=np.zeros(action_dim)) model_save_path = get_model_path(window_length, predictor_type, use_batch_norm) summary_path = get_result_path(window_length, predictor_type, use_batch_norm) print("MODEL :", model_save_path) tf.reset_default_graph() sess = tf.Session() with sess.as_default(): # Missing this was the source of one of the most challenging an insidious bugs that I've ever encountered. # it was impossible to save the model. K.set_session(sess) ddpg_model = DDPG2(env, sess, actor_noise, action_dim=action_dim, obs_normalizer="history", predictor_type=predictor_type, use_batch_norm=use_batch_norm, model_save_path=model_save_path, summary_path=summary_path) ddpg_model.initialize(load_weights=True) observation = env.reset() done = False while not done: action = ddpg_model.predict_single(observation) observation, reward, done, infos = env.step(action) print( datetime.fromtimestamp( infos['date']).strftime('%Y-%m-%d'), infos['portfolio_change'] / infos['y_return'], infos['portfolio_value'], infos['market_value'], infos['weight_cash'], infos['weight_AAPL']) df = env.df_info() print(df.iloc[-1, :]) print(df.describe()) sess.close()
def initialize_vars(sess): sess.run(tf.local_variables_initializer()) sess.run(tf.global_variables_initializer()) sess.run(tf.tables_initializer()) K.set_session(sess)
def model_to_estimator(keras_model=None, keras_model_path=None, custom_objects=None, model_dir=None, config=None, checkpoint_format=None): # LINT.ThenChange(//tensorflow/python/keras/estimator/__init__.py) """Constructs an `Estimator` instance from given keras model. For usage example, please see: [Creating estimators from Keras Models](https://tensorflow.org/guide/estimators#model_to_estimator). Args: keras_model: A compiled Keras model object. This argument is mutually exclusive with `keras_model_path`. keras_model_path: Path to a compiled Keras model saved on disk, in HDF5 format, which can be generated with the `save()` method of a Keras model. This argument is mutually exclusive with `keras_model`. custom_objects: Dictionary for custom objects. model_dir: Directory to save `Estimator` model parameters, graph, summary files for TensorBoard, etc. config: `RunConfig` to config `Estimator`. checkpoint_format: Sets the format of the checkpoint saved by the estimator when training. May be `saver` or `checkpoint`, depending on whether to save checkpoints from `tf.compat.v1.train.Saver` or `tf.train.Checkpoint`. The default is `checkpoint`. Estimators use name-based `tf.train.Saver` checkpoints, while Keras models use object-based checkpoints from `tf.train.Checkpoint`. Currently, saving object-based checkpoints from `model_to_estimator` is only supported by Functional and Sequential models. Returns: An Estimator from given keras model. Raises: ValueError: if neither keras_model nor keras_model_path was given. ValueError: if both keras_model and keras_model_path was given. ValueError: if the keras_model_path is a GCS URI. ValueError: if keras_model has not been compiled. ValueError: if an invalid checkpoint_format was given. """ if not (keras_model or keras_model_path): raise ValueError( 'Either `keras_model` or `keras_model_path` needs to be provided.') if keras_model and keras_model_path: raise ValueError( 'Please specity either `keras_model` or `keras_model_path`, ' 'but not both.') config = estimator_lib.maybe_overwrite_model_dir_and_session_config( config, model_dir) if not keras_model: if keras_model_path.startswith( 'gs://') or 'storage.googleapis.com' in keras_model_path: keras_model_path = _get_file_from_google_storage( keras_model_path, config.model_dir) logging.info('Loading models from %s', keras_model_path) keras_model = models.load_model(keras_model_path) else: logging.info('Using the Keras model provided.') keras_model = keras_model if checkpoint_format is None or checkpoint_format == 'checkpoint': if not (keras_model._is_graph_network or isinstance(keras_model, models.Sequential)): raise ValueError( 'Object-based checkpoints are currently not supported ' 'with subclassed models.') save_object_ckpt = True elif checkpoint_format == 'saver': save_object_ckpt = False else: raise ValueError( 'Checkpoint format must be one of "checkpoint" or "saver". Got {}'. format(checkpoint_format)) if not hasattr(keras_model, 'optimizer') or not keras_model.optimizer: raise ValueError('The given keras model has not been compiled yet. ' 'Please compile the model with `model.compile()` ' 'before calling `model_to_estimator()`.') keras_model_fn = _create_keras_model_fn(keras_model, custom_objects, save_object_ckpt) if _any_weight_initialized(keras_model): # Warn if config passed to estimator tries to update GPUOptions. If a # session has already been created, the GPUOptions passed to the first # session sticks. if config.session_config.HasField('gpu_options'): logging.warning( 'The Keras backend session has already been set. ' 'The _session_config passed to model_to_estimator will not be used.' ) else: # Pass the config into keras backend's default session. sess = session.Session(config=config.session_config) K.set_session(sess) warm_start_path = None if keras_model._is_graph_network: warm_start_path = _save_first_checkpoint(keras_model, custom_objects, config, save_object_ckpt) elif keras_model.built: logging.warning( 'You are creating an Estimator from a Keras model manually ' 'subclassed from `Model`, that was already called on some ' 'inputs (and thus already had weights). We are currently ' 'unable to preserve the model\'s state (its weights) as ' 'part of the estimator in this case. Be warned that the ' 'estimator has been created using a freshly initialized ' 'version of your model.\n' 'Note that this doesn\'t affect the state of the model ' 'instance you passed as `keras_model` argument.') estimator = estimator_lib.Estimator(keras_model_fn, config=config, warm_start_from=warm_start_path) return estimator
from tensorflow.keras.models import model_from_json from tensorflow.python.keras.backend import set_session import numpy as np import tensorflow as tf config = tf.compat.v1.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.15 session = tf.compat.v1.Session(config=config) set_session(session) class FacialExpressionModel(object): EMOTIONS_LIST = [ "Angry", "Disgust", "Fear", "Happy", "Neutral", "Sad", "Surprise" ] def __init__(self, model_json_file, model_weights_file): # load model from JSON file with open(model_json_file, "r") as json_file: loaded_model_json = json_file.read() self.loaded_model = model_from_json(loaded_model_json) # load weights into the new model self.loaded_model.load_weights(model_weights_file) #self.loaded_model.compile() #self.loaded_model._make_predict_function() def predict_emotion(self, img): global session
def __init__(self, hyper_parameters): """ 模型初始化 :param hyper_parameters:json, json["model"] and json["embedding"] """ self.len_max = hyper_parameters.get("len_max", 50) # 文本最大长度 self.embed_size = hyper_parameters.get("embed_size", 300) # 嵌入层尺寸 self.trainable = hyper_parameters.get( "trainable", False) # 是否微调, 例如静态词向量、动态词向量、微调bert层等, random也可以 self.embedding_type = hyper_parameters.get( "embedding_type", "word2vec") # 词嵌入方式,可以选择"xlnet"、"bert"、"gpt-2"、"word2vec"或者"None" self.gpu_memory_fraction = hyper_parameters.get( "gpu_memory_fraction", None) # gpu使用率, 默认不配置 self.hyper_parameters = hyper_parameters hyper_parameters_model = hyper_parameters["model"] self.label = hyper_parameters_model.get("label", 2) # 类型 self.batch_size = hyper_parameters_model.get("batch_size", 32) # 批向量 self.filters = hyper_parameters_model.get("filters", [3, 4, 5]) # 卷积核大小 self.filters_num = hyper_parameters_model.get("filters_num", 300) # 核数 self.channel_size = hyper_parameters_model.get("channel_size", 1) # 通道数 self.dropout = hyper_parameters_model.get("dropout", 0.5) # dropout层系数,舍弃 self.decay_step = hyper_parameters_model.get("decay_step", 100) # 衰减步数 self.decay_rate = hyper_parameters_model.get("decay_rate", 0.9) # 衰减系数 self.epochs = hyper_parameters_model.get("epochs", 20) # 训练轮次 self.vocab_size = hyper_parameters_model.get("vocab_size", 20000) # 字典词典大小 self.lr = hyper_parameters_model.get("lr", 1e-3) # 学习率 self.l2 = hyper_parameters_model.get("l2", 1e-6) # l2正则化系数 self.activate_rnn = hyper_parameters_model.get( "activate_rnn", "tanh") # RNN激活函数, tanh, relu, signmod self.activate_classify = hyper_parameters_model.get( "activate_classify", "softmax") # 分类激活函数,softmax或者signmod self.loss = hyper_parameters_model.get( "loss", "categorical_crossentropy" ) # 损失函数, mse, categorical_crossentropy, sparse_categorical_crossentropy, binary_crossentropy等 self.metrics = hyper_parameters_model.get( "metrics", "accuracy" ) # acc, binary_accuracy, categorical_accuracy, sparse_categorical_accuracy, sparse_top_k_categorical_accuracy self.is_training = hyper_parameters_model.get( "is_training", False) # 是否训练, 保存时候为Flase,方便预测 self.patience = hyper_parameters_model.get("patience", 3) # 早停, 2-3就可以了 self.optimizer_name = hyper_parameters_model.get( "optimizer_name", "RAdam,Lookahead") # 早停, 2-3就可以了 self.path_model_dir = hyper_parameters_model.get( "path_model_dir", path_model_dir) # 模型目录 self.path_fineture = os.path.join( self.path_model_dir, "embedding_trainable.h5") # embedding层保存地址, 例如静态词向量、动态词向量、微调bert层等 self.path_model = os.path.join(self.path_model_dir, "model.h5") # 模型weight绝对地址 self.path_hyper_parameters = os.path.join(self.path_model_dir, "params.json") # 超参数保存绝对地址 self.path_model_l2i_i2l = os.path.join(self.path_model_dir, "l2i_i2l.json") # 模型字典保存绝对地址 self.path_model_graph = os.path.join(self.path_model_dir, "graph.json") # 模型图结构绝对地址 if self.gpu_memory_fraction: # keras, tensorflow控制GPU使用率等 import tensorflow.python.keras.backend as K import tensorflow as tf config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = self.gpu_memory_fraction # config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) self.create_model(hyper_parameters) # 模型初始化 if self.is_training: # 是否是训练阶段, 与预测区分开 self.create_compile() self.save_graph() # 保存图结构
def testDiversVaries(): #tf.keras.backend.clear_session() #tf.reset_default_graph() #K.set_learning_phase(0) sess = tf.Session() #graph = tf.get_default_graph() #keras.backend.set_session(sess) # IMPORTANT: models have to be loaded AFTER SETTING THE SESSION for keras! # Otherwise, their weights will be unavailable in the threads after the session there has been set set_session(sess) original_model = resnet_trained(n_retrain_layers=20) # Cela va charger un tf.keras model base_model = resnet_trained(20) predictions = Dense(25, activation='softmax')(base_model.output) net_finetuned = Model(inputs=base_model.input, outputs=predictions) net_finetuned.predict(np.random.rand(1,224,224,3)) trainable_layers_name = [] for original_layer in original_model.layers: if original_layer.trainable: trainable_layers_name += [original_layer.name] #C:\media\gonthier\HDD2\output_exp\rasta_models\resnet_2017_7_31-19_9_45 path_to_model = os.path.join(os.sep,'media','gonthier','HDD2','output_exp','rasta_models','resnet_2017_7_31-19_9_45','model.h5') constrNet = 'LResNet50' # For Lecoutre ResNet50 version model_name = 'Lecoutre2017' input_name_lucid = 'input_1' tf.keras.backend.set_image_data_format('channels_last') net_finetuned.load_weights(path_to_model,by_name=True) net_finetuned.build((224,224,3)) net_finetuned.summary() net_finetuned.predict(np.random.rand(1,224,224,3)) #net_finetuned = keras.models.load_model(path_to_model,compile=True) #net_finetuned = load_model(path_to_model,compile=True) number_of_trainable_layer = 20 # #list_layer_index_to_print = [] #for layer in model.layers: # trainable_l = layer.trainable # name_l = layer.name # if trainable_l and 'res' in name_l: # print(name_l,trainable_l) # num_features = tf.shape(layer.bias).eval(session=sess)[0] # list_layer_index_to_print += [name_l,np.arange(0,num_features)] # #for layer in original_model.layers: # print(layer) # trainable_l = layer.trainable # name_l = layer.name # if trainable_l and 'res' in name_l: # print(name_l,trainable_l) # num_features = tf.shape(layer.bias).eval(session=sess)[0] # list_layer_index_to_print += [name_l,np.arange(0,num_features)] #list_weights,list_name_layers = get_weights_and_name_layers_forPurekerasModel(original_model) list_weights,list_name_layers = CompNet_FT_lucidIm.get_weights_and_name_layers(original_model) dict_layers_relative_diff,dict_layers_argsort = CompNet_FT_lucidIm.get_gap_between_weights(list_name_layers,\ list_weights,net_finetuned) layer_considered_for_print_im = [] for layer in net_finetuned.layers: trainable_l = layer.trainable name_l = layer.name print(name_l,trainable_l) if trainable_l and (name_l in trainable_layers_name): layer_considered_for_print_im += [name_l] num_top = 3 list_layer_index_to_print_base_model = [] list_layer_index_to_print = [] #print(layer_considered_for_print_im) for key in dict_layers_argsort.keys(): #print(key) if not(key in layer_considered_for_print_im): continue for k in range(num_top): topk = dict_layers_argsort[key][k] list_layer_index_to_print += [[key,topk]] list_layer_index_to_print_base_model += [[key,topk]] print('list_layer_index_to_print',list_layer_index_to_print) #dict_list_layer_index_to_print_base_model[model_name+suffix] = list_layer_index_to_print_base_model #dict_layers_relative_diff,dict_layers_argsort = CompNet_FT_lucidIm.get_gap_between_weights(list_name_layers,\ # list_weights,model) # For the fine-tuned model !!! path_lucid_model = os.path.join(os.sep,'media','gonthier','HDD2','output_exp','Covdata','Lucid_model') path = path_lucid_model if path=='': os.makedirs('./model', exist_ok=True) path ='model' else: os.makedirs(path, exist_ok=True) frozen_graph = lucid_utils.freeze_session(sess, output_names=[out.op.name for out in net_finetuned.outputs]) name_pb = 'tf_graph_'+constrNet+model_name+'.pb' #nodes_tab = [n.name for n in tf.get_default_graph().as_graph_def().node] #print(nodes_tab) tf.io.write_graph(frozen_graph,logdir= path,name= name_pb, as_text=False) if platform.system()=='Windows': output_path = os.path.join('CompModifModel',constrNet) else: output_path = os.path.join(os.sep,'media','gonthier','HDD2','output_exp','Covdata','CompModifModel',constrNet) pathlib.Path(output_path).mkdir(parents=True, exist_ok=True) matplotlib.use('Agg') output_path_with_model = os.path.join(output_path,model_name) pathlib.Path(output_path_with_model).mkdir(parents=True, exist_ok=True) # global sess # global graph # with graph.as_default(): # set_session(sess) # net_finetuned.predict(np.random.rand(1,224,224,3)) net_finetuned.predict(np.random.rand(1,224,224,3)) lucid_utils.print_images(model_path=path_lucid_model+'/'+name_pb,list_layer_index_to_print=list_layer_index_to_print\ ,path_output=output_path_with_model,prexif_name=model_name,input_name=input_name_lucid,Net=constrNet) # For the original one !!! original_model.predict(np.random.rand(1,224,224,3)) #sess = keras.backend.get_session() #sess.run() frozen_graph = lucid_utils.freeze_session(sess, output_names=[out.op.name for out in original_model.outputs]) name_pb = 'tf_graph_'+constrNet+'PretrainedImageNet.pb' tf.io.write_graph(frozen_graph,logdir= path,name= name_pb, as_text=False) lucid_utils.print_images(model_path=path_lucid_model+'/'+name_pb,list_layer_index_to_print=list_layer_index_to_print\ ,path_output=output_path_with_model,prexif_name=model_name,input_name=input_name_lucid,Net=constrNet)
try: tf.gfile.DeleteRecursively(output_dir) except: pass mkdirs(output_dir) print('***** Model output directory: {} *****'.format(output_dir)) # initializing graph and session graph = tf.get_default_graph() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.visible_device_list = gpu_id config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 sess = tf.Session(config=config) set_session(sess) # data loading and preprocessing from eraser from eraserbenchmark.rationale_benchmark.utils import load_datasets, load_documents from eraserbenchmark.eraser_utils import extract_doc_ids_from_annotations from itertools import chain if dataset == 'movies': label_list = ['POS', 'NEG'] elif dataset == 'multirc': label_list = ['True', 'False'] elif dataset == 'fever': label_list = ['SUPPORTS', 'REFUTES'] train, val, test = load_datasets(data_dir) if train_on_portion != 0:
def model_to_estimator(keras_model=None, keras_model_path=None, custom_objects=None, model_dir=None, config=None): """Constructs an `Estimator` instance from given keras model. For usage example, please see @{$programmers_guide/estimators$creating_estimators_from_keras_models}. Args: keras_model: A compiled Keras model object. This argument is mutually exclusive with `keras_model_path`. keras_model_path: Path to a compiled Keras model saved on disk, in HDF5 format, which can be generated with the `save()` method of a Keras model. This argument is mutually exclusive with `keras_model`. custom_objects: Dictionary for custom objects. model_dir: Directory to save Estimator model parameters, graph, summary files for TensorBoard, etc. config: Configuration object. Returns: An Estimator from given keras model. Raises: ValueError: if neither keras_model nor keras_model_path was given. ValueError: if both keras_model and keras_model_path was given. ValueError: if the keras_model_path is a GCS URI. ValueError: if keras_model has not been compiled. """ if not (keras_model or keras_model_path): raise ValueError( 'Either `keras_model` or `keras_model_path` needs to be provided.') if keras_model and keras_model_path: raise ValueError( 'Please specity either `keras_model` or `keras_model_path`, ' 'but not both.') if not keras_model: if keras_model_path.startswith( 'gs://') or 'storage.googleapis.com' in keras_model_path: raise ValueError( '%s is not a local path. Please copy the model locally first.' % keras_model_path) logging.info('Loading models from %s', keras_model_path) keras_model = models.load_model(keras_model_path) else: logging.info('Using the Keras model provided.') keras_model = keras_model if not hasattr(keras_model, 'optimizer') or not keras_model.optimizer: raise ValueError( 'The given keras model has not been compiled yet. ' 'Please compile the model with `model.compile()` ' 'before calling `model_to_estimator()`.') if isinstance(config, dict): config = run_config_lib.RunConfig(**config) keras_model_fn = _create_keras_model_fn(keras_model, custom_objects) estimator = estimator_lib.Estimator( keras_model_fn, model_dir=model_dir, config=config) # Check if we need to call get_weights: if _any_variable_initialized(): keras_weights = keras_model.get_weights() # Warn if config passed to estimator tries to update GPUOptions. If a # session has already been created, the GPUOptions passed to the first # session sticks. if estimator._session_config.HasField('gpu_options'): logging.warning( 'The Keras backend session has already been set. ' 'The _session_config passed to model_to_estimator will not be used.') else: # Pass the config into keras backend's default session. sess = session.Session(config=estimator._session_config) K.set_session(sess) keras_weights = None if keras_model._is_graph_network: # TODO(yifeif): move checkpoint initialization to scaffold.init_fn _save_first_checkpoint(keras_model, estimator, custom_objects, keras_weights) elif keras_model.built: logging.warning('You are creating an Estimator from a Keras model ' 'manually subclassed from `Model`, that was ' 'already called on some inputs (and thus already had ' 'weights). We are currently unable to preserve ' 'the model\'s state (its weights) ' 'as part of the estimator ' 'in this case. Be warned that the estimator ' 'has been created using ' 'a freshly initialized version of your model.\n' 'Note that this doesn\'t affect the state of the ' 'model instance you passed as `keras_model` argument.') return estimator
def predict_emotion(self, img): global session set_session(session) self.preds = self.loaded_model.predict(img) return FacialExpressionModel.EMOTIONS_LIST[np.argmax(self.preds)]
pre_processing_time = pre_end_time - start_time # print(pre_processing_time) fp.write(str(pre_processing_time) + '\n') # TensorFlow part from tensorflow import keras from tensorflow.python.keras import regularizers from tensorflow.python.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten from tensorflow.python.keras.models import Sequential from keras.models import load_model import tensorflow as tf from tensorflow.python.keras.backend import set_session config = tf.ConfigProto() config.gpu_options.allow_growth = True set_session(tf.Session(config=config)) input_shape = (w, h, c) learning_rate = 0.0001 regularization_rate = 0.0001 category_count = 13 + 1 train_path = r'D:\Projects\IoT_recognition\20181205\Keras\TRAIN/' model = load_model(train_path + '/model.h5') ml_start_time = time.time() # for img in sub_images: result = model.predict(np.asarray(sub_images, np.float32)) # cat = model.predict_classes(np.asarray(sub_images, np.float32)) cat = get_max_and_confidence(result) cat_list = [ '16_pin_chip',
# SPDX-License-Identifier: Apache-2.0 import gym from delayed_env import DelayedEnv import wandb import warnings # from keras import backend as K from tensorflow.python.keras import backend as K import tensorflow as tf config = tf.compat.v1.ConfigProto(allow_soft_placement=True) config.gpu_options.per_process_gpu_memory_fraction = 0.9 config.gpu_options.allow_growth = True sess = tf.compat.v1.Session(config=config) K.set_session(sess) def init_main(): hyperparameter_defaults = dict( is_delayed_agent=False, double_q=True, delay_value=5, epsilon_decay= 0.999, # Cartpole: 0.999, Acrobot: 0.9999, MountainCar: 0.99999 epsilon_min=0.001, #0.001 learning_rate=0.005, # Cartpole & Acrobot: 0.005, #mountainCar: 0.0001 seed=1, epsilon=1.0, use_m_step_reward=False, use_latest_reward=False,
def create_model(self, n_dim, r): # load inputs X, _, _ = self.inputs K.set_session(self.sess) with tf.compat.v1.name_scope('generator'): x = X L = self.layers n_filters = [ 128, 256, 512, 512, 512, 512, 512, 512] n_blocks = [ 128, 64, 32, 16, 8] n_filtersizes = [65, 33, 17, 9, 9, 9, 9, 9, 9] downsampling_l = [] print('building model...') def _make_normalizer(x_in, n_filters, n_block): """applies an lstm layer on top of x_in""" x_shape = tf.shape(input=x_in) n_steps = x_shape[1] / int(n_block) # will be 32 at training # first, apply standard conv layer to reduce the dimension # input of (-1, 4096, 128) becomes (-1, 32, 128) # input of (-1, 512, 512) becomes (-1, 32, 512) x_in_down = (MaxPooling1D(pool_size=int(n_block), padding='valid'))(x_in) # pooling to reduce dimension x_shape = tf.shape(input=x_in_down) x_rnn = LSTM(units = n_filters, return_sequences = True)(x_in_down) # output: (-1, n_steps, n_filters) return x_rnn def _apply_normalizer(x_in, x_norm, n_filters, n_block): x_shape = tf.shape(input=x_in) n_steps = x_shape[1] / int(n_block) # will be 32 at training # reshape input into blocks x_in = tf.reshape(x_in, shape=(-1, n_steps, int(n_block), n_filters)) x_norm = tf.reshape(x_norm, shape=(-1, n_steps, 1, n_filters)) # multiply x_out = x_norm * x_in # return to original shape x_out = tf.reshape(x_out, shape=x_shape) return x_out # downsampling layers for l, nf, fs in zip(list(range(L)), n_filters, n_filtersizes): with tf.compat.v1.name_scope('downsc_conv%d' % l): x = (Conv1D(filters=nf, kernel_size=fs, dilation_rate = DRATE, activation=None, padding='same', kernel_initializer=Orthogonal()))(x) x = (MaxPooling1D(pool_size=self.pool_size, padding='valid', strides=self.strides))(x) x = LeakyReLU(0.2)(x) # create and apply the normalizer nb = 128 / (2**l) params_before = np.sum([np.prod(v.get_shape().as_list()) for v in tf.compat.v1.trainable_variables()]) x_norm = _make_normalizer(x, nf, nb) params_after = np.sum([np.prod(v.get_shape().as_list()) for v in tf.compat.v1.trainable_variables()]) x = _apply_normalizer(x, x_norm, nf, nb) print('D-Block: ', x.get_shape()) downsampling_l.append(x) # bottleneck layer with tf.compat.v1.name_scope('bottleneck_conv'): x = (Conv1D(filters=n_filters[-1], kernel_size=n_filtersizes[-1], dilation_rate = DRATE, activation=None, padding='same', kernel_initializer=Orthogonal))(x) x = (MaxPooling1D(pool_size=self.pool_size, padding='valid', strides=self.strides))(x) x = Dropout(rate=0.5)(x) x = LeakyReLU(0.2)(x) # create and apply the normalizer nb = 128 / (2**L) x_norm = _make_normalizer(x, n_filters[-1], nb) x = _apply_normalizer(x, x_norm, n_filters[-1], nb) # upsampling layers for l, nf, fs, l_in in reversed(list(zip(list(range(L)), n_filters, n_filtersizes, downsampling_l))): with tf.compat.v1.name_scope('upsc_conv%d' % l): # (-1, n/2, 2f) x = (Conv1D(filters=2*nf, kernel_size=fs, dilation_rate = DRATE, activation=None, padding='same', kernel_initializer=Orthogonal()))(x) x = Dropout(rate=0.5)(x) x = Activation('relu')(x) # (-1, n, f) x = SubPixel1D(x, r=2) # create and apply the normalizer x_norm = _make_normalizer(x, nf, nb) x = _apply_normalizer(x, x_norm, nf, nb) # (-1, n, 2f) x = Concatenate()([x, l_in]) print('U-Block: ', x.get_shape()) # final conv layer with tf.compat.v1.name_scope('lastconv'): x = Conv1D(filters=2, kernel_size=9, activation=None, padding='same', kernel_initializer=RandomNormal(stddev=1e-3))(x) x = SubPixel1D(x, r=2) g = Add()([x, X]) return g
def get_img_features(self, img): img = self.pre_input(img) with self.session.graph.as_default(): set_session(self.session) yhat = self.model.predict(img) return yhat
def model_to_estimator(keras_model=None, keras_model_path=None, custom_objects=None, model_dir=None, config=None): """Constructs an `Estimator` instance from given keras model. For usage example, please see: [Creating estimators from Keras Models](https://tensorflow.org/guide/estimators#model_to_estimator). Args: keras_model: A compiled Keras model object. This argument is mutually exclusive with `keras_model_path`. keras_model_path: Path to a compiled Keras model saved on disk, in HDF5 format, which can be generated with the `save()` method of a Keras model. This argument is mutually exclusive with `keras_model`. custom_objects: Dictionary for custom objects. model_dir: Directory to save `Estimator` model parameters, graph, summary files for TensorBoard, etc. config: `RunConfig` to config `Estimator`. Returns: An Estimator from given keras model. Raises: ValueError: if neither keras_model nor keras_model_path was given. ValueError: if both keras_model and keras_model_path was given. ValueError: if the keras_model_path is a GCS URI. ValueError: if keras_model has not been compiled. """ if not (keras_model or keras_model_path): raise ValueError( 'Either `keras_model` or `keras_model_path` needs to be provided.') if keras_model and keras_model_path: raise ValueError( 'Please specity either `keras_model` or `keras_model_path`, ' 'but not both.') if not keras_model: if keras_model_path.startswith( 'gs://') or 'storage.googleapis.com' in keras_model_path: raise ValueError( '%s is not a local path. Please copy the model locally first.' % keras_model_path) logging.info('Loading models from %s', keras_model_path) keras_model = models.load_model(keras_model_path) else: logging.info('Using the Keras model provided.') keras_model = keras_model if not hasattr(keras_model, 'optimizer') or not keras_model.optimizer: raise ValueError( 'The given keras model has not been compiled yet. ' 'Please compile the model with `model.compile()` ' 'before calling `model_to_estimator()`.') config = estimator_lib.maybe_overwrite_model_dir_and_session_config(config, model_dir) keras_model_fn = _create_keras_model_fn(keras_model, custom_objects) if _any_weight_initialized(keras_model): # Warn if config passed to estimator tries to update GPUOptions. If a # session has already been created, the GPUOptions passed to the first # session sticks. if config.session_config.HasField('gpu_options'): logging.warning( 'The Keras backend session has already been set. ' 'The _session_config passed to model_to_estimator will not be used.') else: # Pass the config into keras backend's default session. sess = session.Session(config=config.session_config) K.set_session(sess) warm_start_path = None if keras_model._is_graph_network: warm_start_path = _save_first_checkpoint(keras_model, custom_objects, config) elif keras_model.built: logging.warning('You are creating an Estimator from a Keras model manually ' 'subclassed from `Model`, that was already called on some ' 'inputs (and thus already had weights). We are currently ' 'unable to preserve the model\'s state (its weights) as ' 'part of the estimator in this case. Be warned that the ' 'estimator has been created using a freshly initialized ' 'version of your model.\n' 'Note that this doesn\'t affect the state of the model ' 'instance you passed as `keras_model` argument.') estimator = estimator_lib.Estimator(keras_model_fn, config=config, warm_start_from=warm_start_path) return estimator
def model_to_estimator(keras_model=None, keras_model_path=None, custom_objects=None, model_dir=None, config=None): """Constructs an `Estimator` instance from given keras model. For usage example, please see @{$guide/estimators$creating_estimators_from_keras_models}. Args: keras_model: A compiled Keras model object. This argument is mutually exclusive with `keras_model_path`. keras_model_path: Path to a compiled Keras model saved on disk, in HDF5 format, which can be generated with the `save()` method of a Keras model. This argument is mutually exclusive with `keras_model`. custom_objects: Dictionary for custom objects. model_dir: Directory to save `Estimator` model parameters, graph, summary files for TensorBoard, etc. config: `RunConfig` to config `Estimator`. Returns: An Estimator from given keras model. Raises: ValueError: if neither keras_model nor keras_model_path was given. ValueError: if both keras_model and keras_model_path was given. ValueError: if the keras_model_path is a GCS URI. ValueError: if keras_model has not been compiled. """ if not (keras_model or keras_model_path): raise ValueError( 'Either `keras_model` or `keras_model_path` needs to be provided.') if keras_model and keras_model_path: raise ValueError( 'Please specity either `keras_model` or `keras_model_path`, ' 'but not both.') if not keras_model: if keras_model_path.startswith( 'gs://') or 'storage.googleapis.com' in keras_model_path: raise ValueError( '%s is not a local path. Please copy the model locally first.' % keras_model_path) logging.info('Loading models from %s', keras_model_path) keras_model = models.load_model(keras_model_path) else: logging.info('Using the Keras model provided.') keras_model = keras_model if not hasattr(keras_model, 'optimizer') or not keras_model.optimizer: raise ValueError( 'The given keras model has not been compiled yet. ' 'Please compile the model with `model.compile()` ' 'before calling `model_to_estimator()`.') config = estimator_lib.maybe_overwrite_model_dir_and_session_config(config, model_dir) keras_model_fn = _create_keras_model_fn(keras_model, custom_objects) if _any_weight_initialized(keras_model): # Warn if config passed to estimator tries to update GPUOptions. If a # session has already been created, the GPUOptions passed to the first # session sticks. if config.session_config.HasField('gpu_options'): logging.warning( 'The Keras backend session has already been set. ' 'The _session_config passed to model_to_estimator will not be used.') else: # Pass the config into keras backend's default session. sess = session.Session(config=config.session_config) K.set_session(sess) warm_start_path = None if keras_model._is_graph_network: warm_start_path = _save_first_checkpoint(keras_model, custom_objects, config) elif keras_model.built: logging.warning('You are creating an Estimator from a Keras model manually ' 'subclassed from `Model`, that was already called on some ' 'inputs (and thus already had weights). We are currently ' 'unable to preserve the model\'s state (its weights) as ' 'part of the estimator in this case. Be warned that the ' 'estimator has been created using a freshly initialized ' 'version of your model.\n' 'Note that this doesn\'t affect the state of the model ' 'instance you passed as `keras_model` argument.') estimator = estimator_lib.Estimator(keras_model_fn, config=config, warm_start_from=warm_start_path) return estimator
def tied_session(): """ Tied the tensorflow Session and keras Session together """ from odin.config import get_session from tensorflow.python.keras.backend import set_session set_session(get_session())