def main(unused_argv): logging.set_verbosity(tf.logging.INFO) if not FLAGS.json_prediction_files_pattern: raise ValueError( "The flag --json_prediction_files_pattern must be specified.") if not FLAGS.csv_output_file: raise ValueError("The flag --csv_output_file must be specified.") logging.info("Looking for prediction files with pattern: %s", FLAGS.json_prediction_files_pattern) file_paths = gfile.Glob(FLAGS.json_prediction_files_pattern) logging.info("Found files: %s", file_paths) logging.info("Writing submission file to: %s", FLAGS.csv_output_file) with gfile.Open(FLAGS.csv_output_file, "w+") as output_file: output_file.write(get_csv_header()) for file_path in file_paths: logging.info("processing file: %s", file_path) with gfile.Open(file_path) as input_file: for line in input_file: json_data = json.loads(line) output_file.write(to_csv_row(json_data)) output_file.flush() logging.info("done")
def main(unused_argv): # Load the environment. env = json.loads(os.environ.get("TF_CONFIG", "{}")) # Load the cluster data from the environment. cluster_data = env.get("cluster", None) cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None # Load the task data from the environment. task_data = env.get("task", None) or {"type": "master", "index": 0} task = type("TaskSpec", (object,), task_data) # Logging the version. logging.set_verbosity(tf.logging.INFO) logging.info("%s: Tensorflow version: %s.", task_as_string(task), tf.__version__) # Dispatch to a master, a worker, or a parameter server. if not cluster or task.type == "master" or task.type == "worker": model = find_class_by_name(FLAGS.model, [frame_level_models, video_level_models])() reader = get_reader() model_exporter = export_model.ModelExporter( frame_features=FLAGS.frame_features, model=model, reader=reader) Trainer(cluster, task, FLAGS.train_dir, model, reader, model_exporter, FLAGS.log_device_placement, FLAGS.max_steps, FLAGS.export_model_steps).run(start_new_model=FLAGS.start_new_model) elif task.type == "ps": ParameterServer(cluster, task).run() else: raise ValueError("%s: Invalid task_type: %s." % (task_as_string(task), task.type))
def main(unused_argv): logging.set_verbosity(tf.logging.INFO) if FLAGS.input_model_tgz: if FLAGS.train_dir: raise ValueError("You cannot supply --train_dir if supplying " "--input_model_tgz") # Untar. if not os.path.exists(FLAGS.untar_model_dir): os.makedirs(FLAGS.untar_model_dir) tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir) FLAGS.train_dir = FLAGS.untar_model_dir flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json") if not os.path.exists(flags_dict_file): raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file) flags_dict = json.loads(open(flags_dict_file).read()) # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( flags_dict["feature_names"], flags_dict["feature_sizes"]) if flags_dict["frame_features"]: reader = readers.YT8MFrameFeatureReader(feature_names=feature_names, feature_sizes=feature_sizes) else: reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names, feature_sizes=feature_sizes) if FLAGS.output_file is "": raise ValueError("'output_file' was not specified. " "Unable to continue with inference.") if FLAGS.input_data_pattern is "": raise ValueError("'input_data_pattern' was not specified. " "Unable to continue with inference.") inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern, FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
def main(unused_argv): logging.set_verbosity(tf.logging.INFO) # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( FLAGS.feature_names, FLAGS.feature_sizes) if FLAGS.frame_features: reader = readers.YT8MFrameFeatureReader(feature_names=feature_names, feature_sizes=feature_sizes) else: reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names, feature_sizes=feature_sizes) if FLAGS.output_file is "": raise ValueError("'output_file' was not specified. " "Unable to continue with inference.") if FLAGS.input_data_pattern is "": raise ValueError("'input_data_pattern' was not specified. " "Unable to continue with inference.") inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern, FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
def main(unused_argv): # Load the environment. env = json.loads(os.environ.get("TF_CONFIG", "{}")) # Load the cluster data from the environment. cluster_data = env.get("cluster", None) cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None # Load the task data from the environment. task_data = env.get("task", None) or {"type": "master", "index": 0} task = type("TaskSpec", (object, ), task_data) # Logging the version. logging.set_verbosity(tf.logging.INFO) logging.info("%s: Tensorflow version: %s.", task_as_string(task), tf.__version__) # if os.path.exists(FLAGS.train_dir): # shutil.rmtree(FLAGS.train_dir) # subdirs = [x for x in os.listdir(FLAGS.Ensemble_Models) if os.path.isdir(x)] subdirs = [ os.path.join(FLAGS.Ensemble_Models, x) for x in FLAGS.model_path.split(',') ] flags_dict = [] save_flag = True for subdir in subdirs: model_flags_path = os.path.join( os.path.join(FLAGS.Ensemble_Models, subdir), "model_flags.json") print "Load model from " + model_flags_path + "\n" flags_dict.append( json.loads(file_io.FileIO(model_flags_path, mode="r").read())) # save model_flags.json to inference dictory if save_flag: if os.path.exists(FLAGS.train_dir) == False: os.mkdir(FLAGS.train_dir) shutil.copyfile(model_flags_path, os.path.join(FLAGS.train_dir, "model_flags.json")) save_flag = False models = [] model_nums = len(subdirs) for m in range(model_nums): model = get_params(flags_dict[m]) models.append(model) # Dispatch to a master, a worker, or a parameter server. if not cluster or task.type == "master" or task.type == "worker": # model = find_class_by_name(FLAGS.model, # [frame_level_models, video_level_models])() reader = get_reader() # model_exporter = export_model.ModelExporter( # frame_features=FLAGS.frame_features, # model=models, # reader=reader) model_exporter = None Trainer(cluster, task, FLAGS.train_dir, subdirs, models, reader, model_exporter, FLAGS.log_device_placement, FLAGS.max_steps, FLAGS.export_model_steps).run( start_new_model=FLAGS.start_new_model) elif task.type == "ps": ParameterServer(cluster, task).run() else: raise ValueError("%s: Invalid task_type: %s." % (task_as_string(task), task.type))
import time from datetime import datetime import os import subprocess import sys # Attempt to disable annoying logging messages import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import tensorflow as tf from tensorflow import logging logging.set_verbosity(logging.ERROR) os.system('cls') from keras.models import load_model from keras.preprocessing.text import text_to_word_sequence from keras.preprocessing.text import one_hot import h5py as h5 import pandas as pd import numpy as np from scapy.all import * from math import floor, log2 from _params import params ''' Plan of Action 1) Sniff and write to pcap file 2) Process pcap file using CICFlowMeter cmd program and save to csv
import os # Suppress some level of logs os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import tensorflow as tf import matplotlib.pyplot as plt import numpy as np from tensorflow import logging logging.set_verbosity(logging.FATAL) def weight_variable(shape, name=None): # initialize weighted variables. initial = tf.truncated_normal(shape, stddev=0.001) return tf.Variable(initial, name=name) def conv2d(x, W, strides=[1, 1, 1, 1], p='SAME', name=None): # set convolution layers. with tf.name_scope(name): assert isinstance(x, tf.Tensor) return tf.nn.conv2d(x, W, strides=strides, padding=p, name=name) def batch_norm(x): assert isinstance(x, tf.Tensor) # reduce dimension 1, 2, 3, which would produce batch mean and batch variance. mean, var = tf.nn.moments(x, axes=[1, 2, 3], keep_dims=True) return tf.nn.batch_normalization(x, mean, var, 0, 1, 1e-5)
def init_eval_environment(): """Prepares the dataset and datagenerators used to train a model.""" config = ConfigProto() config.gpu_options.allow_growth = True logging.set_verbosity( logging.ERROR) # Supress all kind of deprecation warnings (x_train, y_train), (x_test, y_test) = cifar100.load_data() x_train = x_train.astype('float32') x_test = x_test.astype('float32') # Preprocess mean = np.mean(x_train, axis=(0, 1, 2)) # Per channel normalization std = np.std(x_train, axis=(0, 1, 2)) x_train = (x_train - mean) / (std + 1e-7) x_test = (x_test - mean) / (std + 1e-7) y_train = np_utils.to_categorical(y_train, params.NUM_CLASSES) y_test = np_utils.to_categorical(y_test, params.NUM_CLASSES) # Data input class Dataset: """Class required for retrieving and preprocessing a single image.""" def __init__(self, img_data, label_data, augmentation=False): self.img_data = img_data self.label_data = label_data self.augmentation = augmentation def __getitem__(self, i): label = self.label_data[i] image = self.img_data[i] if self.augmentation: image = preprocess_img.augment_img(image) return image, label def __len__(self): return len(self.label_data) class Dataloader(K.utils.Sequence): """Class required for iterating over batches of processed images.""" def __init__(self, dataset, batch_size, shuffle=False): self.dataset = dataset self.batch_size = batch_size self.shuffle = shuffle self.indexes = np.arange(len(dataset)) self.on_epoch_end() def __getitem__(self, i): start = i * self.batch_size stop = (i + 1) * self.batch_size data = [] for j in range(start, stop): data.append(self.dataset[j]) # Transpose list of lists batch = [np.stack(samples, axis=0) for samples in zip(*data)] return batch def __len__(self): """Denotes the number of batches per epoch""" return len(self.indexes) // self.batch_size def on_epoch_end(self): """Callback function to shuffle indexes each epoch""" if self.shuffle: self.indexes = np.random.permutation(self.indexes) train_dataset = Dataset(x_train, y_train, augmentation=True) val_dataset = Dataset(x_test, y_test) train_dataloader = Dataloader(train_dataset, batch_size=params.BATCH_SIZE, shuffle=True) val_dataloader = Dataloader(val_dataset, batch_size=params.BATCH_SIZE, shuffle=False) return train_dataloader, val_dataloader
def main(_): logging.set_verbosity(tf.logging.INFO) assert os.path.isfile(FLAGS.pipeline_proto) g = tf.Graph() with g.as_default(): pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto) logging.info("Pipeline configure: %s", '=' * 128) logging.info(pipeline_proto) # Get examples from reader. examples = ads_examples.get_examples(pipeline_proto.example_reader) # Build model for evaluation. global_step = slim.get_or_create_global_step() model = builder.build(pipeline_proto.model, is_training=False) predictions = model.build_inference(examples) loss_dict = model.build_loss(predictions) uninitialized_variable_names = tf.report_uninitialized_variables() saver = tf.train.Saver() init_op = tf.group(tf.local_variables_initializer(), tf.global_variables_initializer()) session_config = train_utils.default_session_config( FLAGS.per_process_gpu_memory_fraction) # Start session. logging.info('=' * 128) eval_config = pipeline_proto.eval_config writer = tf.summary.FileWriter(FLAGS.eval_log_dir, g) prev_step = -1 while True: start = time.time() try: model_path = tf.train.latest_checkpoint(FLAGS.train_log_dir) if model_path is not None: with tf.Session(graph=g, config=session_config) as sess: # Initialize model. sess.run(init_op) saver.restore(sess, model_path) logging.info('Restore from %s.', model_path) warn_names = sess.run(uninitialized_variable_names) assert len(warn_names) == 0 step = sess.run(global_step) if step != prev_step and step > eval_config.eval_min_global_steps: prev_step = step # Evaluation loop. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) model_metric = evaluate_once(sess, writer, step, examples['video_id'], predictions=predictions) step_best, metric_best = train_utils.save_model_if_it_is_better( step, model_metric, model_path, FLAGS.saved_ckpts_dir, reverse=False) # We improved the model, record it. if step_best == step: summary = tf.Summary() summary.value.add( tag='metrics/metric_best', simple_value=metric_best) writer.add_summary(summary, global_step=step) writer.flush() coord.request_stop() coord.join(threads, stop_grace_period_secs=10) if step >= eval_config.number_of_steps: break except Exception as ex: pass # Sleep a while. sleep_secs = eval_config.eval_interval_secs - (time.time() - start) if sleep_secs > 0: logging.info('Now sleep for %.2lf secs.', sleep_secs) time.sleep(sleep_secs) writer.close() logging.info('Done')
import os os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' from tensorflow import logging logging.set_verbosity(logging.INFO) from keras.constraints import maxnorm import numpy as np import pandas as pd from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder from keras.models import Model, load_model from keras.layers import Input, Reshape, Dot from keras.layers.embeddings import Embedding from keras.optimizers import Adam from keras.regularizers import l2 from keras.layers import Add, Activation, Lambda from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint from keras import backend from matplotlib import pyplot import math import sys clear = lambda: os.system('cls') clear() path = "ml-latest" ratings = pd.read_csv(path + '/jw_ratings.csv', sep=',', encoding='latin-1', usecols=['userId', 'movieId', 'rating'])
# import matplotlib to visualization import matplotlib.pyplot as plt # helper librarys from math import ceil from numpy import array, argmax # import tensorflow tools and datasets from tensorflow import enable_eager_execution, cast, float32 from tensorflow_datasets import load from tensorflow.nn import relu, softmax from tensorflow.keras import Sequential from tensorflow.keras.layers import Dense, Flatten from tensorflow.logging import set_verbosity, ERROR set_verbosity(ERROR) enable_eager_execution() dataset, metadata = load('fashion_mnist', as_supervised=True, with_info=True) clothes_classes = [ 'T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot' ] BATCH_SIZE = 32 num_train_examples = metadata.splits['train'].num_examples num_test_examples = metadata.splits['test'].num_examples steps = num_train_examples / BATCH_SIZE
def main(unused_argv): logging.set_verbosity(tf.logging.INFO) print("tensorflow version: %s" % tf.__version__) check_video_id()
def main(): env = json.loads(os.environ.get("TF_CONFIG", "{}")) task_data = env.get("task", None) or {"type": "master", "index": 0} task = type("TaskSpec", (object, ), task_data) logging.set_verbosity(tf.logging.INFO) logging.info("%s: Tensorflow version: %s.", task_as_string(task), tf.__version__) video_ids, video_features, video_labels, video_frames = gen_input( data_pattern, reader_batch_size=reader_batch_size, num_classes=num_classes, num_readers=num_readers, mini_batch_size=mini_batch_size) result = gen_model(model_input=video_features, vocab_size=num_classes, labels=video_labels, num_frames=video_frames) predictions = result["predictions"] global_step = tf.Variable(0, trainable=False, name="global_step") label_loss = label_loss_fn.calculate_loss(predictions, video_labels) if "regularization_loss" in result.keys(): reg_loss = result["regularization_loss"] else: reg_loss = tf.constant(0.0) reg_losses = tf.losses.get_regularization_losses() if reg_losses: reg_loss += tf.add_n(reg_losses) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) if "update_ops" in result.keys(): update_ops += result["update_ops"] if update_ops: with tf.control_dependencies(update_ops): barrier = tf.no_op(name="gradient_barrier") with tf.control_dependencies([barrier]): label_loss = tf.identity(label_loss) final_loss = regularization_penalty * reg_loss + label_loss learning_rate = tf.train.exponential_decay(base_learning_rate, global_step * mini_batch_size * num_towers, learning_rate_decay_examples, learning_rate_decay, staircase=True) tf.summary.scalar('learning_rate', learning_rate) optimizer = optimizer_class(learning_rate) gradients = optimizer.compute_gradients(final_loss, colocate_gradients_with_ops=False) tf.summary.scalar("label_loss", label_loss) tf.summary.scalar("reg_loss", reg_loss) if clip_gradient_norm > 0: with tf.name_scope('clip_grads'): gradients = utils.clip_gradient_norms(gradients, clip_gradient_norm) train_op = optimizer.apply_gradients(gradients, global_step=global_step) with tf.Session() as sess: tf.global_variables_initializer().run() tf.local_variables_initializer().run() #init_local_op = tf.local_variables_initializer() #sess.run(init_local_op) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) total_step = 0 try: while total_step < 100000: batch_start_time = time.time() # v_ids, v_features, v_labels, v_frames = sess.run([video_ids, video_features, video_labels, video_frames]) _, global_step_val, loss_val, predictions_val, labels_val = sess.run( [ train_op, global_step, label_loss, predictions, tf.cast(video_labels, tf.float32) ]) seconds_per_batch = time.time() - batch_start_time examples_per_second = labels_val.shape[0] / seconds_per_batch # if max_steps <= global_step_val: # max_steps_reached = True # print(v_features.shape) # print(v_ids) if total_step % 10 == 0: eval_start_time = time.time() hit_at_one = eval_util.calculate_hit_at_one( predictions_val, labels_val) perr = eval_util.calculate_precision_at_equal_recall_rate( predictions_val, labels_val) gap = eval_util.calculate_gap(predictions_val, labels_val) eval_end_time = time.time() eval_time = eval_end_time - eval_start_time logging.info("training step " + str(global_step_val) + " | Loss: " + ("%.2f" % loss_val) + " Examples/sec: " + ("%.2f" % examples_per_second) + " | Hit@1: " + ("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " + ("%.2f" % gap)) else: logging.info("training step " + str(global_step_val) + " | Loss: " + ("%.2f" % loss_val) + " Examples/sec: " + ("%.2f" % examples_per_second)) total_step = total_step + 1 except tf.errors.OutOfRangeError: logging.info("%s: Done training -- epoch limit reached.", task_as_string(task)) coord.request_stop() coord.join(threads)
def get_model_from_name(model_name, training_params=None): global keras_imported # For Keras epochs = 250 if os.environ.get('is_test_suite', 0) == 'True' and model_name[:12] == 'DeepLearning': print('Heard that this is the test suite. Limiting number of epochs, which will increase training speed dramatically at the expense of model accuracy') epochs = 30 all_model_params = { 'LogisticRegression': {'n_jobs': -2}, 'RandomForestClassifier': {'n_jobs': -2}, 'ExtraTreesClassifier': {'n_jobs': -1}, 'AdaBoostClassifier': {'n_estimators': 10}, 'SGDClassifier': {'n_jobs': -1}, 'Perceptron': {'n_jobs': -1}, 'LinearSVC': {'dual': False}, 'LinearRegression': {'n_jobs': -2}, 'RandomForestRegressor': {'n_jobs': -2}, 'LinearSVR': {'dual': False, 'loss': 'squared_epsilon_insensitive'}, 'ExtraTreesRegressor': {'n_jobs': -1}, 'MiniBatchKMeans': {'n_clusters': 8}, 'GradientBoostingRegressor': {'presort': False, 'learning_rate': 0.05, 'warm_start': True}, 'GradientBoostingClassifier': {'presort': False, 'learning_rate': 0.05, 'warm_start': True}, 'SGDRegressor': {'shuffle': False}, 'PassiveAggressiveRegressor': {'shuffle': False}, 'AdaBoostRegressor': {'n_estimators': 10}, 'XGBRegressor': {'nthread':-1, 'n_estimators': 200}, 'XGBClassifier': {'nthread':-1, 'n_estimators': 200}, 'LGBMRegressor': {'n_estimators': 2000, 'learning_rate': 0.05, 'num_leaves': 8, 'lambda_l2': 0.001}, 'LGBMClassifier': {'n_estimators': 2000, 'learning_rate': 0.05, 'num_leaves': 8, 'lambda_l2': 0.001}, 'DeepLearningRegressor': {'epochs': epochs, 'batch_size': 50, 'verbose': 2}, 'DeepLearningClassifier': {'epochs': epochs, 'batch_size': 50, 'verbose': 2}, 'CatBoostRegressor': {}, 'CatBoostClassifier': {} } model_params = all_model_params.get(model_name, None) if model_params is None: model_params = {} if training_params is not None: print('Now using the model training_params that you passed in:') print(training_params) # Overwrite our stock params with what the user passes in (i.e., if the user wants 10,000 trees, we will let them do it) model_params.update(training_params) print('After overwriting our defaults with your values, here are the final params that will be used to initialize the model:') print(model_params) model_map = { # Classifiers 'LogisticRegression': LogisticRegression(), 'RandomForestClassifier': RandomForestClassifier(), 'RidgeClassifier': RidgeClassifier(), 'GradientBoostingClassifier': GradientBoostingClassifier(), 'ExtraTreesClassifier': ExtraTreesClassifier(), 'AdaBoostClassifier': AdaBoostClassifier(), 'SGDClassifier': SGDClassifier(), 'Perceptron': Perceptron(), 'PassiveAggressiveClassifier': PassiveAggressiveClassifier(), 'LinearSVC': LinearSVC(), # Regressors 'LinearRegression': LinearRegression(), 'RandomForestRegressor': RandomForestRegressor(), 'Ridge': Ridge(), 'LinearSVR': LinearSVR(), 'ExtraTreesRegressor': ExtraTreesRegressor(), 'AdaBoostRegressor': AdaBoostRegressor(), 'RANSACRegressor': RANSACRegressor(), 'GradientBoostingRegressor': GradientBoostingRegressor(), 'Lasso': Lasso(), 'ElasticNet': ElasticNet(), 'LassoLars': LassoLars(), 'OrthogonalMatchingPursuit': OrthogonalMatchingPursuit(), 'BayesianRidge': BayesianRidge(), 'ARDRegression': ARDRegression(), 'SGDRegressor': SGDRegressor(), 'PassiveAggressiveRegressor': PassiveAggressiveRegressor(), # Clustering 'MiniBatchKMeans': MiniBatchKMeans() } if xgb_installed: model_map['XGBClassifier'] = XGBClassifier() model_map['XGBRegressor'] = XGBRegressor() if lgb_installed: model_map['LGBMRegressor'] = LGBMRegressor() model_map['LGBMClassifier'] = LGBMClassifier() if catboost_installed: model_map['CatBoostRegressor'] = CatBoostRegressor(calc_feature_importance=True) model_map['CatBoostClassifier'] = CatBoostClassifier(calc_feature_importance=True) if model_name[:12] == 'DeepLearning': if keras_imported == False: # Suppress some level of logs if TF is installed (but allow it to not be installed, and use Theano instead) try: os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' from tensorflow import logging logging.set_verbosity(logging.INFO) except: pass global maxnorm global Dense, Dropout global LeakyReLU, PReLU global Sequential global keras_load_model global regularizers global KerasRegressor, KerasClassifier from keras.constraints import maxnorm from keras.layers import Dense, Dropout from keras.layers.advanced_activations import LeakyReLU, PReLU from keras.models import Sequential from keras.models import load_model as keras_load_model from keras import regularizers from keras.wrappers.scikit_learn import KerasRegressor, KerasClassifier maxnorm Dense Dropout LeakyReLU PReLU Sequential keras_load_model regularizers KerasRegressor KerasClassifier keras_imported = True model_map['DeepLearningClassifier'] = KerasClassifier(build_fn=make_deep_learning_classifier) model_map['DeepLearningRegressor'] = KerasRegressor(build_fn=make_deep_learning_model) try: model_without_params = model_map[model_name] except KeyError as e: print('It appears you are trying to use a library that is not available when we try to import it, or using a value for model_names that we do not recognize') raise(e) model_with_params = model_without_params.set_params(**model_params) return model_with_params
def main(unused_argv): logging.set_verbosity(tf.logging.INFO) print("tensorflow version: %s" % tf.__version__) evaluate()
def main(argv): logging.set_verbosity(tf.logging.INFO) label_map = label_map_util.load_labelmap(FLAGS.label_map) id2name = {} for item_id, item in enumerate(label_map.item): if item.HasField('display_name'): id2name[item.id - 1] = item.display_name else: id2name[item.id - 1] = item.name with open(FLAGS.output_vocab, 'w') as fp: for cid, cname in id2name.iteritems(): fp.write('%i\t%s\n' % (cid, cname)) logging.info('Dump %s words', len(id2name)) # Build tensorflow graph. g = tf.Graph() with g.as_default(): input_node = tf.placeholder(tf.uint8, shape=(None, None, None, 3)) # Create detection model. model_proto = _load_model_proto(FLAGS.model_proto) model = model_builder.build(model_proto, is_training=False) predictions = model.predict( model.preprocess(tf.cast(input_node, tf.float32))) detections = model.postprocess(predictions) init_fn = slim.assign_from_checkpoint_fn(FLAGS.checkpoint, tf.global_variables()) uninitialized_variable_names = tf.report_uninitialized_variables() # Start session to extract video features. with tf.Session(graph=g) as sess: init_fn(sess) warn_names = sess.run(uninitialized_variable_names) assert len(warn_names) == 0 def _extract_feature(video_id, images_data): results = sess.run(detections, feed_dict={input_node: images_data}) n_frames = len(images_data) features = np.zeros((n_frames, FLAGS.num_classes)) det_results = [] for i in xrange(n_frames): det_result = [] for j in xrange(results['num_detections'][i]): cid = int(results['detection_classes'][i, j]) cscore = float(results['detection_scores'][i, j]) features[i, cid] = max(cscore, features[i, cid]) y1, x1, y2, x2 = [ round(float(x), 4) for x in results['detection_boxes'][i, j] ] if x2 - x1 >= FLAGS.min_object_size and y2 - y1 > FLAGS.min_object_size: det_result.append({ 'cid': cid, 'cname': id2name.get(cid, 'UNK'), 'score': round(cscore, 4), 'bounding_box': { 'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2 } }) #print(id2name.get(cid, 'UNK')) #print(cscore) #vis.image_draw_bounding_box(images_data[i], [x1, y1, x2, y2]) det_results.append(det_result) #vis.image_save('tmp/%i.jpg' % (i), images_data[i], True) return features, det_results # Iterate through video ids. for video_id in _video_id_iterator(FLAGS.video_id_path): _extract_video_feature(video_id, FLAGS.video_dir, FLAGS.output_dir, _extract_feature) logging.info('Done')
def main(argv): logging.set_verbosity(tf.logging.INFO) label_map = label_map_util.load_labelmap(FLAGS.label_map) id2name = {} for item_id, item in enumerate(label_map.item): if item.HasField('display_name'): id2name[item.id - 1] = item.display_name else: id2name[item.id - 1] = item.name with open(FLAGS.output_vocab, 'w') as fp: for cid, cname in id2name.iteritems(): fp.write('%i\t%s\n' % (cid, cname)) logging.info('Dump %s words', len(id2name)) # Build tensorflow graph. g = tf.Graph() with g.as_default(): input_node = tf.placeholder(tf.uint8, shape=(None, None, None, 3)) # Create detection model. model_proto = _load_model_proto(FLAGS.model_proto) model = model_builder.build(model_proto, is_training=False) predictions = model.predict( model.preprocess(tf.cast(input_node, tf.float32))) detections = model.postprocess(predictions) # Load the latest model checkpoint. model_path = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) init_fn = slim.assign_from_checkpoint_fn(model_path, tf.global_variables()) logging.info('Load variables from %s.', model_path) uninitialized_variable_names = tf.report_uninitialized_variables() # Start session to extract video features. with tf.Session(graph=g) as sess: init_fn(sess) warn_names = sess.run(uninitialized_variable_names) assert len(warn_names) == 0 def _extract_feature(video_id, images_data): results = sess.run(detections, feed_dict={input_node: images_data}) n_frames = len(images_data) det_results = [] num_classes = len(age_vocab) + len(gender_vocab) + len( category_vocab) features = [] vis_dir = os.path.join('tmp', video_id) if not os.path.isdir(vis_dir): os.makedirs(vis_dir) for i in xrange(n_frames): det_result = [] person_feature_list = [] image_disp = cv2.resize(images_data[i], (512, 512)) for j in xrange(results['num_detections'][i]): detection_score = results['detection_scores'][i, j] emotic_ages = results['detection_emotic_ages'][i, j] emotic_genders = results['detection_emotic_genders'][i, j] emotic_categories = results['detection_emotic_categories'][ i, j] person_feature = np.concatenate( [emotic_ages, emotic_genders, emotic_categories], 0) person_feature_list.append(person_feature) age_ind = emotic_ages.argmax() age_pro = int(100 * emotic_ages[age_ind]) age_str = '{}: {}%'.format(age_vocab[age_ind], age_pro) gender_ind = emotic_genders.argmax() gender_pro = int(100 * emotic_genders[gender_ind]) gender_str = '{}: {}%'.format(gender_vocab[gender_ind], gender_pro) cat_str_list = [] for cat_ind in emotic_categories.argsort()[::-1]: cat_pro = int(100 * emotic_categories[cat_ind]) if cat_pro < 20: break cat_str_list.append('{}: {}%'.format( category_vocab[cat_ind], cat_pro)) display_str_list = [age_str] + [gender_str] + cat_str_list # Process max pooling. cat_list = [] for cat_id, cat_score in enumerate(emotic_categories): if cat_score > 0.2: cat_list.append({ 'category': category_vocab[cat_id], 'score': round(float(emotic_categories[cat_id]), 3) }) emotic = { 'age': { 'age': age_vocab[emotic_ages.argmax()], 'score': round(float(emotic_ages.max()), 3), }, 'gender': { 'gender': gender_vocab[emotic_genders.argmax()], 'score': round(float(emotic_genders.max()), 3), }, 'categories': cat_list } y1, x1, y2, x2 = [ round(float(x), 4) for x in results['detection_boxes'][i, j] ] if x2 - x1 >= FLAGS.min_object_size and y2 - y1 > FLAGS.min_object_size: det_result.append({ 'score': round(float(results['detection_scores'][i, j]), 4), 'emotic': emotic, 'bounding_box': { 'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2 } }) # vis.image_draw_bounding_box(image_disp, [x1, y1, x2, y2]) # vis.image_draw_text(image_disp, # [x1, y1], # '{}%'.format(int(100 * detection_score)), # (0, 0, 0)) # text_label_height = _get_text_label_height() # for disp_ind, display_str in enumerate(display_str_list): # vis.image_draw_text(image_disp, # [x1, y1 + 1.0 * text_label_height / image_disp.shape[0] * (disp_ind + 1)], # display_str, # color=(0, 0, 0), # bkg_color=(0, 0, 0)) det_results.append(det_result) features.append(person_feature_list) #vis.image_save('%s/%i.jpg' % (vis_dir, i), image_disp, True) return features, det_results # Iterate through video ids. for video_id in _video_id_iterator(FLAGS.video_id_path): _extract_video_feature(video_id, FLAGS.video_dir, FLAGS.output_dir, _extract_feature) logging.info('Done')
def __init__(self, name, params={}, use_gpu=True, verbose=True): """Parameter initialization. Arguments ----------- - name. String which will be used to identify the model in any folders or files created. - params. Optional. Dictionary containing the parameters that the user whishes to specify. - use_gpu. Boolean specifying whether a GPU should be used. True by default. - verbose. Boolean specifying whether output must be produced in-line. """ self.verbose = verbose # Print logo. Isn't it cool? # (Although it is cool, we won't print it if you don't want) if self.verbose: print(__logo__.format(__version__)) # Set minimum verbosity for RDKit, Keras and TF backends os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' logging.set_verbosity(logging.INFO) rdBase.DisableLog('rdApp.error') # Set configuration for GPU self.config = tf.ConfigProto() self.config.gpu_options.allow_growth = True # Set parameters self.PREFIX = name if 'PRETRAIN_GEN_EPOCHS' in params: self.PRETRAIN_GEN_EPOCHS = params['PRETRAIN_GEN_EPOCHS'] else: self.PRETRAIN_GEN_EPOCHS = 240 if 'PRETRAIN_DIS_EPOCHS' in params: self.PRETRAIN_DIS_EPOCHS = params['PRETRAIN_DIS_EPOCHS'] else: self.PRETRAIN_DIS_EPOCHS = 50 if 'GEN_ITERATIONS' in params: self.GEN_ITERATIONS = params['GEN_ITERATIONS'] else: self.GEN_ITERATIONS = 2 if 'GEN_BATCH_SIZE' in params: self.GEN_BATCH_SIZE = params['GEN_BATCH_SIZE'] else: self.GEN_BATCH_SIZE = 64 if 'SEED' in params: self.SEED = params['SEED'] else: self.SEED = None random.seed(self.SEED) np.random.seed(self.SEED) if 'DIS_BATCH_SIZE' in params: self.DIS_BATCH_SIZE = params['DIS_BATCH_SIZE'] else: self.DIS_BATCH_SIZE = 64 if 'DIS_EPOCHS' in params: self.DIS_EPOCHS = params['DIS_EPOCHS'] else: self.DIS_EPOCHS = 3 if 'EPOCH_SAVES' in params: self.EPOCH_SAVES = params['EPOCH_SAVES'] else: self.EPOCH_SAVES = 20 if 'CHK_PATH' in params: self.CHK_PATH = params['CHK_PATH'] else: self.CHK_PATH = os.path.join(os.getcwd(), 'checkpoints/{}'.format(self.PREFIX)) if 'GEN_EMB_DIM' in params: self.GEN_EMB_DIM = params['GEN_EMB_DIM'] else: self.GEN_EMB_DIM = 32 if 'GEN_HIDDEN_DIM' in params: self.GEN_HIDDEN_DIM = params['GEN_HIDDEN_DIM'] else: self.GEN_HIDDEN_DIM = 32 if 'START_TOKEN' in params: self.START_TOKEN = params['START_TOKEN'] else: self.START_TOKEN = 0 if 'SAMPLE_NUM' in params: self.SAMPLE_NUM = params['SAMPLE_NUM'] else: self.SAMPLE_NUM = 6400 if 'BIG_SAMPLE_NUM' in params: self.BIG_SAMPLE_NUM = params['BIG_SAMPLE_NUM'] else: self.BIG_SAMPLE_NUM = self.SAMPLE_NUM * 5 if 'LAMBDA' in params: self.LAMBDA = params['LAMBDA'] else: self.LAMBDA = 0.5 # In case this parameter is not specified by the user, # it will be determined later, in the training set # loading. if 'MAX_LENGTH' in params: self.MAX_LENGTH = params['MAX_LENGTH'] if 'DIS_EMB_DIM' in params: self.DIS_EMB_DIM = params['DIS_EMB_DIM'] else: self.DIS_EMB_DIM = 64 if 'DIS_FILTER_SIZES' in params: self.DIS_FILTER_SIZES = params['DIS_FILTER_SIZES'] else: self.DIS_FILTER_SIZES = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20] if 'DIS_NUM_FILTERS' in params: self.DIS_NUM_FILTERS = params['DIS_FILTER_SIZES'] else: self.DIS_NUM_FILTERS = [ 100, 200, 200, 200, 200, 100, 100, 100, 100, 100, 160, 160 ] if 'DIS_DROPOUT' in params: self.DIS_DROPOUT = params['DIS_DROPOUT'] else: self.DIS_DROPOUT = 0.75 if 'DIS_L2REG' in params: self.DIS_L2REG = params['DIS_L2REG'] else: self.DIS_L2REG = 0.2 self.AV_METRICS = get_metrics() self.LOADINGS = metrics_loading() self.PRETRAINED = False self.SESS_LOADED = False self.USERDEF_METRIC = False
def main(unused_argv): logging.set_verbosity(tf.logging.INFO) inference_loop()
import os import tensorflow as tf from backend.container_fillers.generator_queue_fillers import GeneratorInputV3 from tensorflow import logging as logging logging.set_verbosity(tf.logging.INFO) os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" def train_generator_input_fn_v3(questions, non_questions, input_fn_params, graph_params): " Main input function -- use this one " kwargs = { 'questions': questions, 'non_questions': non_questions, 'Q_size': 700, 'num_proc': 6, 'word2index': graph_params['word2index'], 'max_seq_length': graph_params['input_max_length'], 'max_num_questions': input_fn_params['max_num_questions'], 'max_num_elements': input_fn_params['max_num_elements'], 'randomize_num_questions': input_fn_params['randomize_num_questions'] } gen_obj = GeneratorInputV3( **kwargs) # This should take max a few seconds to prefill the queue generator = gen_obj.from_queue_generator() msl = graph_params['input_max_length'] bsize = input_fn_params['batch_size']
def main(unused_argv): logging.set_verbosity(logging.INFO) start_new_model = FLAGS.start_new_model output_dir = FLAGS.output_dir init_learning_rate = FLAGS.init_learning_rate decay_steps = FLAGS.decay_steps decay_rate = FLAGS.decay_rate l1_reg_rate = FLAGS.l1_reg_rate l2_reg_rate = FLAGS.l2_reg_rate is_bootstrap = FLAGS.is_bootstrap train_epochs = FLAGS.train_epochs model_type, feature_names, feature_sizes = FLAGS.model_type, FLAGS.feature_names, FLAGS.feature_sizes reader = get_reader(model_type, feature_names, feature_sizes) train_data_pattern = FLAGS.train_data_pattern validate_data_pattern = FLAGS.validate_data_pattern batch_size = FLAGS.batch_size num_readers = FLAGS.num_readers if tf.gfile.Exists(path_join(output_dir, 'validate_data.pickle')): with open(path_join(output_dir, 'validate_data.pickle'), 'rb') as f: validate_data = pickle.load(f) with open(path_join(output_dir, 'validate_labels.pickle'), 'rb') as f: validate_labels = pickle.load(f) else: # Increase num_readers. validate_data_pipeline = DataPipeline( reader=reader, data_pattern=validate_data_pattern, batch_size=batch_size, num_readers=num_readers) # Sample validate set. _, validate_data, validate_labels, _ = random_sample( 0.05, mask=(False, True, True, False), data_pipeline=validate_data_pipeline, name_scope='sample_validate') with open(path_join(output_dir, 'validate_data.pickle'), 'wb') as f: pickle.dump(validate_data, f) with open(path_join(output_dir, 'validate_labels.pickle'), 'wb') as f: pickle.dump(validate_labels, f) train_data_pipeline = DataPipeline(reader=reader, data_pattern=train_data_pattern, batch_size=batch_size, num_readers=num_readers) model_save_path = path_join(output_dir, 'mlp_fuse') if start_new_model and tf.gfile.Exists(model_save_path): logging.info('Starting a new model...') # Start new model, delete existing checkpoints. try: tf.gfile.DeleteRecursively(model_save_path) except tf.errors.OpError: logging.error('Failed to delete dir {}.'.format(model_save_path)) else: logging.info( 'Succeeded to delete train dir {}.'.format(model_save_path)) # Set pos_weights for extremely imbalanced situation in one-vs-all classifiers. try: # Load sum_labels in training set, numpy float format to compute pos_weights. train_sum_labels = load_sum_labels() # num_neg / num_pos, assuming neg_weights === 1.0. pos_weights = np.sqrt( float(NUM_TRAIN_EXAMPLES) / train_sum_labels - 1.0) logging.info( 'Computing pos_weights based on sum_labels in train set successfully.' ) except IOError: logging.error('Cannot load train sum_labels. Use default value.') pos_weights = None finally: logging.warn('Not to use positive weights.') pos_weights = None train(train_data_pipeline, epochs=train_epochs, pos_weights=pos_weights, l1_reg_rate=l1_reg_rate, l2_reg_rate=l2_reg_rate, init_learning_rate=init_learning_rate, bootstrap=is_bootstrap, validate_set=(validate_data, validate_labels), validate_fn=gap_fn, logdir=model_save_path)
def _basic_setup(self, config): ''' 基本的配置TPU通信协议,如果use_tpu=False,或略一切关于tpu的配置 :return: ''' if self.use_tpu: tpu_name = config.get('tpu_name') if 'tpu_name' in config else None tpu_zone = config.get('tpu_zone') if 'tpu_zone' in config else None try: self.tpu = tfcontrib.cluster_resolver.TPUClusterResolver( tpu=tpu_name, zone=tpu_zone) info = self.tpu.cluster_spec().as_dict() tflogging.info('Run on TPU:%s' % (','.join(info['worker']))) except: raise ValueError('当前环境不支持TPU') tpu_iterations = config.get( 'tpu_iterations') if 'tpu_iterations' in config else None self.tpu_run_config = tfcontrib.tpu.TPUConfig(tpu_iterations) tpu_master = config.get( 'tpu_master') if 'tpu_master' in config else None self.run_config = tfcontrib.tpu.RunConfig(tpu_config=config, master=tpu_master, cluster=self.tpu) else: self.run_config = tfcontrib.tpu.RunConfig() if __name__ == '__main__': tflogging.set_verbosity(tflogging.INFO) TPU_Solver('config.json')
def main(_): logging.set_verbosity(tf.logging.INFO) assert os.path.isfile(FLAGS.pipeline_proto), FLAGS.pipeline_proto g = tf.Graph() with g.as_default(): pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto) logging.info("Pipeline configure: %s", '=' * 128) logging.info(pipeline_proto) train_config = pipeline_proto.train_config # Get examples from reader. examples = ads_examples.get_examples(pipeline_proto.example_reader) # Build model for training. global_step = slim.get_or_create_global_step() model = builder.build(pipeline_proto.model, is_training=True) predictions = model.build_inference(examples) loss_dict = model.build_loss(predictions) init_fn = model.get_init_fn() uninitialized_variable_names = tf.report_uninitialized_variables() # Loss and optimizer. for loss_name, loss_tensor in loss_dict.iteritems(): tf.losses.add_loss(loss_tensor) tf.summary.scalar('losses/{}'.format(loss_name), loss_tensor) total_loss = tf.losses.get_total_loss() tf.summary.scalar('losses/total_loss', total_loss) optimizer = train_utils.build_optimizer(train_config) if train_config.moving_average: optimizer = tf.contrib.opt.MovingAverageOptimizer(optimizer, average_decay=0.99) gradient_multipliers = train_utils.build_multipler( train_config.gradient_multiplier) variables_to_train = model.get_variables_to_train() for var in variables_to_train: logging.info(var) train_op = slim.learning.create_train_op(total_loss, variables_to_train=variables_to_train, clip_gradient_norm=5.0, gradient_multipliers=gradient_multipliers, summarize_gradients=True, optimizer=optimizer) saver = None if train_config.moving_average: saver = optimizer.swapping_saver() # Starts training. logging.info('Start training.') session_config = train_utils.default_session_config( FLAGS.per_process_gpu_memory_fraction) slim.learning.train(train_op, logdir=FLAGS.train_log_dir, graph=g, master='', is_chief=True, number_of_steps=train_config.number_of_steps, log_every_n_steps=train_config.log_every_n_steps, save_interval_secs=train_config.save_interval_secs, save_summaries_secs=train_config.save_summaries_secs, session_config=session_config, init_fn=init_fn, saver=saver) logging.info('Done')
main of salted fish """ import os import matplotlib.pyplot as plt from sklearn.metrics import accuracy_score, recall_score, precision_score from tensorflow import logging from models import Cnn1DMultiChannelModel, CnnWithEmbedding, \ CnnWithSingleDayStatus, CnnWithStatusAutoEncode, \ CnnWithSingleDayStatusEncode from util import timer from util.utils import save_figure os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' logging.set_verbosity(logging.WARN) def top_n_precision(y_pred, y_true, codes): pred_label_code_lst = list(zip(y_pred, y_true, codes)) sorted_lst = sorted(pred_label_code_lst, key=lambda x: x[0], reverse=True) for pred, label, s_code in sorted_lst[:10]: print(f"pred_prob: {pred}, label: {label}, stock_id: {s_code}") for n in [1, 3, 5, 10, 15, 20]: res_lst = sorted_lst[:n] if res_lst[-1][0] < 0.5: # raise Exception(f"{n}th prob < 0.5") print(f"{n}th prob < 0.5") correct_cnt = sum([x[1] for x in res_lst])
from keras.models import Model from keras.layers import Input, Add, PReLU, Conv2DTranspose, Concatenate, MaxPooling2D, UpSampling2D, Dropout, concatenate from keras.layers.convolutional import Conv2D from keras.layers.normalization import BatchNormalization from keras.callbacks import Callback from tensorflow import log from keras import backend as K from tensorflow.logging import set_verbosity, FATAL set_verbosity(FATAL) def get_red30_model(): """ Define the model architecture Return the model """ inputs = Input(shape=(256, 256, 2)) enc_conv0 = Conv2D(32, (3, 3), padding="same", activation='relu', kernel_initializer="he_normal")(inputs) enc_conv1 = Conv2D(32, (3, 3), padding="SAME", activation='relu')(enc_conv0) max_pool1 = MaxPooling2D((2, 2))(enc_conv1) enc_conv2 = Conv2D(32, (3, 3), padding="SAME", activation='relu')(max_pool1)
def get_model_from_name(model_name, training_params=None, is_hp_search=False): # For Keras epochs = 1000 # if os.environ.get('is_test_suite', 0) == 'True' and model_name[:12] == 'DeepLearning': # print('Heard that this is the test suite. Limiting number of epochs, which will increase ' # 'training speed dramatically at the expense of model accuracy') # epochs = 100 all_model_params = { 'LogisticRegression': {}, 'RandomForestClassifier': { 'n_jobs': -2, 'n_estimators': 30 }, 'ExtraTreesClassifier': { 'n_jobs': -1 }, 'AdaBoostClassifier': {}, 'SGDClassifier': { 'n_jobs': -1 }, 'Perceptron': { 'n_jobs': -1 }, 'LinearSVC': { 'dual': False }, 'LinearRegression': { 'n_jobs': -2 }, 'RandomForestRegressor': { 'n_jobs': -2, 'n_estimators': 30 }, 'LinearSVR': { 'dual': False, 'loss': 'squared_epsilon_insensitive' }, 'ExtraTreesRegressor': { 'n_jobs': -1 }, 'MiniBatchKMeans': { 'n_clusters': 8 }, 'GradientBoostingRegressor': { 'presort': False, 'learning_rate': 0.1, 'warm_start': True }, 'GradientBoostingClassifier': { 'presort': False, 'learning_rate': 0.1, 'warm_start': True }, 'SGDRegressor': { 'shuffle': False }, 'PassiveAggressiveRegressor': { 'shuffle': False }, 'AdaBoostRegressor': {}, 'LGBMRegressor': { 'n_estimators': 2000, 'learning_rate': 0.15, 'num_leaves': 8, 'lambda_l2': 0.001, 'histogram_pool_size': 16384 }, 'LGBMClassifier': { 'n_estimators': 2000, 'learning_rate': 0.15, 'num_leaves': 8, 'lambda_l2': 0.001, 'histogram_pool_size': 16384 }, 'DeepLearningRegressor': { 'epochs': epochs, 'batch_size': 50, 'verbose': 2 }, 'DeepLearningClassifier': { 'epochs': epochs, 'batch_size': 50, 'verbose': 2 }, 'CatBoostRegressor': {}, 'CatBoostClassifier': {} } # if os.environ.get('is_test_suite', 0) == 'True': # all_model_params model_params = all_model_params.get(model_name, None) if model_params is None: model_params = {} if is_hp_search is True: if model_name[:12] == 'DeepLearning': model_params['epochs'] = 50 if model_name[:4] == 'LGBM': model_params['n_estimators'] = 500 if training_params is not None: print('Now using the model training_params that you passed in:') print(training_params) # Overwrite our stock params with what the user passes in (i.e., if the user wants 10, # 000 trees, we will let them do it) model_params.update(training_params) print( 'After overwriting our defaults with your values, here are the final params that will ' 'be used to initialize the model:') print(model_params) model_map = { # Classifiers 'LogisticRegression': LogisticRegression(), 'RandomForestClassifier': RandomForestClassifier(), 'RidgeClassifier': RidgeClassifier(), 'GradientBoostingClassifier': GradientBoostingClassifier(), 'ExtraTreesClassifier': ExtraTreesClassifier(), 'AdaBoostClassifier': AdaBoostClassifier(), 'LinearSVC': LinearSVC(), # Regressors 'LinearRegression': LinearRegression(), 'RandomForestRegressor': RandomForestRegressor(), 'Ridge': Ridge(), 'LinearSVR': LinearSVR(), 'ExtraTreesRegressor': ExtraTreesRegressor(), 'AdaBoostRegressor': AdaBoostRegressor(), 'RANSACRegressor': RANSACRegressor(), 'GradientBoostingRegressor': GradientBoostingRegressor(), 'Lasso': Lasso(), 'ElasticNet': ElasticNet(), 'LassoLars': LassoLars(), 'OrthogonalMatchingPursuit': OrthogonalMatchingPursuit(), 'BayesianRidge': BayesianRidge(), 'ARDRegression': ARDRegression(), # Clustering 'MiniBatchKMeans': MiniBatchKMeans(), } try: model_map['SGDClassifier'] = SGDClassifier(max_iter=1000, tol=0.001) model_map['Perceptron'] = Perceptron(max_iter=1000, tol=0.001) model_map['PassiveAggressiveClassifier'] = PassiveAggressiveClassifier( max_iter=1000, tol=0.001) model_map['SGDRegressor'] = SGDRegressor(max_iter=1000, tol=0.001) model_map['PassiveAggressiveRegressor'] = PassiveAggressiveRegressor( max_iter=1000, tol=0.001) except TypeError: model_map['SGDClassifier'] = SGDClassifier() model_map['Perceptron'] = Perceptron() model_map['PassiveAggressiveClassifier'] = PassiveAggressiveClassifier( ) model_map['SGDRegressor'] = SGDRegressor() model_map['PassiveAggressiveRegressor'] = PassiveAggressiveRegressor() if xgb_installed: model_map['XGBClassifier'] = XGBClassifier() model_map['XGBRegressor'] = XGBRegressor() if lgb_installed: model_map['LGBMRegressor'] = LGBMRegressor() model_map['LGBMClassifier'] = LGBMClassifier() if catboost_installed: model_map['CatBoostRegressor'] = CatBoostRegressor() model_map['CatBoostClassifier'] = CatBoostClassifier() if model_name[:12] == 'DeepLearning': if keras_installed is False: # Suppress some level of logs if TF is installed (but allow it to not be installed, # and use Theano instead) try: os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' from tensorflow import logging logging.set_verbosity(logging.INFO) except: # TODO: Fix bare Except pass model_map['DeepLearningClassifier'] = KerasClassifier( build_fn=make_deep_learning_classifier) model_map['DeepLearningRegressor'] = KerasRegressor( build_fn=make_deep_learning_model) try: model_without_params = model_map[model_name] except KeyError as e: print( 'It appears you are trying to use a library that is not available when we try to ' 'import it, or using a value for model_names that we do not recognize.' ) raise e if os.environ.get('is_test_suite', False) == 'True': if 'n_jobs' in model_params: model_params['n_jobs'] = 1 model_with_params = model_without_params.set_params(**model_params) return model_with_params
def main(unused_argv): os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpus logging.set_verbosity(tf.logging.INFO) print("tensorflow version: %s" % tf.__version__) evaluate()
def main(unused_argv): logging.set_verbosity(tf.logging.INFO) print("tensorflow version: %s" % tf.__version__) evaluate()
name="batchnorm_params") LAYER_CONFIG_ORDER = StructNode( (("in_channels", -1), ("filter_width", -1), ("filter_height", -1), ("out_channels", -1), ("skip_width", 1), ("skip_height", 1), ("maxpool", 0), ("weights", ""), ("bias", ""), ("pad", "PADDING_SAME_ZERO"), ("activation", "NONE"), ("deconvolve", 0), ("branch", 0), ("branch_copy_type", "BRANCH_NO_COPY"), ("branch_combine_type", "BRANCH_NOC"), ("branch_config", BRANCH_CONFIG_ORDER), ("bn_params", BATCHNORM_PARAMS_ORDER), ("output_num", -1))) CNN_CONFIG_ORDER = {} BIT_ALIGNMENT = 32 WEIGHT_STRING = "w" BIAS_STRING = "b" logging.set_verbosity("INFO") def _get_weight_tensor_from_prefix(tensor_prefix): return tensor_prefix + WEIGHT_STRING def _get_bias_tensor_from_prefix(tensor_prefix): return tensor_prefix + BIAS_STRING def _generate_layer_index_tensor_name_map(input_reader, var_regex): """Create a map from tuples of indices to tensor names.""" layer_index_tensor_name_map = {} # Create a index-tuple to tensor name map. for k, _ in input_reader.get_variable_to_shape_map().iteritems():
def main(unused_argv): logging.set_verbosity(tf.logging.INFO) print("tensorflow version: %s" % tf.__version__) is_chief = (FLAGS.task == 0) # Recover session saver = None latest_checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir) if FLAGS.start_new_model: logging.info("'start_new_model' flag is set. Removing existing train dir.") try: gfile.DeleteRecursively(FLAGS.train_dir) except: logging.error( "Failed to delete directory " + FLAGS.train_dir + " when starting a new model. Please delete it manually and" + " try again.") elif not latest_checkpoint: logging.info("No checkpoint file found. Building a new model.") else: meta_filename = latest_checkpoint + ".meta" if not gfile.Exists(meta_filename): logging.info("No meta graph file found. Building a new model.") else: logging.info("Restoring from meta graph file %s", meta_filename) saver = tf.train.import_meta_graph(meta_filename) if not saver: # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( FLAGS.feature_names, FLAGS.feature_sizes) if FLAGS.frame_features: reader = readers.YT8MFrameFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) else: reader = readers.YT8MAggregatedFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) model = find_class_by_name(FLAGS.model, [frame_level_models, video_level_models])() label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])() optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train]) build_graph(reader=reader, model=model, optimizer_class=optimizer_class, train_data_pattern=FLAGS.train_data_pattern, label_loss_fn=label_loss_fn, base_learning_rate=FLAGS.base_learning_rate, regularization_penalty=FLAGS.regularization_penalty, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size) logging.info("built graph") saver = tf.train.Saver() train_loop(is_chief=is_chief, train_dir=FLAGS.train_dir, saver=saver, master=FLAGS.master)
running_reward / batch_size)) if reward_sum / batch_size > 200: logging.info("Task solved in", episode_number, 'episodes!') break reward_sum = 0 time_to_export = ((last_model_export_step == 0) or (global_step_val - last_model_export_step >= export_model_steps)) if is_master and time_to_export: last_checkpoint = saver.save(sess, sv.save_path, global_step_val) last_model_export_step = global_step_val observation = env.reset() if is_master: last_checkpoint = saver.save(sess, sv.save_path, global_step_val) last_model_export_step = global_step_val model.after() if __name__ == "__main__": logging.set_verbosity(logging.DEBUG) app.run()