def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if not FLAGS.json_prediction_files_pattern:
raise ValueError(
"The flag --json_prediction_files_pattern must be specified.")
if not FLAGS.csv_output_file:
raise ValueError("The flag --csv_output_file must be specified.")
logging.info("Looking for prediction files with pattern: %s",
FLAGS.json_prediction_files_pattern)
file_paths = gfile.Glob(FLAGS.json_prediction_files_pattern)
logging.info("Found files: %s", file_paths)
logging.info("Writing submission file to: %s", FLAGS.csv_output_file)
with gfile.Open(FLAGS.csv_output_file, "w+") as output_file:
output_file.write(get_csv_header())
for file_path in file_paths:
logging.info("processing file: %s", file_path)
with gfile.Open(file_path) as input_file:
for line in input_file:
json_data = json.loads(line)
output_file.write(to_csv_row(json_data))
output_file.flush()
logging.info("done")
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
reader = get_reader()
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features,
model=model,
reader=reader)
Trainer(cluster, task, FLAGS.train_dir, model, reader, model_exporter,
FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps).run(start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if FLAGS.input_model_tgz:
if FLAGS.train_dir:
raise ValueError("You cannot supply --train_dir if supplying "
"--input_model_tgz")
# Untar.
if not os.path.exists(FLAGS.untar_model_dir):
os.makedirs(FLAGS.untar_model_dir)
tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir)
FLAGS.train_dir = FLAGS.untar_model_dir
flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json")
if not os.path.exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(open(flags_dict_file).read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# if os.path.exists(FLAGS.train_dir):
# shutil.rmtree(FLAGS.train_dir)
# subdirs = [x for x in os.listdir(FLAGS.Ensemble_Models) if os.path.isdir(x)]
subdirs = [
os.path.join(FLAGS.Ensemble_Models, x)
for x in FLAGS.model_path.split(',')
]
flags_dict = []
save_flag = True
for subdir in subdirs:
model_flags_path = os.path.join(
os.path.join(FLAGS.Ensemble_Models, subdir), "model_flags.json")
print "Load model from " + model_flags_path + "\n"
flags_dict.append(
json.loads(file_io.FileIO(model_flags_path, mode="r").read()))
# save model_flags.json to inference dictory
if save_flag:
if os.path.exists(FLAGS.train_dir) == False:
os.mkdir(FLAGS.train_dir)
shutil.copyfile(model_flags_path,
os.path.join(FLAGS.train_dir, "model_flags.json"))
save_flag = False
models = []
model_nums = len(subdirs)
for m in range(model_nums):
model = get_params(flags_dict[m])
models.append(model)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
# model = find_class_by_name(FLAGS.model,
# [frame_level_models, video_level_models])()
reader = get_reader()
# model_exporter = export_model.ModelExporter(
# frame_features=FLAGS.frame_features,
# model=models,
# reader=reader)
model_exporter = None
Trainer(cluster, task, FLAGS.train_dir, subdirs, models, reader,
model_exporter, FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
import time
from datetime import datetime
import os
import subprocess
import sys
# Attempt to disable annoying logging messages
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
from tensorflow import logging
logging.set_verbosity(logging.ERROR)
os.system('cls')
from keras.models import load_model
from keras.preprocessing.text import text_to_word_sequence
from keras.preprocessing.text import one_hot
import h5py as h5
import pandas as pd
import numpy as np
from scapy.all import *
from math import floor, log2
from _params import params
'''
Plan of Action
1) Sniff and write to pcap file
2) Process pcap file using CICFlowMeter cmd program and save to csv
import os
# Suppress some level of logs
os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
from tensorflow import logging
logging.set_verbosity(logging.FATAL)
def weight_variable(shape, name=None):
# initialize weighted variables.
initial = tf.truncated_normal(shape, stddev=0.001)
return tf.Variable(initial, name=name)
def conv2d(x, W, strides=[1, 1, 1, 1], p='SAME', name=None):
# set convolution layers.
with tf.name_scope(name):
assert isinstance(x, tf.Tensor)
return tf.nn.conv2d(x, W, strides=strides, padding=p, name=name)
def batch_norm(x):
assert isinstance(x, tf.Tensor)
# reduce dimension 1, 2, 3, which would produce batch mean and batch variance.
mean, var = tf.nn.moments(x, axes=[1, 2, 3], keep_dims=True)
return tf.nn.batch_normalization(x, mean, var, 0, 1, 1e-5)
def init_eval_environment():
"""Prepares the dataset and datagenerators used to train a model."""
config = ConfigProto()
config.gpu_options.allow_growth = True
logging.set_verbosity(
logging.ERROR) # Supress all kind of deprecation warnings
(x_train, y_train), (x_test, y_test) = cifar100.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
# Preprocess
mean = np.mean(x_train, axis=(0, 1, 2)) # Per channel normalization
std = np.std(x_train, axis=(0, 1, 2))
x_train = (x_train - mean) / (std + 1e-7)
x_test = (x_test - mean) / (std + 1e-7)
y_train = np_utils.to_categorical(y_train, params.NUM_CLASSES)
y_test = np_utils.to_categorical(y_test, params.NUM_CLASSES)
# Data input
class Dataset:
"""Class required for retrieving and preprocessing a single image."""
def __init__(self, img_data, label_data, augmentation=False):
self.img_data = img_data
self.label_data = label_data
self.augmentation = augmentation
def __getitem__(self, i):
label = self.label_data[i]
image = self.img_data[i]
if self.augmentation:
image = preprocess_img.augment_img(image)
return image, label
def __len__(self):
return len(self.label_data)
class Dataloader(K.utils.Sequence):
"""Class required for iterating over batches of processed images."""
def __init__(self, dataset, batch_size, shuffle=False):
self.dataset = dataset
self.batch_size = batch_size
self.shuffle = shuffle
self.indexes = np.arange(len(dataset))
self.on_epoch_end()
def __getitem__(self, i):
start = i * self.batch_size
stop = (i + 1) * self.batch_size
data = []
for j in range(start, stop):
data.append(self.dataset[j])
# Transpose list of lists
batch = [np.stack(samples, axis=0) for samples in zip(*data)]
return batch
def __len__(self):
"""Denotes the number of batches per epoch"""
return len(self.indexes) // self.batch_size
def on_epoch_end(self):
"""Callback function to shuffle indexes each epoch"""
if self.shuffle:
self.indexes = np.random.permutation(self.indexes)
train_dataset = Dataset(x_train, y_train, augmentation=True)
val_dataset = Dataset(x_test, y_test)
train_dataloader = Dataloader(train_dataset,
batch_size=params.BATCH_SIZE,
shuffle=True)
val_dataloader = Dataloader(val_dataset,
batch_size=params.BATCH_SIZE,
shuffle=False)
return train_dataloader, val_dataloader
def main(_):
logging.set_verbosity(tf.logging.INFO)
assert os.path.isfile(FLAGS.pipeline_proto)
g = tf.Graph()
with g.as_default():
pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto)
logging.info("Pipeline configure: %s", '=' * 128)
logging.info(pipeline_proto)
# Get examples from reader.
examples = ads_examples.get_examples(pipeline_proto.example_reader)
# Build model for evaluation.
global_step = slim.get_or_create_global_step()
model = builder.build(pipeline_proto.model, is_training=False)
predictions = model.build_inference(examples)
loss_dict = model.build_loss(predictions)
uninitialized_variable_names = tf.report_uninitialized_variables()
saver = tf.train.Saver()
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
session_config = train_utils.default_session_config(
FLAGS.per_process_gpu_memory_fraction)
# Start session.
logging.info('=' * 128)
eval_config = pipeline_proto.eval_config
writer = tf.summary.FileWriter(FLAGS.eval_log_dir, g)
prev_step = -1
while True:
start = time.time()
try:
model_path = tf.train.latest_checkpoint(FLAGS.train_log_dir)
if model_path is not None:
with tf.Session(graph=g, config=session_config) as sess:
# Initialize model.
sess.run(init_op)
saver.restore(sess, model_path)
logging.info('Restore from %s.', model_path)
warn_names = sess.run(uninitialized_variable_names)
assert len(warn_names) == 0
step = sess.run(global_step)
if step != prev_step and step > eval_config.eval_min_global_steps:
prev_step = step
# Evaluation loop.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
model_metric = evaluate_once(sess,
writer, step, examples['video_id'], predictions=predictions)
step_best, metric_best = train_utils.save_model_if_it_is_better(
step, model_metric, model_path, FLAGS.saved_ckpts_dir,
reverse=False)
# We improved the model, record it.
if step_best == step:
summary = tf.Summary()
summary.value.add(
tag='metrics/metric_best', simple_value=metric_best)
writer.add_summary(summary, global_step=step)
writer.flush()
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
if step >= eval_config.number_of_steps:
break
except Exception as ex:
pass
# Sleep a while.
sleep_secs = eval_config.eval_interval_secs - (time.time() - start)
if sleep_secs > 0:
logging.info('Now sleep for %.2lf secs.', sleep_secs)
time.sleep(sleep_secs)
writer.close()
logging.info('Done')
import os
os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
from tensorflow import logging
logging.set_verbosity(logging.INFO)
from keras.constraints import maxnorm
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from keras.models import Model, load_model
from keras.layers import Input, Reshape, Dot
from keras.layers.embeddings import Embedding
from keras.optimizers import Adam
from keras.regularizers import l2
from keras.layers import Add, Activation, Lambda
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
from keras import backend
from matplotlib import pyplot
import math
import sys
clear = lambda: os.system('cls')
clear()
path = "ml-latest"
ratings = pd.read_csv(path + '/jw_ratings.csv',
sep=',',
encoding='latin-1',
usecols=['userId', 'movieId', 'rating'])
# import matplotlib to visualization
import matplotlib.pyplot as plt
# helper librarys
from math import ceil
from numpy import array, argmax
# import tensorflow tools and datasets
from tensorflow import enable_eager_execution, cast, float32
from tensorflow_datasets import load
from tensorflow.nn import relu, softmax
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.logging import set_verbosity, ERROR
set_verbosity(ERROR)
enable_eager_execution()
dataset, metadata = load('fashion_mnist', as_supervised=True, with_info=True)
clothes_classes = [
'T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt',
'Sneaker', 'Bag', 'Ankle boot'
]
BATCH_SIZE = 32
num_train_examples = metadata.splits['train'].num_examples
num_test_examples = metadata.splits['test'].num_examples
steps = num_train_examples / BATCH_SIZE
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
check_video_id()
def main():
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
video_ids, video_features, video_labels, video_frames = gen_input(
data_pattern,
reader_batch_size=reader_batch_size,
num_classes=num_classes,
num_readers=num_readers,
mini_batch_size=mini_batch_size)
result = gen_model(model_input=video_features,
vocab_size=num_classes,
labels=video_labels,
num_frames=video_frames)
predictions = result["predictions"]
global_step = tf.Variable(0, trainable=False, name="global_step")
label_loss = label_loss_fn.calculate_loss(predictions, video_labels)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
final_loss = regularization_penalty * reg_loss + label_loss
learning_rate = tf.train.exponential_decay(base_learning_rate,
global_step * mini_batch_size *
num_towers,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = optimizer_class(learning_rate)
gradients = optimizer.compute_gradients(final_loss,
colocate_gradients_with_ops=False)
tf.summary.scalar("label_loss", label_loss)
tf.summary.scalar("reg_loss", reg_loss)
if clip_gradient_norm > 0:
with tf.name_scope('clip_grads'):
gradients = utils.clip_gradient_norms(gradients,
clip_gradient_norm)
train_op = optimizer.apply_gradients(gradients, global_step=global_step)
with tf.Session() as sess:
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
#init_local_op = tf.local_variables_initializer()
#sess.run(init_local_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
total_step = 0
try:
while total_step < 100000:
batch_start_time = time.time()
# v_ids, v_features, v_labels, v_frames = sess.run([video_ids, video_features, video_labels, video_frames])
_, global_step_val, loss_val, predictions_val, labels_val = sess.run(
[
train_op, global_step, label_loss, predictions,
tf.cast(video_labels, tf.float32)
])
seconds_per_batch = time.time() - batch_start_time
examples_per_second = labels_val.shape[0] / seconds_per_batch
# if max_steps <= global_step_val:
# max_steps_reached = True
# print(v_features.shape)
# print(v_ids)
if total_step % 10 == 0:
eval_start_time = time.time()
hit_at_one = eval_util.calculate_hit_at_one(
predictions_val, labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
gap = eval_util.calculate_gap(predictions_val, labels_val)
eval_end_time = time.time()
eval_time = eval_end_time - eval_start_time
logging.info("training step " + str(global_step_val) +
" | Loss: " + ("%.2f" % loss_val) +
" Examples/sec: " +
("%.2f" % examples_per_second) +
" | Hit@1: " + ("%.2f" % hit_at_one) +
" PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap))
else:
logging.info("training step " + str(global_step_val) +
" | Loss: " + ("%.2f" % loss_val) +
" Examples/sec: " +
("%.2f" % examples_per_second))
total_step = total_step + 1
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(task))
coord.request_stop()
coord.join(threads)
def get_model_from_name(model_name, training_params=None):
global keras_imported
# For Keras
epochs = 250
if os.environ.get('is_test_suite', 0) == 'True' and model_name[:12] == 'DeepLearning':
print('Heard that this is the test suite. Limiting number of epochs, which will increase training speed dramatically at the expense of model accuracy')
epochs = 30
all_model_params = {
'LogisticRegression': {'n_jobs': -2},
'RandomForestClassifier': {'n_jobs': -2},
'ExtraTreesClassifier': {'n_jobs': -1},
'AdaBoostClassifier': {'n_estimators': 10},
'SGDClassifier': {'n_jobs': -1},
'Perceptron': {'n_jobs': -1},
'LinearSVC': {'dual': False},
'LinearRegression': {'n_jobs': -2},
'RandomForestRegressor': {'n_jobs': -2},
'LinearSVR': {'dual': False, 'loss': 'squared_epsilon_insensitive'},
'ExtraTreesRegressor': {'n_jobs': -1},
'MiniBatchKMeans': {'n_clusters': 8},
'GradientBoostingRegressor': {'presort': False, 'learning_rate': 0.05, 'warm_start': True},
'GradientBoostingClassifier': {'presort': False, 'learning_rate': 0.05, 'warm_start': True},
'SGDRegressor': {'shuffle': False},
'PassiveAggressiveRegressor': {'shuffle': False},
'AdaBoostRegressor': {'n_estimators': 10},
'XGBRegressor': {'nthread':-1, 'n_estimators': 200},
'XGBClassifier': {'nthread':-1, 'n_estimators': 200},
'LGBMRegressor': {'n_estimators': 2000, 'learning_rate': 0.05, 'num_leaves': 8, 'lambda_l2': 0.001},
'LGBMClassifier': {'n_estimators': 2000, 'learning_rate': 0.05, 'num_leaves': 8, 'lambda_l2': 0.001},
'DeepLearningRegressor': {'epochs': epochs, 'batch_size': 50, 'verbose': 2},
'DeepLearningClassifier': {'epochs': epochs, 'batch_size': 50, 'verbose': 2},
'CatBoostRegressor': {},
'CatBoostClassifier': {}
}
model_params = all_model_params.get(model_name, None)
if model_params is None:
model_params = {}
if training_params is not None:
print('Now using the model training_params that you passed in:')
print(training_params)
# Overwrite our stock params with what the user passes in (i.e., if the user wants 10,000 trees, we will let them do it)
model_params.update(training_params)
print('After overwriting our defaults with your values, here are the final params that will be used to initialize the model:')
print(model_params)
model_map = {
# Classifiers
'LogisticRegression': LogisticRegression(),
'RandomForestClassifier': RandomForestClassifier(),
'RidgeClassifier': RidgeClassifier(),
'GradientBoostingClassifier': GradientBoostingClassifier(),
'ExtraTreesClassifier': ExtraTreesClassifier(),
'AdaBoostClassifier': AdaBoostClassifier(),
'SGDClassifier': SGDClassifier(),
'Perceptron': Perceptron(),
'PassiveAggressiveClassifier': PassiveAggressiveClassifier(),
'LinearSVC': LinearSVC(),
# Regressors
'LinearRegression': LinearRegression(),
'RandomForestRegressor': RandomForestRegressor(),
'Ridge': Ridge(),
'LinearSVR': LinearSVR(),
'ExtraTreesRegressor': ExtraTreesRegressor(),
'AdaBoostRegressor': AdaBoostRegressor(),
'RANSACRegressor': RANSACRegressor(),
'GradientBoostingRegressor': GradientBoostingRegressor(),
'Lasso': Lasso(),
'ElasticNet': ElasticNet(),
'LassoLars': LassoLars(),
'OrthogonalMatchingPursuit': OrthogonalMatchingPursuit(),
'BayesianRidge': BayesianRidge(),
'ARDRegression': ARDRegression(),
'SGDRegressor': SGDRegressor(),
'PassiveAggressiveRegressor': PassiveAggressiveRegressor(),
# Clustering
'MiniBatchKMeans': MiniBatchKMeans()
}
if xgb_installed:
model_map['XGBClassifier'] = XGBClassifier()
model_map['XGBRegressor'] = XGBRegressor()
if lgb_installed:
model_map['LGBMRegressor'] = LGBMRegressor()
model_map['LGBMClassifier'] = LGBMClassifier()
if catboost_installed:
model_map['CatBoostRegressor'] = CatBoostRegressor(calc_feature_importance=True)
model_map['CatBoostClassifier'] = CatBoostClassifier(calc_feature_importance=True)
if model_name[:12] == 'DeepLearning':
if keras_imported == False:
# Suppress some level of logs if TF is installed (but allow it to not be installed, and use Theano instead)
try:
os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
from tensorflow import logging
logging.set_verbosity(logging.INFO)
except:
pass
global maxnorm
global Dense, Dropout
global LeakyReLU, PReLU
global Sequential
global keras_load_model
global regularizers
global KerasRegressor, KerasClassifier
from keras.constraints import maxnorm
from keras.layers import Dense, Dropout
from keras.layers.advanced_activations import LeakyReLU, PReLU
from keras.models import Sequential
from keras.models import load_model as keras_load_model
from keras import regularizers
from keras.wrappers.scikit_learn import KerasRegressor, KerasClassifier
maxnorm
Dense
Dropout
LeakyReLU
PReLU
Sequential
keras_load_model
regularizers
KerasRegressor
KerasClassifier
keras_imported = True
model_map['DeepLearningClassifier'] = KerasClassifier(build_fn=make_deep_learning_classifier)
model_map['DeepLearningRegressor'] = KerasRegressor(build_fn=make_deep_learning_model)
try:
model_without_params = model_map[model_name]
except KeyError as e:
print('It appears you are trying to use a library that is not available when we try to import it, or using a value for model_names that we do not recognize')
raise(e)
model_with_params = model_without_params.set_params(**model_params)
return model_with_params
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
def main(argv):
logging.set_verbosity(tf.logging.INFO)
label_map = label_map_util.load_labelmap(FLAGS.label_map)
id2name = {}
for item_id, item in enumerate(label_map.item):
if item.HasField('display_name'):
id2name[item.id - 1] = item.display_name
else:
id2name[item.id - 1] = item.name
with open(FLAGS.output_vocab, 'w') as fp:
for cid, cname in id2name.iteritems():
fp.write('%i\t%s\n' % (cid, cname))
logging.info('Dump %s words', len(id2name))
# Build tensorflow graph.
g = tf.Graph()
with g.as_default():
input_node = tf.placeholder(tf.uint8, shape=(None, None, None, 3))
# Create detection model.
model_proto = _load_model_proto(FLAGS.model_proto)
model = model_builder.build(model_proto, is_training=False)
predictions = model.predict(
model.preprocess(tf.cast(input_node, tf.float32)))
detections = model.postprocess(predictions)
init_fn = slim.assign_from_checkpoint_fn(FLAGS.checkpoint,
tf.global_variables())
uninitialized_variable_names = tf.report_uninitialized_variables()
# Start session to extract video features.
with tf.Session(graph=g) as sess:
init_fn(sess)
warn_names = sess.run(uninitialized_variable_names)
assert len(warn_names) == 0
def _extract_feature(video_id, images_data):
results = sess.run(detections, feed_dict={input_node: images_data})
n_frames = len(images_data)
features = np.zeros((n_frames, FLAGS.num_classes))
det_results = []
for i in xrange(n_frames):
det_result = []
for j in xrange(results['num_detections'][i]):
cid = int(results['detection_classes'][i, j])
cscore = float(results['detection_scores'][i, j])
features[i, cid] = max(cscore, features[i, cid])
y1, x1, y2, x2 = [
round(float(x), 4)
for x in results['detection_boxes'][i, j]
]
if x2 - x1 >= FLAGS.min_object_size and y2 - y1 > FLAGS.min_object_size:
det_result.append({
'cid': cid,
'cname': id2name.get(cid, 'UNK'),
'score': round(cscore, 4),
'bounding_box': {
'x1': x1,
'y1': y1,
'x2': x2,
'y2': y2
}
})
#print(id2name.get(cid, 'UNK'))
#print(cscore)
#vis.image_draw_bounding_box(images_data[i], [x1, y1, x2, y2])
det_results.append(det_result)
#vis.image_save('tmp/%i.jpg' % (i), images_data[i], True)
return features, det_results
# Iterate through video ids.
for video_id in _video_id_iterator(FLAGS.video_id_path):
_extract_video_feature(video_id, FLAGS.video_dir, FLAGS.output_dir,
_extract_feature)
logging.info('Done')
def main(argv):
logging.set_verbosity(tf.logging.INFO)
label_map = label_map_util.load_labelmap(FLAGS.label_map)
id2name = {}
for item_id, item in enumerate(label_map.item):
if item.HasField('display_name'):
id2name[item.id - 1] = item.display_name
else:
id2name[item.id - 1] = item.name
with open(FLAGS.output_vocab, 'w') as fp:
for cid, cname in id2name.iteritems():
fp.write('%i\t%s\n' % (cid, cname))
logging.info('Dump %s words', len(id2name))
# Build tensorflow graph.
g = tf.Graph()
with g.as_default():
input_node = tf.placeholder(tf.uint8, shape=(None, None, None, 3))
# Create detection model.
model_proto = _load_model_proto(FLAGS.model_proto)
model = model_builder.build(model_proto, is_training=False)
predictions = model.predict(
model.preprocess(tf.cast(input_node, tf.float32)))
detections = model.postprocess(predictions)
# Load the latest model checkpoint.
model_path = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
init_fn = slim.assign_from_checkpoint_fn(model_path,
tf.global_variables())
logging.info('Load variables from %s.', model_path)
uninitialized_variable_names = tf.report_uninitialized_variables()
# Start session to extract video features.
with tf.Session(graph=g) as sess:
init_fn(sess)
warn_names = sess.run(uninitialized_variable_names)
assert len(warn_names) == 0
def _extract_feature(video_id, images_data):
results = sess.run(detections, feed_dict={input_node: images_data})
n_frames = len(images_data)
det_results = []
num_classes = len(age_vocab) + len(gender_vocab) + len(
category_vocab)
features = []
vis_dir = os.path.join('tmp', video_id)
if not os.path.isdir(vis_dir):
os.makedirs(vis_dir)
for i in xrange(n_frames):
det_result = []
person_feature_list = []
image_disp = cv2.resize(images_data[i], (512, 512))
for j in xrange(results['num_detections'][i]):
detection_score = results['detection_scores'][i, j]
emotic_ages = results['detection_emotic_ages'][i, j]
emotic_genders = results['detection_emotic_genders'][i, j]
emotic_categories = results['detection_emotic_categories'][
i, j]
person_feature = np.concatenate(
[emotic_ages, emotic_genders, emotic_categories], 0)
person_feature_list.append(person_feature)
age_ind = emotic_ages.argmax()
age_pro = int(100 * emotic_ages[age_ind])
age_str = '{}: {}%'.format(age_vocab[age_ind], age_pro)
gender_ind = emotic_genders.argmax()
gender_pro = int(100 * emotic_genders[gender_ind])
gender_str = '{}: {}%'.format(gender_vocab[gender_ind],
gender_pro)
cat_str_list = []
for cat_ind in emotic_categories.argsort()[::-1]:
cat_pro = int(100 * emotic_categories[cat_ind])
if cat_pro < 20: break
cat_str_list.append('{}: {}%'.format(
category_vocab[cat_ind], cat_pro))
display_str_list = [age_str] + [gender_str] + cat_str_list
# Process max pooling.
cat_list = []
for cat_id, cat_score in enumerate(emotic_categories):
if cat_score > 0.2:
cat_list.append({
'category':
category_vocab[cat_id],
'score':
round(float(emotic_categories[cat_id]), 3)
})
emotic = {
'age': {
'age': age_vocab[emotic_ages.argmax()],
'score': round(float(emotic_ages.max()), 3),
},
'gender': {
'gender': gender_vocab[emotic_genders.argmax()],
'score': round(float(emotic_genders.max()), 3),
},
'categories': cat_list
}
y1, x1, y2, x2 = [
round(float(x), 4)
for x in results['detection_boxes'][i, j]
]
if x2 - x1 >= FLAGS.min_object_size and y2 - y1 > FLAGS.min_object_size:
det_result.append({
'score':
round(float(results['detection_scores'][i, j]), 4),
'emotic':
emotic,
'bounding_box': {
'x1': x1,
'y1': y1,
'x2': x2,
'y2': y2
}
})
# vis.image_draw_bounding_box(image_disp, [x1, y1, x2, y2])
# vis.image_draw_text(image_disp,
# [x1, y1],
# '{}%'.format(int(100 * detection_score)),
# (0, 0, 0))
# text_label_height = _get_text_label_height()
# for disp_ind, display_str in enumerate(display_str_list):
# vis.image_draw_text(image_disp,
# [x1, y1 + 1.0 * text_label_height / image_disp.shape[0] * (disp_ind + 1)],
# display_str,
# color=(0, 0, 0),
# bkg_color=(0, 0, 0))
det_results.append(det_result)
features.append(person_feature_list)
#vis.image_save('%s/%i.jpg' % (vis_dir, i), image_disp, True)
return features, det_results
# Iterate through video ids.
for video_id in _video_id_iterator(FLAGS.video_id_path):
_extract_video_feature(video_id, FLAGS.video_dir, FLAGS.output_dir,
_extract_feature)
logging.info('Done')
def __init__(self, name, params={}, use_gpu=True, verbose=True):
"""Parameter initialization.
Arguments
-----------
- name. String which will be used to identify the
model in any folders or files created.
- params. Optional. Dictionary containing the parameters
that the user whishes to specify.
- use_gpu. Boolean specifying whether a GPU should be
used. True by default.
- verbose. Boolean specifying whether output must be
produced in-line.
"""
self.verbose = verbose
# Print logo. Isn't it cool?
# (Although it is cool, we won't print it if you don't want)
if self.verbose:
print(__logo__.format(__version__))
# Set minimum verbosity for RDKit, Keras and TF backends
os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
logging.set_verbosity(logging.INFO)
rdBase.DisableLog('rdApp.error')
# Set configuration for GPU
self.config = tf.ConfigProto()
self.config.gpu_options.allow_growth = True
# Set parameters
self.PREFIX = name
if 'PRETRAIN_GEN_EPOCHS' in params:
self.PRETRAIN_GEN_EPOCHS = params['PRETRAIN_GEN_EPOCHS']
else:
self.PRETRAIN_GEN_EPOCHS = 240
if 'PRETRAIN_DIS_EPOCHS' in params:
self.PRETRAIN_DIS_EPOCHS = params['PRETRAIN_DIS_EPOCHS']
else:
self.PRETRAIN_DIS_EPOCHS = 50
if 'GEN_ITERATIONS' in params:
self.GEN_ITERATIONS = params['GEN_ITERATIONS']
else:
self.GEN_ITERATIONS = 2
if 'GEN_BATCH_SIZE' in params:
self.GEN_BATCH_SIZE = params['GEN_BATCH_SIZE']
else:
self.GEN_BATCH_SIZE = 64
if 'SEED' in params:
self.SEED = params['SEED']
else:
self.SEED = None
random.seed(self.SEED)
np.random.seed(self.SEED)
if 'DIS_BATCH_SIZE' in params:
self.DIS_BATCH_SIZE = params['DIS_BATCH_SIZE']
else:
self.DIS_BATCH_SIZE = 64
if 'DIS_EPOCHS' in params:
self.DIS_EPOCHS = params['DIS_EPOCHS']
else:
self.DIS_EPOCHS = 3
if 'EPOCH_SAVES' in params:
self.EPOCH_SAVES = params['EPOCH_SAVES']
else:
self.EPOCH_SAVES = 20
if 'CHK_PATH' in params:
self.CHK_PATH = params['CHK_PATH']
else:
self.CHK_PATH = os.path.join(os.getcwd(),
'checkpoints/{}'.format(self.PREFIX))
if 'GEN_EMB_DIM' in params:
self.GEN_EMB_DIM = params['GEN_EMB_DIM']
else:
self.GEN_EMB_DIM = 32
if 'GEN_HIDDEN_DIM' in params:
self.GEN_HIDDEN_DIM = params['GEN_HIDDEN_DIM']
else:
self.GEN_HIDDEN_DIM = 32
if 'START_TOKEN' in params:
self.START_TOKEN = params['START_TOKEN']
else:
self.START_TOKEN = 0
if 'SAMPLE_NUM' in params:
self.SAMPLE_NUM = params['SAMPLE_NUM']
else:
self.SAMPLE_NUM = 6400
if 'BIG_SAMPLE_NUM' in params:
self.BIG_SAMPLE_NUM = params['BIG_SAMPLE_NUM']
else:
self.BIG_SAMPLE_NUM = self.SAMPLE_NUM * 5
if 'LAMBDA' in params:
self.LAMBDA = params['LAMBDA']
else:
self.LAMBDA = 0.5
# In case this parameter is not specified by the user,
# it will be determined later, in the training set
# loading.
if 'MAX_LENGTH' in params:
self.MAX_LENGTH = params['MAX_LENGTH']
if 'DIS_EMB_DIM' in params:
self.DIS_EMB_DIM = params['DIS_EMB_DIM']
else:
self.DIS_EMB_DIM = 64
if 'DIS_FILTER_SIZES' in params:
self.DIS_FILTER_SIZES = params['DIS_FILTER_SIZES']
else:
self.DIS_FILTER_SIZES = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20]
if 'DIS_NUM_FILTERS' in params:
self.DIS_NUM_FILTERS = params['DIS_FILTER_SIZES']
else:
self.DIS_NUM_FILTERS = [
100, 200, 200, 200, 200, 100, 100, 100, 100, 100, 160, 160
]
if 'DIS_DROPOUT' in params:
self.DIS_DROPOUT = params['DIS_DROPOUT']
else:
self.DIS_DROPOUT = 0.75
if 'DIS_L2REG' in params:
self.DIS_L2REG = params['DIS_L2REG']
else:
self.DIS_L2REG = 0.2
self.AV_METRICS = get_metrics()
self.LOADINGS = metrics_loading()
self.PRETRAINED = False
self.SESS_LOADED = False
self.USERDEF_METRIC = False
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
inference_loop()
import os
import tensorflow as tf
from backend.container_fillers.generator_queue_fillers import GeneratorInputV3
from tensorflow import logging as logging
logging.set_verbosity(tf.logging.INFO)
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
def train_generator_input_fn_v3(questions, non_questions, input_fn_params,
graph_params):
" Main input function -- use this one "
kwargs = {
'questions': questions,
'non_questions': non_questions,
'Q_size': 700,
'num_proc': 6,
'word2index': graph_params['word2index'],
'max_seq_length': graph_params['input_max_length'],
'max_num_questions': input_fn_params['max_num_questions'],
'max_num_elements': input_fn_params['max_num_elements'],
'randomize_num_questions': input_fn_params['randomize_num_questions']
}
gen_obj = GeneratorInputV3(
**kwargs) # This should take max a few seconds to prefill the queue
generator = gen_obj.from_queue_generator()
msl = graph_params['input_max_length']
bsize = input_fn_params['batch_size']
def main(unused_argv):
logging.set_verbosity(logging.INFO)
start_new_model = FLAGS.start_new_model
output_dir = FLAGS.output_dir
init_learning_rate = FLAGS.init_learning_rate
decay_steps = FLAGS.decay_steps
decay_rate = FLAGS.decay_rate
l1_reg_rate = FLAGS.l1_reg_rate
l2_reg_rate = FLAGS.l2_reg_rate
is_bootstrap = FLAGS.is_bootstrap
train_epochs = FLAGS.train_epochs
model_type, feature_names, feature_sizes = FLAGS.model_type, FLAGS.feature_names, FLAGS.feature_sizes
reader = get_reader(model_type, feature_names, feature_sizes)
train_data_pattern = FLAGS.train_data_pattern
validate_data_pattern = FLAGS.validate_data_pattern
batch_size = FLAGS.batch_size
num_readers = FLAGS.num_readers
if tf.gfile.Exists(path_join(output_dir, 'validate_data.pickle')):
with open(path_join(output_dir, 'validate_data.pickle'), 'rb') as f:
validate_data = pickle.load(f)
with open(path_join(output_dir, 'validate_labels.pickle'), 'rb') as f:
validate_labels = pickle.load(f)
else:
# Increase num_readers.
validate_data_pipeline = DataPipeline(
reader=reader,
data_pattern=validate_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
# Sample validate set.
_, validate_data, validate_labels, _ = random_sample(
0.05,
mask=(False, True, True, False),
data_pipeline=validate_data_pipeline,
name_scope='sample_validate')
with open(path_join(output_dir, 'validate_data.pickle'), 'wb') as f:
pickle.dump(validate_data, f)
with open(path_join(output_dir, 'validate_labels.pickle'), 'wb') as f:
pickle.dump(validate_labels, f)
train_data_pipeline = DataPipeline(reader=reader,
data_pattern=train_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
model_save_path = path_join(output_dir, 'mlp_fuse')
if start_new_model and tf.gfile.Exists(model_save_path):
logging.info('Starting a new model...')
# Start new model, delete existing checkpoints.
try:
tf.gfile.DeleteRecursively(model_save_path)
except tf.errors.OpError:
logging.error('Failed to delete dir {}.'.format(model_save_path))
else:
logging.info(
'Succeeded to delete train dir {}.'.format(model_save_path))
# Set pos_weights for extremely imbalanced situation in one-vs-all classifiers.
try:
# Load sum_labels in training set, numpy float format to compute pos_weights.
train_sum_labels = load_sum_labels()
# num_neg / num_pos, assuming neg_weights === 1.0.
pos_weights = np.sqrt(
float(NUM_TRAIN_EXAMPLES) / train_sum_labels - 1.0)
logging.info(
'Computing pos_weights based on sum_labels in train set successfully.'
)
except IOError:
logging.error('Cannot load train sum_labels. Use default value.')
pos_weights = None
finally:
logging.warn('Not to use positive weights.')
pos_weights = None
train(train_data_pipeline,
epochs=train_epochs,
pos_weights=pos_weights,
l1_reg_rate=l1_reg_rate,
l2_reg_rate=l2_reg_rate,
init_learning_rate=init_learning_rate,
bootstrap=is_bootstrap,
validate_set=(validate_data, validate_labels),
validate_fn=gap_fn,
logdir=model_save_path)
def _basic_setup(self, config):
'''
基本的配置TPU通信协议,如果use_tpu=False,或略一切关于tpu的配置
:return:
'''
if self.use_tpu:
tpu_name = config.get('tpu_name') if 'tpu_name' in config else None
tpu_zone = config.get('tpu_zone') if 'tpu_zone' in config else None
try:
self.tpu = tfcontrib.cluster_resolver.TPUClusterResolver(
tpu=tpu_name, zone=tpu_zone)
info = self.tpu.cluster_spec().as_dict()
tflogging.info('Run on TPU:%s' % (','.join(info['worker'])))
except:
raise ValueError('当前环境不支持TPU')
tpu_iterations = config.get(
'tpu_iterations') if 'tpu_iterations' in config else None
self.tpu_run_config = tfcontrib.tpu.TPUConfig(tpu_iterations)
tpu_master = config.get(
'tpu_master') if 'tpu_master' in config else None
self.run_config = tfcontrib.tpu.RunConfig(tpu_config=config,
master=tpu_master,
cluster=self.tpu)
else:
self.run_config = tfcontrib.tpu.RunConfig()
if __name__ == '__main__':
tflogging.set_verbosity(tflogging.INFO)
TPU_Solver('config.json')
def main(_):
logging.set_verbosity(tf.logging.INFO)
assert os.path.isfile(FLAGS.pipeline_proto), FLAGS.pipeline_proto
g = tf.Graph()
with g.as_default():
pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto)
logging.info("Pipeline configure: %s", '=' * 128)
logging.info(pipeline_proto)
train_config = pipeline_proto.train_config
# Get examples from reader.
examples = ads_examples.get_examples(pipeline_proto.example_reader)
# Build model for training.
global_step = slim.get_or_create_global_step()
model = builder.build(pipeline_proto.model, is_training=True)
predictions = model.build_inference(examples)
loss_dict = model.build_loss(predictions)
init_fn = model.get_init_fn()
uninitialized_variable_names = tf.report_uninitialized_variables()
# Loss and optimizer.
for loss_name, loss_tensor in loss_dict.iteritems():
tf.losses.add_loss(loss_tensor)
tf.summary.scalar('losses/{}'.format(loss_name), loss_tensor)
total_loss = tf.losses.get_total_loss()
tf.summary.scalar('losses/total_loss', total_loss)
optimizer = train_utils.build_optimizer(train_config)
if train_config.moving_average:
optimizer = tf.contrib.opt.MovingAverageOptimizer(optimizer,
average_decay=0.99)
gradient_multipliers = train_utils.build_multipler(
train_config.gradient_multiplier)
variables_to_train = model.get_variables_to_train()
for var in variables_to_train:
logging.info(var)
train_op = slim.learning.create_train_op(total_loss,
variables_to_train=variables_to_train,
clip_gradient_norm=5.0,
gradient_multipliers=gradient_multipliers,
summarize_gradients=True,
optimizer=optimizer)
saver = None
if train_config.moving_average:
saver = optimizer.swapping_saver()
# Starts training.
logging.info('Start training.')
session_config = train_utils.default_session_config(
FLAGS.per_process_gpu_memory_fraction)
slim.learning.train(train_op,
logdir=FLAGS.train_log_dir,
graph=g,
master='',
is_chief=True,
number_of_steps=train_config.number_of_steps,
log_every_n_steps=train_config.log_every_n_steps,
save_interval_secs=train_config.save_interval_secs,
save_summaries_secs=train_config.save_summaries_secs,
session_config=session_config,
init_fn=init_fn,
saver=saver)
logging.info('Done')
main of salted fish
"""
import os
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, recall_score, precision_score
from tensorflow import logging
from models import Cnn1DMultiChannelModel, CnnWithEmbedding, \
CnnWithSingleDayStatus, CnnWithStatusAutoEncode, \
CnnWithSingleDayStatusEncode
from util import timer
from util.utils import save_figure
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
logging.set_verbosity(logging.WARN)
def top_n_precision(y_pred, y_true, codes):
pred_label_code_lst = list(zip(y_pred, y_true, codes))
sorted_lst = sorted(pred_label_code_lst, key=lambda x: x[0], reverse=True)
for pred, label, s_code in sorted_lst[:10]:
print(f"pred_prob: {pred}, label: {label}, stock_id: {s_code}")
for n in [1, 3, 5, 10, 15, 20]:
res_lst = sorted_lst[:n]
if res_lst[-1][0] < 0.5:
# raise Exception(f"{n}th prob < 0.5")
print(f"{n}th prob < 0.5")
correct_cnt = sum([x[1] for x in res_lst])
from keras.models import Model
from keras.layers import Input, Add, PReLU, Conv2DTranspose, Concatenate, MaxPooling2D, UpSampling2D, Dropout, concatenate
from keras.layers.convolutional import Conv2D
from keras.layers.normalization import BatchNormalization
from keras.callbacks import Callback
from tensorflow import log
from keras import backend as K
from tensorflow.logging import set_verbosity, FATAL
set_verbosity(FATAL)
def get_red30_model():
"""
Define the model architecture
Return the model
"""
inputs = Input(shape=(256, 256, 2))
enc_conv0 = Conv2D(32, (3, 3),
padding="same",
activation='relu',
kernel_initializer="he_normal")(inputs)
enc_conv1 = Conv2D(32, (3, 3), padding="SAME",
activation='relu')(enc_conv0)
max_pool1 = MaxPooling2D((2, 2))(enc_conv1)
enc_conv2 = Conv2D(32, (3, 3), padding="SAME",
activation='relu')(max_pool1)
def get_model_from_name(model_name, training_params=None, is_hp_search=False):
# For Keras
epochs = 1000
# if os.environ.get('is_test_suite', 0) == 'True' and model_name[:12] == 'DeepLearning':
# print('Heard that this is the test suite. Limiting number of epochs, which will increase '
# 'training speed dramatically at the expense of model accuracy')
# epochs = 100
all_model_params = {
'LogisticRegression': {},
'RandomForestClassifier': {
'n_jobs': -2,
'n_estimators': 30
},
'ExtraTreesClassifier': {
'n_jobs': -1
},
'AdaBoostClassifier': {},
'SGDClassifier': {
'n_jobs': -1
},
'Perceptron': {
'n_jobs': -1
},
'LinearSVC': {
'dual': False
},
'LinearRegression': {
'n_jobs': -2
},
'RandomForestRegressor': {
'n_jobs': -2,
'n_estimators': 30
},
'LinearSVR': {
'dual': False,
'loss': 'squared_epsilon_insensitive'
},
'ExtraTreesRegressor': {
'n_jobs': -1
},
'MiniBatchKMeans': {
'n_clusters': 8
},
'GradientBoostingRegressor': {
'presort': False,
'learning_rate': 0.1,
'warm_start': True
},
'GradientBoostingClassifier': {
'presort': False,
'learning_rate': 0.1,
'warm_start': True
},
'SGDRegressor': {
'shuffle': False
},
'PassiveAggressiveRegressor': {
'shuffle': False
},
'AdaBoostRegressor': {},
'LGBMRegressor': {
'n_estimators': 2000,
'learning_rate': 0.15,
'num_leaves': 8,
'lambda_l2': 0.001,
'histogram_pool_size': 16384
},
'LGBMClassifier': {
'n_estimators': 2000,
'learning_rate': 0.15,
'num_leaves': 8,
'lambda_l2': 0.001,
'histogram_pool_size': 16384
},
'DeepLearningRegressor': {
'epochs': epochs,
'batch_size': 50,
'verbose': 2
},
'DeepLearningClassifier': {
'epochs': epochs,
'batch_size': 50,
'verbose': 2
},
'CatBoostRegressor': {},
'CatBoostClassifier': {}
}
# if os.environ.get('is_test_suite', 0) == 'True':
# all_model_params
model_params = all_model_params.get(model_name, None)
if model_params is None:
model_params = {}
if is_hp_search is True:
if model_name[:12] == 'DeepLearning':
model_params['epochs'] = 50
if model_name[:4] == 'LGBM':
model_params['n_estimators'] = 500
if training_params is not None:
print('Now using the model training_params that you passed in:')
print(training_params)
# Overwrite our stock params with what the user passes in (i.e., if the user wants 10,
# 000 trees, we will let them do it)
model_params.update(training_params)
print(
'After overwriting our defaults with your values, here are the final params that will '
'be used to initialize the model:')
print(model_params)
model_map = {
# Classifiers
'LogisticRegression': LogisticRegression(),
'RandomForestClassifier': RandomForestClassifier(),
'RidgeClassifier': RidgeClassifier(),
'GradientBoostingClassifier': GradientBoostingClassifier(),
'ExtraTreesClassifier': ExtraTreesClassifier(),
'AdaBoostClassifier': AdaBoostClassifier(),
'LinearSVC': LinearSVC(),
# Regressors
'LinearRegression': LinearRegression(),
'RandomForestRegressor': RandomForestRegressor(),
'Ridge': Ridge(),
'LinearSVR': LinearSVR(),
'ExtraTreesRegressor': ExtraTreesRegressor(),
'AdaBoostRegressor': AdaBoostRegressor(),
'RANSACRegressor': RANSACRegressor(),
'GradientBoostingRegressor': GradientBoostingRegressor(),
'Lasso': Lasso(),
'ElasticNet': ElasticNet(),
'LassoLars': LassoLars(),
'OrthogonalMatchingPursuit': OrthogonalMatchingPursuit(),
'BayesianRidge': BayesianRidge(),
'ARDRegression': ARDRegression(),
# Clustering
'MiniBatchKMeans': MiniBatchKMeans(),
}
try:
model_map['SGDClassifier'] = SGDClassifier(max_iter=1000, tol=0.001)
model_map['Perceptron'] = Perceptron(max_iter=1000, tol=0.001)
model_map['PassiveAggressiveClassifier'] = PassiveAggressiveClassifier(
max_iter=1000, tol=0.001)
model_map['SGDRegressor'] = SGDRegressor(max_iter=1000, tol=0.001)
model_map['PassiveAggressiveRegressor'] = PassiveAggressiveRegressor(
max_iter=1000, tol=0.001)
except TypeError:
model_map['SGDClassifier'] = SGDClassifier()
model_map['Perceptron'] = Perceptron()
model_map['PassiveAggressiveClassifier'] = PassiveAggressiveClassifier(
)
model_map['SGDRegressor'] = SGDRegressor()
model_map['PassiveAggressiveRegressor'] = PassiveAggressiveRegressor()
if xgb_installed:
model_map['XGBClassifier'] = XGBClassifier()
model_map['XGBRegressor'] = XGBRegressor()
if lgb_installed:
model_map['LGBMRegressor'] = LGBMRegressor()
model_map['LGBMClassifier'] = LGBMClassifier()
if catboost_installed:
model_map['CatBoostRegressor'] = CatBoostRegressor()
model_map['CatBoostClassifier'] = CatBoostClassifier()
if model_name[:12] == 'DeepLearning':
if keras_installed is False:
# Suppress some level of logs if TF is installed (but allow it to not be installed,
# and use Theano instead)
try:
os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
from tensorflow import logging
logging.set_verbosity(logging.INFO)
except:
# TODO: Fix bare Except
pass
model_map['DeepLearningClassifier'] = KerasClassifier(
build_fn=make_deep_learning_classifier)
model_map['DeepLearningRegressor'] = KerasRegressor(
build_fn=make_deep_learning_model)
try:
model_without_params = model_map[model_name]
except KeyError as e:
print(
'It appears you are trying to use a library that is not available when we try to '
'import it, or using a value for model_names that we do not recognize.'
)
raise e
if os.environ.get('is_test_suite', False) == 'True':
if 'n_jobs' in model_params:
model_params['n_jobs'] = 1
model_with_params = model_without_params.set_params(**model_params)
return model_with_params
def main(unused_argv):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpus
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
name="batchnorm_params")
LAYER_CONFIG_ORDER = StructNode(
(("in_channels", -1), ("filter_width", -1), ("filter_height",
-1), ("out_channels", -1),
("skip_width", 1), ("skip_height", 1), ("maxpool", 0), ("weights", ""),
("bias", ""), ("pad", "PADDING_SAME_ZERO"), ("activation", "NONE"),
("deconvolve", 0), ("branch", 0), ("branch_copy_type", "BRANCH_NO_COPY"),
("branch_combine_type", "BRANCH_NOC"), ("branch_config",
BRANCH_CONFIG_ORDER),
("bn_params", BATCHNORM_PARAMS_ORDER), ("output_num", -1)))
CNN_CONFIG_ORDER = {}
BIT_ALIGNMENT = 32
WEIGHT_STRING = "w"
BIAS_STRING = "b"
logging.set_verbosity("INFO")
def _get_weight_tensor_from_prefix(tensor_prefix):
return tensor_prefix + WEIGHT_STRING
def _get_bias_tensor_from_prefix(tensor_prefix):
return tensor_prefix + BIAS_STRING
def _generate_layer_index_tensor_name_map(input_reader, var_regex):
"""Create a map from tuples of indices to tensor names."""
layer_index_tensor_name_map = {}
# Create a index-tuple to tensor name map.
for k, _ in input_reader.get_variable_to_shape_map().iteritems():
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
is_chief = (FLAGS.task == 0)
# Recover session
saver = None
latest_checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if FLAGS.start_new_model:
logging.info("'start_new_model' flag is set. Removing existing train dir.")
try:
gfile.DeleteRecursively(FLAGS.train_dir)
except:
logging.error(
"Failed to delete directory " + FLAGS.train_dir +
" when starting a new model. Please delete it manually and" +
" try again.")
elif not latest_checkpoint:
logging.info("No checkpoint file found. Building a new model.")
else:
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("No meta graph file found. Building a new model.")
else:
logging.info("Restoring from meta graph file %s", meta_filename)
saver = tf.train.import_meta_graph(meta_filename)
if not saver:
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes)
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(reader=reader,
model=model,
optimizer_class=optimizer_class,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built graph")
saver = tf.train.Saver()
train_loop(is_chief=is_chief,
train_dir=FLAGS.train_dir,
saver=saver,
master=FLAGS.master)
running_reward / batch_size))
if reward_sum / batch_size > 200:
logging.info("Task solved in", episode_number,
'episodes!')
break
reward_sum = 0
time_to_export = ((last_model_export_step == 0) or
(global_step_val - last_model_export_step
>= export_model_steps))
if is_master and time_to_export:
last_checkpoint = saver.save(sess, sv.save_path,
global_step_val)
last_model_export_step = global_step_val
observation = env.reset()
if is_master:
last_checkpoint = saver.save(sess, sv.save_path,
global_step_val)
last_model_export_step = global_step_val
model.after()
if __name__ == "__main__":
logging.set_verbosity(logging.DEBUG)
app.run()
