def db_trainer(estimator, dataset, build_name, limit, db_uri, evaluate):
runs = gather_results.get_runs_by_name(db_uri, build_name=build_name)
model_config = {'build_name': build_name}
gather_results.save_model_config(dataset, model_config)
if limit > 0:
runs = runs[:limit]
gather_results.save_run_uuids(dataset, runs)
for run in runs:
if estimator == 'tf.estimator.DNNClassifier':
gather_results.get_subunit_results_for_run(run,
'1s',
db_uri,
use_cache=True)
print('Acquired run %s' % run.uuid)
else:
result = gather_results.get_subunit_results_for_run(
run, '1s', db_uri)[0]
print('Acquired run %s' % run.uuid)
try:
features, labels = nn_trainer.normalize_data(result)
except TypeError:
print('Unable to normalize data in run %s, '
'skipping' % run.uuid)
continue
if not evaluate:
nn_trainer.train_model(features, labels, dataset_name=dataset)
else:
nn_trainer.evaluate_model(features,
labels,
dataset_name=dataset)
def data_sizes_and_labels(sample_run,
features_regex,
sample_interval='1s',
data_path=None,
s3=None):
"""Takes a sample run from a dataset and filters and does calculations
Returns:
- the normalized example lenght
- the number of dstat features
- the unrolled labels
"""
# Normalized lenght before resampling
normalized_length = 5500
if sample_interval:
# Calculate the desired normalized lenght after resample
normalized_length = get_downsampled_example_lenght(
sample_interval, normalized_length)
# Load the list of runs and base labels
sample_result = gather_results.get_subunit_results_for_run(
sample_run, sample_interval, data_path=data_path, s3=s3)
filtered_sample_result = filter_example(sample_result, features_regex)
filtered_dstat_labels = filtered_sample_result['dstat'].columns
unrolled_labels = unroll_labels(filtered_dstat_labels, normalized_length)
return normalized_length, len(filtered_dstat_labels), unrolled_labels
def run_nn_training():
for run in runs:
uuid = run.uuid
result = gather_results.get_subunit_results_for_run(
run, '1s', session=session, use_cache=False,
data_path=model_dir)[0]
try:
features, labels = nn_trainer.normalize_data(result)
except TypeError:
print('Unable to normalize data in run %s, '
'skipping' % uuid)
continue
nn_trainer.train_model(features, labels,
dataset_name=dataset,
model_path=model_dir)
print('done')
def prepare_dataset(dataset,
normalized_length,
num_dstat_features,
data_type,
features_regex,
sample_interval='1s',
class_label='status',
aggregation_functions=None,
visualize=False,
data_path=None,
target_data_path=None,
s3=None):
"""Takes a dataset and filters and does the magic
Loads the run ids from the dataset configuration.
Loads the data (dsv + meta) for every run from cache.
Builds the unrolled examples as a numpy ndarray.
Builds the classes as a numpy array.
Saves the data setup to the dataset config.
Does some visualization (if enabled).
"""
if visualize:
data_plots_folder = [
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'data',
dataset, 'plots'
]
os.makedirs(os.sep.join(data_plots_folder), exist_ok=True)
# Load the list of runs and base labels
runs = gather_results.load_run_uuids(dataset,
name=data_type,
data_path=target_data_path,
s3=s3)
# run_uuids are the example_ids
sizes = []
# The data for each example.
examples = examples_ndarray(len(runs), num_dstat_features,
normalized_length)
# The test result for each example
classes = []
skips = []
print("Loading %s data:" % data_type, end='\r', flush=True)
for count, run in enumerate(runs):
print("Loading %s data: %d of %d" % (data_type, count + 1, len(runs)),
end='\r',
flush=True)
result = gather_results.get_subunit_results_for_run(
run, sample_interval, data_path=data_path, s3=s3)
# For one run_uuid we must only get on example (result)
# Filtering by columns
if not result:
skips.append(run.uuid)
continue
# Apply column filtering
result = filter_example(result, features_regex)
# Normalize data
example = fixed_lenght_example(result, normalized_length,
aggregation_functions)
vector = unroll_example(example, normalized_length)
# Normalize status
status = get_class(result, class_label)
# Examples is an np ndarrays
examples[count] = vector.values
classes.append(status)
# Plot from figures
if visualize and not aggregation_functions:
# Prepare some more data if we are going to visualize
sizes.append((result['dstat'].shape[0], status))
figure_name = sample_interval + "_%s_" + str(count)
# Plot un-normalized data
data_plot = result['dstat'].plot()
fig = data_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "downsampled"]))
plt.close(fig)
# Plot fixed size data
fixed_plot = example.plot()
fig = fixed_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "fixedsize"]))
plt.close(fig)
# Plot unrolled data
unrolled_plot = pd.Series(vector).plot()
fig = unrolled_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "unrolled"]))
plt.close(fig)
print("Loading %s data: %d done!" % (data_type, len(runs)))
# Check that everything went well
if len(skips) > 0:
print('Unable to train model because of missing runs %s' % skips)
safe_runs = [run.uuid for run in runs if run.uuid not in skips]
gather_results.save_run_uuids(dataset, safe_runs)
print('The model has been updated to exclude those runs.')
print('Please re-run the training step.')
sys.exit(1)
classes = np.array(classes)
figure_sizes = np.array(sizes)
example_ids = np.array(runs)
print("%s set: examples: %s, classes: %s, example IDs: %s" %
(data_type, str(examples.shape), str(
classes.shape), str(example_ids.shape)))
data = {
'examples': examples,
'example_ids': example_ids,
'classes': classes
}
if visualize and aggregation_functions and len(examples) > 0:
if len(aggregation_functions) > 3:
print('Visualization skipped, cannot represent more than 3D')
sys.exit(1)
else:
fig = plt.figure()
if len(aggregation_functions) == 3:
ax = fig.add_subplot(111, projection='3d')
else:
ax = fig.add_subplot(111)
# Build a dict [class] -> [int ID]
unique_classes = list(set(classes))
dict_classes = dict(
zip(unique_classes, list(range(len(unique_classes)))))
# Setup colours
cm = plt.get_cmap('jet')
cNorm = pltcolors.Normalize(vmin=0, vmax=len(unique_classes))
scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=cm)
# Scatter the data
for ii in range(len(examples)):
ax.scatter(*examples[ii],
marker='o',
c=scalarMap.to_rgba(dict_classes[classes[ii]]))
# Set axis labels
ax.set_xlabel(aggregation_functions[0].__name__)
if len(aggregation_functions) > 1:
ax.set_ylabel(aggregation_functions[1].__name__)
if len(aggregation_functions) > 2:
ax.set_zlabel(aggregation_functions[2].__name__)
# scalarMap.set_array(classes)
# fig.colorbar(scalarMap)
# Save the plot
fig.savefig(
os.sep.join(data_plots_folder + [data_type + "_3d_plot"]))
plt.close(fig)
return data, figure_sizes
def db_batch_predict(db_uri, dataset, slice, gpu, debug):
"""Run predict on all DB items on included in the dataset yet
Takes a dataset and a build name. It builds the list of runs in the DB
that fit the specified build name, and that are not yet used for training
in the specified dataset. It runs prediction on all of them.
"""
if debug:
tf.logging.set_verbosity(tf.logging.DEBUG)
# Get the configuration for the model
model_config = gather_results.load_model_config(dataset)
# Get the list of runs from the dataset
run_uuids = gather_results.load_run_uuids(dataset)
# Get the list of runs from the DB
runs = gather_results.get_runs_by_name(
db_uri=db_uri, build_name=model_config['build_name'])
# Run a predict loop, include all runs not in the train dataset
predict_runs = [r for r in runs if r.uuid not in run_uuids]
predict_runs = predict_runs[]
if len(predict_runs) == 0:
print("Empty prediction set, nothing to do.")
sys.exit(0)
# Initialize the array
examples = np.ndarray(
shape=(len(predict_runs), model_config['num_features']))
idx = 0
classes = []
labels = []
print("All runs: %d, dataset size: %d, predict size: %d" % (
len(runs), len(run_uuids), len(predict_runs)))
for run in predict_runs:
# This will also store new runs in cache. In future we may want to
# train on those as well, but nor now let's try to predict only
results = gather_results.get_subunit_results_for_run(
run, model_config['sample_interval'], db_uri=db_uri)
for result in results:
# Skip runs with no data
if result is None:
continue
if model_config['features_regex']:
df = result['dstat']
col_regex = re.compile(model_config['features_regex'])
result['dstat'] = df[list(filter(
col_regex.search, df.columns))]
# Normalize examples
vector, status, labels = trainer.normalize_example(
result, model_config['normalized_length'],
model_config['labels'])
examples[idx] = vector.values
classes.append(status)
idx += 1
# Normalize dataset
n_examples, _ = trainer.normalize_dataset(
examples, labels, params=model_config['normalization_params'])
# Prepare other arrays
classes = np.array(classes)
run_uuids = [r.uuid for r in predict_runs]
# Configure TF
config = tf.ConfigProto(log_device_placement=True,)
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# Now do the prediction
model = svm_trainer.SVMTrainer(n_examples, run_uuids, labels,
classes, dataset_name=dataset,
force_gpu=gpu)
predictions = model.predict()
errors = []
for prediction, actual in zip(predictions, classes):
if prediction['classes'] != actual:
errors.append((prediction, actual))
print("Prediction of %d inputs completed." % len(classes))
print("Input set composition: %d PASS, %s FAIL" % (
len([x for x in classes if x == 0]),
len([x for x in classes if x == 1])))
if len(errors) > 0:
print("There were some prediction errors: %s" % errors)
else:
print("All predicted correctly.")
def local_trainer(train, estimator, dataset, sample_interval, features_regex,
class_label, visualize, steps, gpu, debug):
# Normalized lenght before resampling
normalized_length = 5500
if sample_interval:
# Calculate the desired normalized lenght after resample
normalized_length = get_downsampled_example_lenght(
sample_interval, normalized_length)
data_plots_folder = [
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'data',
dataset, 'plots'
]
os.makedirs(os.sep.join(data_plots_folder), exist_ok=True)
runs = gather_results.load_run_uuids(dataset)
# run_uuids are the example_ids
sizes = []
# The data for each example. We don't know yet the pre-set shape, so
# wait until the first result comes in
examples = []
# Model configuration. We need to cache sample_interval, features-regex and
# the normalization parameters for each feature so we can re-use them
# during prediction.
model_config = {
'sample_interval': sample_interval,
'features_regex': features_regex,
'normalized_length': normalized_length
}
# The test result for each example
classes = []
labels = []
idx = 0
skips = []
for run in runs:
results = gather_results.get_subunit_results_for_run(
run, sample_interval)
# For one run_uuid we must only get on example (result)
result = results[0]
# Filtering by columns
if not result:
skips.append(run.uuid)
continue
df = result['dstat']
if features_regex:
col_regex = re.compile(features_regex)
result['dstat'] = df[list(filter(col_regex.search, df.columns))]
# Setup the numpy matrix and sizes
if len(examples) == 0:
# Adjust normalized_length to the actual re-sample one
examples = np.ndarray(shape=(len(runs),
len(result['dstat'].columns) *
normalized_length))
model_config['num_columns'] = len(result['dstat'].columns)
model_config['num_features'] = (len(result['dstat'].columns) *
normalized_length)
# Normalize data
example = fixed_lenght_example(result, normalized_length)
# Normalize status
status = get_class(result, class_label)
vector, new_labels = unroll_example(example, normalized_length, labels)
# Only calculate labels for the first example
if len(labels) == 0:
labels = new_labels
model_config['labels'] = labels
print("Normalized example %d of %d" % (runs.index(run) + 1, len(runs)),
end='\r',
flush=True)
# Examples is an np ndarrays
examples[idx] = vector.values
classes.append(status)
if visualize:
# Prepare some more data if we are going to visualize
sizes.append((result['dstat'].shape[0], status))
figure_name = sample_interval + "_%s_" + str(idx)
# Plot un-normalized data
data_plot = result['dstat'].plot()
fig = data_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "downsampled"]))
plt.close(fig)
# Plot fixed size data
fixed_plot = example.plot()
fig = fixed_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "fixedsize"]))
plt.close(fig)
# Plot unrolled data
unrolled_plot = pd.Series(vector).plot()
fig = unrolled_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "unrolled"]))
plt.close(fig)
idx += 1
if len(skips) > 0:
print('Unable to train model because of missing runs %s' % skips)
safe_runs = [run for run in runs if run.uuid not in skips]
gather_results.save_run_uuids(dataset, safe_runs)
print('The model has been updated to exclude those runs.')
print('Please re-run the training step.')
sys.exit(1)
# Perform dataset-wise normalization
# NOTE(andreaf) When we train the model we ignore any saved normalization
# parameter, since the sample interval and features may be different.
n_examples, normalization_params = normalize_dataset(examples, labels)
# We do cache the result to normalize the prediction set.
model_config['normalization_params'] = normalization_params
gather_results.save_model_config(dataset, model_config)
if visualize:
for n in range(len(runs)):
figure_name = sample_interval + "_%s_" + str(n)
unrolled_norm_plot = pd.Series(n_examples[n]).plot()
fig = unrolled_norm_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "normalized"]))
plt.close(fig)
np_sizes = np.array(sizes)
df = pd.DataFrame(np_sizes, columns=['size', 'status'])
size_plot = df.plot.scatter(x='size', y='status')
fig = size_plot.get_figure()
fig.savefig(os.sep.join(data_plots_folder + ['sizes_by_result.png']))
plt.close(fig)
# Now do the training
exmple_ids = [run.uuid for run in runs]
classes = np.array(classes)
print("\nTraining data shape: (%d, %d)" % n_examples.shape)
if train:
if debug:
tf.logging.set_verbosity(tf.logging.DEBUG)
config = tf.ConfigProto(log_device_placement=True, )
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
model = svm_trainer.SVMTrainer(n_examples,
exmple_ids,
labels,
classes,
dataset_name=dataset,
force_gpu=gpu)
model.train(steps=steps)
def train_model(build_name):
global estimator
dataset = estimator
global model_dir
with session_scope() as session:
if not os.path.isfile(os.sep.join([model_dir, 'data', dataset,
'runs.json.gz'])):
runs = gather_results.get_runs_by_name(None,
build_name=build_name,
session=session)
model_config = {'build_name': build_name}
gather_results.save_model_config(dataset, model_config,
data_path=model_dir)
gather_results.save_run_uuids(dataset, runs,
data_path=model_dir)
else:
runs = gather_results.load_run_uuids(dataset,
data_path=model_dir)
normalized_length = 5500
if estimator == 'svm':
skips = []
classes = []
labels = []
examples = []
class_label = 'status'
features_regex = None
sample_interval = None
idx = 0
# Model configuration. We need to cache sample_interval,
# features-regex and the normalization parameters for each
# feature so we can re-use them during prediction.
model_config = {
'sample_interval': sample_interval,
'features_regex': features_regex,
'normalized_length': normalized_length
}
for run in runs:
results = gather_results.get_subunit_results_for_run(
run, '1s', session=None, data_path=model_dir,
use_cache=True)
print('Acquired run %s' % run.uuid)
# For one run_uuid we must only get on example (result)
result = results[0]
if not result:
skips.append(run.uuid)
continue
# Setup the numpy matrix and sizes
if len(examples) == 0:
# Adjust normalized_length to the actual re-sample one
examples = np.ndarray(
shape=(
len(runs),
(len(result['dstat'].columns)
* normalized_length)))
model_config['num_columns'] = len(
result['dstat'].columns)
model_config['num_features'] = (len(
result['dstat'].columns) * normalized_length)
# Normalize data
example = fixed_lenght_example(result,
normalized_length)
# Normalize status
status = get_class(result, class_label)
vector, new_labels = unroll_example(
example, normalized_length, labels)
# Only calculate labels for the first example
if len(labels) == 0:
labels = new_labels
model_config['labels'] = labels
# Examples is an np ndarrays
examples[idx] = vector.values
classes.append(status)
if len(skips) > 0:
print('Unable to train model because of missing '
'runs %s' % skips)
safe_runs = [
run for run in runs if run.uuid not in skips]
gather_results.save_run_uuids(dataset, safe_runs,
data_path=model_dir)
message = ('The model has been updated to exclude '
'those runs. Please re-run the training'
' step.')
abort(make_response(message, 400))
def run_training():
# Perform dataset-wise normalization
# NOTE(andreaf) When we train the model we ignore any saved
# normalization
# parameter, since the sample interval and features may be
# different.
n_examples, normalization_params = normalize_dataset(
examples, labels)
# We do cache the result to normalize the prediction set.
model_config['normalization_params'] = normalization_params
gather_results.save_model_config(dataset, model_config,
data_path=model_dir)
# Now do the training
example_ids = [run.uuid for run in runs]
outclasses = np.array(classes)
svm_trainer.SVMTrainer(n_examples, example_ids, labels,
outclasses, dataset_name=dataset,
model_path=model_dir)
thread = threading.Thread(target=run_training)
thread.start()
return "training started", 202
else:
def run_nn_training():
for run in runs:
uuid = run.uuid
result = gather_results.get_subunit_results_for_run(
run, '1s', session=session, use_cache=False,
data_path=model_dir)[0]
try:
features, labels = nn_trainer.normalize_data(result)
except TypeError:
print('Unable to normalize data in run %s, '
'skipping' % uuid)
continue
nn_trainer.train_model(features, labels,
dataset_name=dataset,
model_path=model_dir)
print('done')
thread = threading.Thread(target=run_nn_training)
thread.start()
return "training started", 202
def prepare_dataset(dataset,
normalized_length,
num_dstat_features,
data_type,
features_regex,
sample_interval='1s',
class_label='status',
visualize=False,
data_path=None,
target_data_path=None,
s3=None):
"""Takes a dataset and filters and does the magic
Loads the run ids from the dataset configuration.
Loads the data (dsv + meta) for every run from cache.
Builds the unrolled exaples as a numpy ndarray.
Builds the classes as a numpy array.
Saves the data setup to the dataset config.
Does some visualization (if enabled).
"""
if visualize:
data_plots_folder = [
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'data',
dataset, 'plots'
]
os.makedirs(os.sep.join(data_plots_folder), exist_ok=True)
# Load the list of runs and base labels
runs = gather_results.load_run_uuids(dataset,
name=data_type,
data_path=target_data_path,
s3=s3)
# run_uuids are the example_ids
sizes = []
# The data for each example.
examples = examples_ndarray(len(runs), num_dstat_features,
normalized_length)
# The test result for each example
classes = []
skips = []
print("Loading %s data:" % data_type, end='\r', flush=True)
for count, run in enumerate(runs):
print("Loading %s data: %d of %d" % (data_type, count + 1, len(runs)),
end='\r',
flush=True)
result = gather_results.get_subunit_results_for_run(
run, sample_interval, data_path=data_path, s3=s3)
# For one run_uuid we must only get on example (result)
# Filtering by columns
if not result:
skips.append(run.uuid)
continue
# Apply column filtering
result = filter_example(result, features_regex)
# Normalize data
example = fixed_lenght_example(result, normalized_length)
vector = unroll_example(example, normalized_length)
# Normalize status
status = get_class(result, class_label)
# Examples is an np ndarrays
examples[count] = vector.values
classes.append(status)
# Plot from figures
if visualize:
# Prepare some more data if we are going to visualize
sizes.append((result['dstat'].shape[0], status))
figure_name = sample_interval + "_%s_" + str(count)
# Plot un-normalized data
data_plot = result['dstat'].plot()
fig = data_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "downsampled"]))
plt.close(fig)
# Plot fixed size data
fixed_plot = example.plot()
fig = fixed_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "fixedsize"]))
plt.close(fig)
# Plot unrolled data
unrolled_plot = pd.Series(vector).plot()
fig = unrolled_plot.get_figure()
fig.savefig(
os.sep.join(data_plots_folder + [figure_name % "unrolled"]))
plt.close(fig)
print("Loading %s data: %d done!" % (data_type, len(runs)))
# Check that everything went well
if len(skips) > 0:
print('Unable to train model because of missing runs %s' % skips)
safe_runs = [run.uuid for run in runs if run.uuid not in skips]
gather_results.save_run_uuids(dataset, safe_runs)
print('The model has been updated to exclude those runs.')
print('Please re-run the training step.')
sys.exit(1)
classes = np.array(classes)
figure_sizes = np.array(sizes)
example_ids = np.array(runs)
print("%s set: examples: %s, classes: %s, example IDs: %s" %
(data_type, str(examples.shape), str(
classes.shape), str(example_ids.shape)))
data = {
'examples': examples,
'example_ids': example_ids,
'classes': classes
}
return data, figure_sizes