def db_trainer(estimator, dataset, build_name, limit, db_uri, evaluate): runs = gather_results.get_runs_by_name(db_uri, build_name=build_name) model_config = {'build_name': build_name} gather_results.save_model_config(dataset, model_config) if limit > 0: runs = runs[:limit] gather_results.save_run_uuids(dataset, runs) for run in runs: if estimator == 'tf.estimator.DNNClassifier': gather_results.get_subunit_results_for_run(run, '1s', db_uri, use_cache=True) print('Acquired run %s' % run.uuid) else: result = gather_results.get_subunit_results_for_run( run, '1s', db_uri)[0] print('Acquired run %s' % run.uuid) try: features, labels = nn_trainer.normalize_data(result) except TypeError: print('Unable to normalize data in run %s, ' 'skipping' % run.uuid) continue if not evaluate: nn_trainer.train_model(features, labels, dataset_name=dataset) else: nn_trainer.evaluate_model(features, labels, dataset_name=dataset)
def run_training(): # Perform dataset-wise normalization # NOTE(andreaf) When we train the model we ignore any saved # normalization # parameter, since the sample interval and features may be # different. n_examples, normalization_params = normalize_dataset( examples, labels) # We do cache the result to normalize the prediction set. model_config['normalization_params'] = normalization_params gather_results.save_model_config(dataset, model_config, data_path=model_dir) # Now do the training example_ids = [run.uuid for run in runs] outclasses = np.array(classes) svm_trainer.SVMTrainer(n_examples, example_ids, labels, outclasses, dataset_name=dataset, model_path=model_dir)
def local_trainer(train, estimator, dataset, sample_interval, features_regex, class_label, visualize, steps, gpu, debug): # Normalized lenght before resampling normalized_length = 5500 if sample_interval: # Calculate the desired normalized lenght after resample normalized_length = get_downsampled_example_lenght( sample_interval, normalized_length) data_plots_folder = [ os.path.dirname(os.path.realpath(__file__)), os.pardir, 'data', dataset, 'plots' ] os.makedirs(os.sep.join(data_plots_folder), exist_ok=True) runs = gather_results.load_run_uuids(dataset) # run_uuids are the example_ids sizes = [] # The data for each example. We don't know yet the pre-set shape, so # wait until the first result comes in examples = [] # Model configuration. We need to cache sample_interval, features-regex and # the normalization parameters for each feature so we can re-use them # during prediction. model_config = { 'sample_interval': sample_interval, 'features_regex': features_regex, 'normalized_length': normalized_length } # The test result for each example classes = [] labels = [] idx = 0 skips = [] for run in runs: results = gather_results.get_subunit_results_for_run( run, sample_interval) # For one run_uuid we must only get on example (result) result = results[0] # Filtering by columns if not result: skips.append(run.uuid) continue df = result['dstat'] if features_regex: col_regex = re.compile(features_regex) result['dstat'] = df[list(filter(col_regex.search, df.columns))] # Setup the numpy matrix and sizes if len(examples) == 0: # Adjust normalized_length to the actual re-sample one examples = np.ndarray(shape=(len(runs), len(result['dstat'].columns) * normalized_length)) model_config['num_columns'] = len(result['dstat'].columns) model_config['num_features'] = (len(result['dstat'].columns) * normalized_length) # Normalize data example = fixed_lenght_example(result, normalized_length) # Normalize status status = get_class(result, class_label) vector, new_labels = unroll_example(example, normalized_length, labels) # Only calculate labels for the first example if len(labels) == 0: labels = new_labels model_config['labels'] = labels print("Normalized example %d of %d" % (runs.index(run) + 1, len(runs)), end='\r', flush=True) # Examples is an np ndarrays examples[idx] = vector.values classes.append(status) if visualize: # Prepare some more data if we are going to visualize sizes.append((result['dstat'].shape[0], status)) figure_name = sample_interval + "_%s_" + str(idx) # Plot un-normalized data data_plot = result['dstat'].plot() fig = data_plot.get_figure() fig.savefig( os.sep.join(data_plots_folder + [figure_name % "downsampled"])) plt.close(fig) # Plot fixed size data fixed_plot = example.plot() fig = fixed_plot.get_figure() fig.savefig( os.sep.join(data_plots_folder + [figure_name % "fixedsize"])) plt.close(fig) # Plot unrolled data unrolled_plot = pd.Series(vector).plot() fig = unrolled_plot.get_figure() fig.savefig( os.sep.join(data_plots_folder + [figure_name % "unrolled"])) plt.close(fig) idx += 1 if len(skips) > 0: print('Unable to train model because of missing runs %s' % skips) safe_runs = [run for run in runs if run.uuid not in skips] gather_results.save_run_uuids(dataset, safe_runs) print('The model has been updated to exclude those runs.') print('Please re-run the training step.') sys.exit(1) # Perform dataset-wise normalization # NOTE(andreaf) When we train the model we ignore any saved normalization # parameter, since the sample interval and features may be different. n_examples, normalization_params = normalize_dataset(examples, labels) # We do cache the result to normalize the prediction set. model_config['normalization_params'] = normalization_params gather_results.save_model_config(dataset, model_config) if visualize: for n in range(len(runs)): figure_name = sample_interval + "_%s_" + str(n) unrolled_norm_plot = pd.Series(n_examples[n]).plot() fig = unrolled_norm_plot.get_figure() fig.savefig( os.sep.join(data_plots_folder + [figure_name % "normalized"])) plt.close(fig) np_sizes = np.array(sizes) df = pd.DataFrame(np_sizes, columns=['size', 'status']) size_plot = df.plot.scatter(x='size', y='status') fig = size_plot.get_figure() fig.savefig(os.sep.join(data_plots_folder + ['sizes_by_result.png'])) plt.close(fig) # Now do the training exmple_ids = [run.uuid for run in runs] classes = np.array(classes) print("\nTraining data shape: (%d, %d)" % n_examples.shape) if train: if debug: tf.logging.set_verbosity(tf.logging.DEBUG) config = tf.ConfigProto(log_device_placement=True, ) config.gpu_options.allow_growth = True config.allow_soft_placement = True model = svm_trainer.SVMTrainer(n_examples, exmple_ids, labels, classes, dataset_name=dataset, force_gpu=gpu) model.train(steps=steps)
def train_model(build_name): global estimator dataset = estimator global model_dir with session_scope() as session: if not os.path.isfile(os.sep.join([model_dir, 'data', dataset, 'runs.json.gz'])): runs = gather_results.get_runs_by_name(None, build_name=build_name, session=session) model_config = {'build_name': build_name} gather_results.save_model_config(dataset, model_config, data_path=model_dir) gather_results.save_run_uuids(dataset, runs, data_path=model_dir) else: runs = gather_results.load_run_uuids(dataset, data_path=model_dir) normalized_length = 5500 if estimator == 'svm': skips = [] classes = [] labels = [] examples = [] class_label = 'status' features_regex = None sample_interval = None idx = 0 # Model configuration. We need to cache sample_interval, # features-regex and the normalization parameters for each # feature so we can re-use them during prediction. model_config = { 'sample_interval': sample_interval, 'features_regex': features_regex, 'normalized_length': normalized_length } for run in runs: results = gather_results.get_subunit_results_for_run( run, '1s', session=None, data_path=model_dir, use_cache=True) print('Acquired run %s' % run.uuid) # For one run_uuid we must only get on example (result) result = results[0] if not result: skips.append(run.uuid) continue # Setup the numpy matrix and sizes if len(examples) == 0: # Adjust normalized_length to the actual re-sample one examples = np.ndarray( shape=( len(runs), (len(result['dstat'].columns) * normalized_length))) model_config['num_columns'] = len( result['dstat'].columns) model_config['num_features'] = (len( result['dstat'].columns) * normalized_length) # Normalize data example = fixed_lenght_example(result, normalized_length) # Normalize status status = get_class(result, class_label) vector, new_labels = unroll_example( example, normalized_length, labels) # Only calculate labels for the first example if len(labels) == 0: labels = new_labels model_config['labels'] = labels # Examples is an np ndarrays examples[idx] = vector.values classes.append(status) if len(skips) > 0: print('Unable to train model because of missing ' 'runs %s' % skips) safe_runs = [ run for run in runs if run.uuid not in skips] gather_results.save_run_uuids(dataset, safe_runs, data_path=model_dir) message = ('The model has been updated to exclude ' 'those runs. Please re-run the training' ' step.') abort(make_response(message, 400)) def run_training(): # Perform dataset-wise normalization # NOTE(andreaf) When we train the model we ignore any saved # normalization # parameter, since the sample interval and features may be # different. n_examples, normalization_params = normalize_dataset( examples, labels) # We do cache the result to normalize the prediction set. model_config['normalization_params'] = normalization_params gather_results.save_model_config(dataset, model_config, data_path=model_dir) # Now do the training example_ids = [run.uuid for run in runs] outclasses = np.array(classes) svm_trainer.SVMTrainer(n_examples, example_ids, labels, outclasses, dataset_name=dataset, model_path=model_dir) thread = threading.Thread(target=run_training) thread.start() return "training started", 202 else: def run_nn_training(): for run in runs: uuid = run.uuid result = gather_results.get_subunit_results_for_run( run, '1s', session=session, use_cache=False, data_path=model_dir)[0] try: features, labels = nn_trainer.normalize_data(result) except TypeError: print('Unable to normalize data in run %s, ' 'skipping' % uuid) continue nn_trainer.train_model(features, labels, dataset_name=dataset, model_path=model_dir) print('done') thread = threading.Thread(target=run_nn_training) thread.start() return "training started", 202