def on_dataset_change(args): with dispatcher.get_lock('bootenvs'): if args['operation'] == 'create': bootenvs.propagate(args, convert_bootenv) if args['operation'] == 'delete': logger.warn(args) for i in args['ids']: pool, dataset = split_dataset(i) if pool != boot_pool_name: continue realname = dataset.split('/')[-1] ds = bootenvs.query(('realname', '=', realname), single=True) if ds: bootenvs.remove(ds['id']) if args['operation'] == 'update': for i in args['entities']: pool, dataset = split_dataset(i['id']) if pool != boot_pool_name: continue realname = dataset.split('/')[-1] ds = bootenvs.query(('realname', '=', realname), single=True) if not ds: continue nickname = i.get('properties.beadm:nickname.value', realname) if nickname and nickname != ds['id']: bootenvs.rename(ds['id'], nickname) bootenvs.put(nickname, convert_bootenv(i))
def make_dataset(name): inputs, psi, reference = load('{}.data'.format(name)) # train/test split trainset, testset = utils.split_dataset( # train/test split utils.shuffle((inputs, psi)), # shuffle dataset ratio=0.7) # test/valid split testset, validset = utils.split_dataset(testset, ratio=0.5) return (trainset, testset, validset)
def transfer(source_model, target_path, frozen_layer): # load data of branch df = pd.read_pickle(target_path) # create dataframe with netto sales, month, weekday, year df = pd.DataFrame(data=df.values, index=df.index, columns=['netto']) df = df.assign(month=df.index.month) df = df.assign(weekday=df.index.weekday) df = df.assign(year=df.index.year) # split into train and test train, test = split_dataset(df.values, 365) # prepare input data for branch n_input = 365 train_x, train_y = to_supervised(train, n_input, 365) # load pre-trained model of source branch as base model base_model = load_model(source_model) # freeze specific layers of base model for layer in base_model.layers[:frozen_layer]: layer.trainable = False print("frozen layers: " + str(frozen_layer)) # compile the model base_model.compile(loss='mse', optimizer='adam') # fit base_model with new data from branch n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1] input_data = [train_x[:, :, i].reshape((train_x.shape[0], n_timesteps, 1)) for i in range(n_features)] base_model.fit(input_data, train_y, epochs=20, batch_size=16, verbose=0) # evaluate fitted model mape = evaluate_model(train, test, base_model) return mape
def get_feats_from_csv_in_partitions(): """ Extract the original features that are distributed in the dataset. Features are splitted according with the config.yaml file. """ conf = utils.get_config() rows = [ row for row in utils.load_csv() if utils.check_filter(row, conf['filters']) ] train_rows, valid_rows, test_rows = utils.split_dataset( rows, conf['valid_percent'], conf['test_percent'], rng=conf['rng_seed']) X_train, y_train, X_valid, y_valid, X_test, y_test = [], [], [], [], [], [] prefixes = ['t_', 'i_', 's_'] # Feature names' prefixes datasets = [(X_train, y_train, train_rows), (X_test, y_test, test_rows), (X_valid, y_valid, valid_rows)] out = [] for X, y, rows in datasets: for row in rows: X.append([ float(v) for k, v in row.iteritems() if len(filter(k.startswith, prefixes)) > 0 ]) y.append(int(row['classification'] == 'Malign')) out.extend((np.asarray(X), np.asarray(y))) return out
def get_feats_from_imagenet_in_partitions(): conf = utils.get_config() imagenet_data = os.path.join(conf['models_path'], 'decafnet', 'imagenet.decafnet.epoch90') imagenet_meta = os.path.join(conf['models_path'], 'decafnet', 'imagenet.decafnet.meta') net = DecafNet(imagenet_data, imagenet_meta) rows = utils.get_filtered_rows() sets = utils.split_dataset(rows, conf['valid_percent'], conf['test_percent'], rng=conf['rng_seed']) feats = [] ys = [] for s in sets: X = np.zeros((len(s), 4096)) y = np.zeros(len(s)) for i, row in enumerate(s): try: log.info('processing %i-th of %i' % (i, len(s))) origin, im = utils.extract_roi(row, 30, True) scores = net.classify(np.asarray(im), center_only=True) X[i] = net.feature('fc7_cudanet_out') y[i] = utils.is_positive(row) except: continue feats.append(X) ys.append(y) return feats[0], ys[0], feats[1], ys[1], feats[2], ys[2]
def get_feats_in_partitions(): """ Extracts features from all dataset and split them in train validation and test sets """ conf = utils.get_config() paths = utils.get_paths() rows = utils.load_csv() filters = conf['filters'] region_size = conf['region_size'] region_stride = conf['region_stride'] filtered_rows = [ row for row in rows if utils.check_filter(row, conf['filters']) ] train_rows, valid_rows, test_rows = utils.split_dataset( filtered_rows, conf['valid_percent'], conf['test_percent'], rng=conf['rng_seed']) conv = get_fprop_fn(False) print 'Getting features from train...' X_train = get_feats_from_rows(train_rows, conv, conf['stride']) print 'Getting features from valid...' X_valid = get_feats_from_rows(valid_rows, conv, conf['stride']) print 'Getting features from test...' X_test = get_feats_from_rows(test_rows, conv, conf['stride']) y_train = [row['classification'] == 'Malign' for row in train_rows] y_valid = [row['classification'] == 'Malign' for row in valid_rows] y_test = [row['classification'] == 'Malign' for row in test_rows] return X_train, y_train, X_valid, y_valid, X_test, y_test
def initialize_train_test(self, train_ratio=0.75, sample=True, full=False): ''' This function initializes the training and the test sets. The training set is initialized to a fraction of the original dataset given by the input parameter train_ratio, while the test set will be initialized to the remaining part of the original dataset. Notice that the training and the test sets are copies of the original dataset, so every changes made on those datasets will not affect the original one. The default value for the parameter train_ratio is 0.75. If the input parameter sample is True, then the training set is built sampling rows from the original dataset. Otherwise, it is built taking the first part of the dataset. Default value for sample is True. If the input parameter full is True, then the values of the other parameters are ignored and both the training and the test sets are initialized to be exact copies of the full original one. ''' if full: self.train = self.dataset.copy() self.test = self.dataset.copy() else: if sample: self.train, self.test = utils.split_dataset_sample( self.dataset, train_ratio) else: self.train, self.test = utils.split_dataset( self.dataset, train_ratio)
def get_feats_from_imagenet_in_partitions(): conf = utils.get_config() imagenet_data = os.path.join( conf['models_path'], 'decafnet', 'imagenet.decafnet.epoch90') imagenet_meta = os.path.join( conf['models_path'], 'decafnet', 'imagenet.decafnet.meta') net = DecafNet(imagenet_data, imagenet_meta) rows = utils.get_filtered_rows() sets = utils.split_dataset( rows, conf['valid_percent'], conf['test_percent'], rng=conf['rng_seed']) feats = [] ys = [] for s in sets: X = np.zeros((len(s), 4096)) y = np.zeros(len(s)) for i, row in enumerate(s): try: log.info('processing %i-th of %i' % (i, len(s))) origin, im = utils.extract_roi(row, 30, True) scores = net.classify(np.asarray(im), center_only=True) X[i] = net.feature('fc7_cudanet_out') y[i] = utils.is_positive(row) except: continue feats.append(X) ys.append(y) return feats[0], ys[0], feats[1], ys[1], feats[2], ys[2]
def get_feats_in_partitions(): """ Extracts features from all dataset and split them in train validation and test sets """ conf = utils.get_config() paths = utils.get_paths() rows = utils.load_csv() filters = conf['filters'] region_size = conf['region_size'] region_stride = conf['region_stride'] filtered_rows = [ row for row in rows if utils.check_filter(row, conf['filters'])] train_rows, valid_rows, test_rows = utils.split_dataset( filtered_rows, conf['valid_percent'], conf['test_percent'], rng=conf['rng_seed']) conv = get_fprop_fn(False) print 'Getting features from train...' X_train = get_feats_from_rows( train_rows, conv, conf['stride']) print 'Getting features from valid...' X_valid = get_feats_from_rows( valid_rows, conv, conf['stride']) print 'Getting features from test...' X_test = get_feats_from_rows( test_rows, conv, conf['stride']) y_train = [row['classification'] == 'Malign' for row in train_rows] y_valid = [row['classification'] == 'Malign' for row in valid_rows] y_test = [row['classification'] == 'Malign' for row in test_rows] return X_train, y_train, X_valid, y_valid, X_test, y_test
def xgb_boost_model(): df_all = pickle.load(open("../output/features/basic_features.pkl", 'r')) test_ind = df_all.relevance == -1 test_data = df_all[test_ind] train_data = df_all[~test_ind] test_data = test_data.drop(['relevance'], axis=1) le = preprocessing.LabelEncoder() le.fit(train_data['relevance']) ids = test_data['id'] train, test, hold_out = utils.split_dataset(train_data) relevant_columns =['title_similarity', 'product_desc_similarity', 'title_similarity_common', 'product_desc_similarity_common', 'description_length', 'search_length'] dTrain = xgb.DMatrix(train['X'][relevant_columns], label=train['Y']) dTest = xgb.DMatrix(test['X'][relevant_columns], label=test['Y']) dHold_out = xgb.DMatrix(hold_out['X'][relevant_columns], label=hold_out['Y']) dSubmit = xgb.DMatrix(test_data[relevant_columns]) param = {'bst:max_depth':5 , 'bst:eta':0.05, 'silent':1, 'objective':'reg:linear', 'eval_metric':'rmse'} evallist = [(dTest, 'eval'), (dTrain, 'train')] numRound = 200 bst = xgb.train(param, dTrain, numRound, evallist) predHoldout = bst.predict(dHold_out) print "Mean square hold out error ", utils.rmse(hold_out['Y'], predHoldout) predY = bst.predict(dSubmit) utils.debug_model(hold_out['X'], hold_out['Y'], predY)
def run_network(window, model=None, save_model=False, show_plot=False): start_time = time.time() print('loading and prepare data set...') data = read_dataset('../datasets/internet-traffic-data-5minutes.csv') X_train, y_train, X_test, y_test, mean, std = split_dataset( data, window, ratio=0.90, standardize=True) print('number of training samples ', len(y_train)) print('number of test samples ', len(y_test)) if not model: print('initialize model...') model = compile_model( hidden_neurons=25, loss_fn='mse', input_dim=sum(1 for x in window if x), activation_fn='tanh') print('model ', model.summary()) print('train model...') early_stopping = EarlyStopping(monitor='val_loss', patience=2) model.fit(X_train, y_train, nb_epoch=500, validation_split=0.1, callbacks=[early_stopping]) print('make predictions...') prediction = model.predict(X_test).flatten() if show_plot: plot_result(prediction, y_test, mean, std) print('mase = ', mase(y_train, y_test, prediction)) if save_model: store_model(model) print('totoal duration: {:.2f} seconds'.format(time.time() - start_time))
def main(): model = create_model() model.summary() # Building Phase data = import_data("./dataset/crx_clean.data.txt") X, Y, X_train, X_test, Y_train, Y_test = split_dataset(data) # Expand data dimension for kernel to convolve over X_train = np.expand_dims(X_train, axis=2) # (None, 46, 1) X_test = np.expand_dims(X_test, axis=2) # (None, 46, 1) # create model model = KerasClassifier(build_fn=create_model, verbose=0) # Operational Phase scorer = make_scorer(f1_score, pos_label='+') print("\n### GRID SEARCH CROSS VAL USING STRATIFIED K FOLD###\n") Y_pred_grid_search = grid_search_cv_CNN(model, X_train, Y_train, X_test, Y_test, scorer) Y_pred_grid_search = np.squeeze(Y_pred_grid_search) print() print() print(Y_pred_grid_search) print() print(Y_test) print() print_scores(Y_test, Y_pred_grid_search)
def fit(self, X, y): """fit tree in X, y""" try: # only for numpy arrays for now if not isinstance(X, np.ndarray): X = np.array(X) if not isinstance(y, np.ndarray): y = np.array(y) if self.max_depth is None: # The absolute maximum depth would be N−1, where N is the number of training samples. # https://stats.stackexchange.com/questions/65893/maximal-depth-of-a-decision-tree self.max_depth = X.shape[0] - 1 if self.n_clasess is None: self.n_clasess = len(set(y)) assert (X.shape[0] > self.min_samples_split) if not (self.max_depth is None): assert (self.max_depth > 0) gain, column_idx, threshold = self._find_best_split(X, y) X_left, X_right, y_left, y_right = split_dataset(X, y, column=column_idx, t=threshold) self.node = Node(feature_idx=column_idx, threshold=threshold, labels=y, gain=gain) # build left and right child for max self.node.left = DecisionTree(criterion=self.criterion, debug=self.debug, max_depth=self.max_depth - 1) # if base class for random forest -> remove that attribute self.node.left.x_columns = self.x_columns self.node.left.n_clasess = self.n_clasess self.node.left.fit(X_left, y_left) self.node.right = DecisionTree(criterion=self.criterion, debug=self.debug, max_depth=self.max_depth - 1) # if base class for random forest -> remove that attribute self.node.right.x_columns = self.x_columns self.node.right.n_clasess = self.n_clasess self.node.right.fit(X_right, y_right) # not the best idea, it is impossible to check for other conditions with assert :( except AssertionError: self.node = Node() self._predict_from_leaf(y) # test info about predictions if self.debug: print("Is Last Node: ", self.node.is_last) print("Data shapes: ", X.shape, y.shape) print("Y: ", y) print("Prediction: ", self.node.node_prediction) print("Predict proba: ", self.node.node_prob_prediction) return self
def on_dataset_change(args): if args['operation'] == 'create': with dispatcher.get_lock('bootenvs'): boot_pool = dispatcher.call_sync('zfs.pool.get_boot_pool') bootenvs.propagate(args, lambda x: convert_bootenv(boot_pool, x)) if args['operation'] == 'delete': for i in args['ids']: pool, dataset = split_dataset(i) if pool != boot_pool_name: continue with dispatcher.get_lock('bootenvs'): realname = dataset.split('/')[-1] ds = bootenvs.query(('realname', '=', realname), single=True) if ds: bootenvs.remove(ds['id']) if args['operation'] == 'update': boot_pool = None for i in args['entities']: pool, dataset = split_dataset(i['id']) if pool != boot_pool_name: continue with dispatcher.get_lock('bootenvs'): realname = dataset.split('/')[-1] ds = bootenvs.query(('realname', '=', realname), single=True) if not ds: continue nickname = q.get(i, 'properties.beadm:nickname.value', realname) if nickname and nickname != ds['id']: bootenvs.rename(ds['id'], nickname) if not boot_pool: boot_pool = dispatcher.call_sync( 'zfs.pool.get_boot_pool') bootenvs.put(nickname, convert_bootenv(boot_pool, i))
def test_split_dataset(self): filename = '/path/to/dataset.tsv' train_filename = '/path/to/dataset_train.tsv' dev_filename = '/path/to/dataset_dev.tsv' read_data = 'Rock n Roll is a risk. You rick being ridiculed.\tDo you like rock music?\n' \ 'Rock n Roll is a risk.\tDo you like rock?' open_ = patch('utils.open', mock_open(read_data=read_data)).start() open_.return_value.__iter__.return_value = read_data.split('\n') writer = patch('csv.writer').start() split_dataset(filename) open_.assert_has_calls([ call(filename), call(train_filename, 'w'), call(dev_filename, 'w') ], any_order=True) writer.assert_called_with(open_.return_value, delimiter='\t') writer.return_value.writerow.assert_called() patch.stopall()
def on_dataset_change(args): if args['operation'] == 'create': with dispatcher.get_lock('bootenvs'): boot_pool = dispatcher.call_sync('zfs.pool.get_boot_pool') bootenvs.propagate(args, lambda x: convert_bootenv(boot_pool, x)) if args['operation'] == 'delete': for i in args['ids']: pool, dataset = split_dataset(i) if pool != boot_pool_name: continue with dispatcher.get_lock('bootenvs'): realname = dataset.split('/')[-1] ds = bootenvs.query(('realname', '=', realname), single=True) if ds: bootenvs.remove(ds['id']) if args['operation'] == 'update': boot_pool = None for i in args['entities']: pool, dataset = split_dataset(i['id']) if pool != boot_pool_name: continue with dispatcher.get_lock('bootenvs'): realname = dataset.split('/')[-1] ds = bootenvs.query(('realname', '=', realname), single=True) if not ds: continue nickname = q.get(i, 'properties.beadm:nickname.value', realname) if nickname and nickname != ds['id']: bootenvs.rename(ds['id'], nickname) if not boot_pool: boot_pool = dispatcher.call_sync('zfs.pool.get_boot_pool') bootenvs.put(nickname, convert_bootenv(boot_pool, i))
def do_experiment_for_one_year(run_path, year, config): """Performs the specified experiments for one year.""" X, Y = load_dataset(year, shuffle=config['experiment']['shuffle_data']) if config['experiment']['type'] == 'single': X_train, Y_train, X_test, Y_test = split_dataset(X, Y, config['experiment']['test_share']) results = perform_one_experiment(X_train, Y_train, X_test, Y_test, config) elif config['experiment']['type'] == 'cv': results = perform_cv_runs(X, Y, config) results_path = os.path.join(run_path, 'results_year{}.pkl'.format(year)) with open(results_path, 'wb') as f: pickle.dump(results, f) show_results(results, year, **config['analysis'])
def fit(self, samples, labels): """Train the model with the samples and lables provided according to the parameters of the model.""" # Split into train and dev x_train, y_train, x_dev, y_dev = split_dataset(samples, labels, self.dev_share) # Create batch iterator if self.batch_iterator_type == 'normal': batch_iter = _batch_iter elif self.batch_iterator_type == 'oversample': batch_iter = _oversampling_batch_iter else: raise ValueError('{} is not a valid batch_iterator_type'.format( self.batch_iterator_type)) # Train model train_batch_nr = [] train_loss_val = [] dev_batch_nr = [] dev_loss_val = [] for i, (x, y) in enumerate( batch_iter(x_train, y_train, self.num_epochs, self.batch_size)): # Train feed_dict = { self.graph_nodes['x_input']: x, self.graph_nodes['y_input']: y, self.graph_nodes['dropout_keep_prob']: self.dropout_keep_prob } _, loss_val = self.sess.run( [self.graph_nodes['optimize'], self.graph_nodes['loss']], feed_dict=feed_dict) train_batch_nr.append(i) train_loss_val.append(loss_val) if i % self.evaluate_every_n_steps == 0: feed_dict = { self.graph_nodes['x_input']: x_dev, self.graph_nodes['y_input']: y_dev, self.graph_nodes['dropout_keep_prob']: 1. } loss_val = self.sess.run(self.graph_nodes['loss'], feed_dict=feed_dict) dev_batch_nr.append(i) dev_loss_val.append(loss_val) if self.plot_training: plt.plot(train_batch_nr, train_loss_val) plt.plot(dev_batch_nr, dev_loss_val) plt.show()
def pretrain_model(path): df = pd.read_pickle(path) # create dataframe with netto sales, month, weekday, year df = pd.DataFrame(data=df.values, index=df.index, columns=['netto']) df = df.assign(month=df.index.month) df = df.assign(weekday=df.index.weekday) df = df.assign(year=df.index.year) # split into train and test train, test = split_dataset(df.values, 365) # evaluate model and get scores model = evaluate_model(train, test) # save model model.save("models/" + path + ".h5") del model
def run(self, id, updated_fields): share = self.datastore.get_by_id('shares', id) remove_unchanged(updated_fields, share) path = self.dispatcher.call_sync('share.get_directory_path', share['id']) try: delete_config( path, '{0}-{1}'.format(share['type'], share['name']) ) except OSError: pass if 'type' in updated_fields: old_share_type = share['type'] new_share_type = self.dispatcher.call_sync('share.supported_types').get(updated_fields['type']) if share['target_type'] == 'DATASET': pool, dataset = split_dataset(share['target_path']) self.join_subtasks( self.run_subtask('volume.dataset.update', dataset, { 'permissions_type': new_share_type['perm_type'] }) ) share.update(updated_fields) self.join_subtasks(self.run_subtask('share.{0}.delete'.format(old_share_type), id)) self.join_subtasks(self.run_subtask('share.{0}.create'.format(updated_fields['type']), share)) else: self.join_subtasks(self.run_subtask('share.{0}.update'.format(share['type']), id, updated_fields)) if 'permissions' in updated_fields: path = self.dispatcher.call_sync('share.translate_path', id) self.join_subtasks(self.run_subtask('file.set_permissions', path, updated_fields['permissions'])) self.dispatcher.dispatch_event('share.changed', { 'operation': 'update', 'ids': [share['id']] }) updated_share = self.datastore.get_by_id('shares', id) path = self.dispatcher.call_sync('share.get_directory_path', updated_share['id']) try: save_config( path, '{0}-{1}'.format(updated_share['type'], updated_share['name']), updated_share ) except OSError as err: self.add_warning(TaskWarning(errno.ENXIO, 'Cannot save backup config file: {0}'.format(str(err))))
def train_and_test(df, preds, seed): ''' Run a single trial: Shuffle df and split it into training and testing subsets Train a new model based on the training sets Test the model with testing set Add prediction data into preds array :param df: dataframe with full set of all available samples columns: id, cat1 (primary class), cat2 (secondary), title, titlen (claened title) :param preds: an array of predictions, each prediction is a dictionary cat: true category, pred: predicted category, conf: model confidence in its prediction (< 1.0), title: actual title of the chapter/sample :return: average testing accuracy ''' ret = {} # PREPS # randomly split the dataset df = utils.split_dataset( df, settings.CAT_DEPTH, settings.TRAIN_PER_CLASS_MIN, settings.TEST_PER_CLASS, settings.VALID_PER_CLASS, ) # TRAIN classifier = Classifier.from_name(settings.CLASSIFIER, seed) classifier.set_datasets(df, titles_out_path) classifier.train() df_test = classifier.df_test if settings.EVALUATE_TRAINING_SET: evaluate_model(classifier, classifier.df_train, display_prefix='TRAIN = ') accuracy = evaluate_model(classifier, df_test, preds, display_prefix='TEST = ') classifier_key = utils.get_exp_key(classifier) classifier.release_resources() return classifier_key, accuracy, classifier.df_train
def train(self, X_train, y_train): """ Train a Regression Forest using the given training data by training a number of Regression Trees each with a random sample of the training data. """ train_dataset = np.c_[(X_train, y_train)] for i in range(self.n_estimators): bootstrap_sample = train_dataset[np.random.choice( train_dataset.shape[0], size=int(round(train_dataset.shape[0] * self.split)), replace=True)] X_train, y_train, _, _ = split_dataset(bootstrap_sample, self.split, is_print=False) tree = RegressionTree(self.n_features, self.max_depth) tree.train(X_train, y_train) self.random_forest.append(tree)
def __init__(self, hparams=None): super().__init__() # Metrics self.train_acc = pl.metrics.Accuracy() self.val_acc = pl.metrics.Accuracy(compute_on_step=False) self.test_acc = pl.metrics.Accuracy(compute_on_step=False) # Hyperparameters self.hparams = hparams # Data self.train_data, self.test_data, self.val_data = split_dataset() # Model initialization multiplier = 2 if hparams["bidirectional"] else 1 self.word_vec_size = 300 * multiplier self.amount_classes = 7 self.rnn = nn.LSTM(input_size=self.word_vec_size, hidden_size=hparams["lstm_hidden_dim"], bidirectional=hparams["bidirectional"], num_layers=hparams["lstm_num_layers"]) # First FC layer modules = [ nn.Linear(self.word_vec_size, hparams["FC_layer_dims"][0]), nn.ReLU(), nn.Dropout(hparams["FC_dropouts"][0]) ] # Middle FC layers for i, (dim, d_rate) in enumerate( zip(hparams["FC_layer_dims"], hparams["FC_dropouts"])): if i == len(hparams["FC_layer_dims"]) - 1: continue # we reached the end modules.append(nn.Linear(dim, hparams["FC_layer_dims"][i + 1])) modules.append(nn.ReLU()) modules.append(nn.Dropout(d_rate)) # Last FC layer modules.append( nn.Linear(hparams["FC_layer_dims"][-1], self.amount_classes)) modules.append(nn.ReLU()) modules.append(nn.Dropout(hparams["FC_dropouts"][-1])) self.classifier = nn.Sequential(*modules)
def main(): # Building Phase data = import_data("./dataset/crx_clean.data.txt") X, Y, X_train, X_test, Y_train, Y_test = split_dataset(data) # Operational Phase scorer = make_scorer(f1_score, pos_label='+') print("\n### GRID SEARCH CROSS VAL USING STRATIFIED K FOLD###\n") Y_pred_grid_search = grid_search_cv_mlp(X_train, Y_train, X_test, Y_test, scorer) print() print() print(Y_pred_grid_search) print() print(Y_test) print() print_scores(Y_test, Y_pred_grid_search)
def main(args: Namespace): results_path = args.log_dir / str(datetime.now()) results_path.mkdir(exist_ok=True, parents=True) write_args(results_path, vars(args)) fix_seed(args.seed) height_model = 1000 width_model = 24 filenames_train, filenames_valid, filenames_test = split_dataset(args.data_root, args.fracs_dataset) train_set = SeisDataset(filenames_train, height_model=height_model, width_model=width_model, prob_aug=args.prob_aug) valid_set = SeisDataset(filenames_valid, height_model=height_model, width_model=width_model, prob_aug=args.prob_aug) test_set = SeisDataset(filenames_test, height_model=height_model, width_model=width_model, prob_aug=args.prob_aug) net = UNetFB() picker = Picker(net) stopper = Stopper(args.n_wrongs, args.delta_wrongs) trainer = Trainer(picker=picker, results_path=results_path, train_set=train_set, valid_set=valid_set, test_set=test_set, device=args.device, batch_size=args.batch_size, lr=args.lr, freq_valid=args.freq_valid, num_workers=args.num_workers, dt_ms=args.dt_ms, height_model=height_model, width_model=width_model, visual=args.visual, stopper=stopper, weights=torch.tensor(args.weights)) trainer.train(num_epoch=args.num_epoch)
def run_custom_classifier(weights=None): training_images = utils.load_images_from_dir(TRAINING_DIR) testing_images = utils.load_images_from_dir(TESTING_DIR) X_train, X_test, y_train, y_test = utils.split_dataset(training_images, testing_images, TRAINING_FILE, TESTING_FILE) #preprocess data X_train, y_train = preprocess(X_train, y_train) X_test, y_test = preprocess(X_test, y_test) #compile model model = custom_classifier() #train if no weights are passed in if weights == None: history = model.fit(X_train, y_train, epochs=50, verbose=1, validation_data=(X_test, y_test)) model.save_weights(os.path.join(WEIGHTS_DIR, 'custom_model.h5')) else: model.load_weights(weights) scores = model.evaluate(X_train, y_train, verbose=1) print('Digit 1 loss:', scores[1]) print('Digit 2 loss:', scores[2]) print('Digit 3 loss:', scores[3]) print('Digit 4 loss:', scores[4]) print('Digit 5 loss:', scores[5]) average_loss = sum([scores[i] for i in range(1, 6)]) / 5 print('Average loss:', average_loss) print('Digit 1 accuracy:', scores[6]) print('Digit 2 accuracy:', scores[7]) print('Digit 3 accuracy:', scores[8]) print('Digit 4 accuracy:', scores[9]) print('Digit 5 accuracy:', scores[10]) average_accuracy = sum([scores[i] for i in range(6, 11)]) / 5 print('Average accueracy:', average_accuracy)
def obtain_train_test(path, ifilename, ftrain_name, ftest_name, frac_test=0.2): df = pd.read_csv(os.path.join(path, ifilename)) df['date'] = pd.to_datetime(df['date']) if 'Unnamed: 0' in df.columns: df.drop('Unnamed: 0', axis=1, inplace=True) groups = list(df['inlet'].unique()) dftrain, dftest = split_dataset(df, frac_test=frac_test, groups=groups) #80/20% for all inlets if ((dftrain.shape[0] + dftest.shape[0]) != df.shape[0]): ValueError( 'The shapes of the resulting files are inconsistent with the shape of input table!' ) else: dftrain.to_csv(os.path.join(path, ftrain_name), index=False) dftest.to_csv(os.path.join(path, ftest_name), index=False) return dftrain, dftest
def create_loaders(dataset_name, dataset_train, dataset_val, dataset_test, train_size, val_size, batch_size, test_batch_size, cuda, num_workers, topk=None, noise=False): kwargs = {'num_workers': num_workers, 'pin_memory': True} if cuda else {} dataset_train, dataset_val = split_dataset(dataset_train, dataset_val, train_size, val_size) print('Dataset sizes: \t train: {} \t val: {} \t test: {}'.format( len(dataset_train), len(dataset_val), len(dataset_test))) train_loader = data.DataLoader(dataset_train, batch_size=batch_size, shuffle=True, **kwargs) val_loader = data.DataLoader(dataset_val, batch_size=test_batch_size, shuffle=False, **kwargs) test_loader = data.DataLoader(dataset_test, batch_size=test_batch_size, shuffle=False, **kwargs) train_loader.tag = 'train' val_loader.tag = 'val' test_loader.tag = 'test' return train_loader, val_loader, test_loader
def get_feats_from_csv_in_partitions(): """ Extract the original features that are distributed in the dataset. Features are splitted according with the config.yaml file. """ conf = utils.get_config() rows = [row for row in utils.load_csv() if utils.check_filter(row, conf['filters'])] train_rows, valid_rows, test_rows = utils.split_dataset( rows, conf['valid_percent'], conf['test_percent'], rng=conf['rng_seed']) X_train, y_train, X_valid, y_valid, X_test, y_test = [], [], [], [], [], [] prefixes = ['t_', 'i_', 's_'] # Feature names' prefixes datasets = [(X_train, y_train, train_rows), (X_test, y_test, test_rows), (X_valid, y_valid, valid_rows)] out = [] for X, y, rows in datasets: for row in rows: X.append( [float(v) for k, v in row.iteritems() if len(filter(k.startswith, prefixes)) > 0]) y.append(int(row['classification'] == 'Malign')) out.extend((np.asarray(X), np.asarray(y))) return out
def exercicio2(): utils.print_header(2) data, classes = load_balance_scale(os.path.join(constants.DATA_DIR, constants.FILENAME_BALANCE_DATABASE)) print('Nb samples: {}'.format(data.shape[0])) gaussian_accuracy, discrete_accuracy, laplace_accuracy = [], [], [] np.random.seed(constants.SEED) for i in range(10): x_train, y_train, x_test, y_test = utils.split_dataset(data) params = {'mean': {}, 'std': {}, 'classes': classes, 'prior': {}, 'discrete_prob': {}} for c in classes: params['prior'][c] = sum(y_train == c) / float(x_train.shape[0]) x_c = x_train[y_train == c] params['mean'][c] = np.mean(x_c, axis=0) params['std'][c] = np.std(x_c, axis=0) params['discrete_prob'][c] = {} for j in range(x_c.shape[1]): params['discrete_prob'][c][j] = {} for k in [1, 2, 3, 4, 5]: params['discrete_prob'][c][j][k] = { 'sum': sum(x_c[:, j] == k), 'n': x_c.shape[0], } gaussian_pred = gaussian_predict(x_test, params) gaussian_accuracy.append(utils.accuracy(y_test, gaussian_pred)) discrete_pred = discrete_predict(x_test, params, laplace=False) discrete_accuracy.append(utils.accuracy(y_test, discrete_pred)) laplace_pred = discrete_predict(x_test, params, laplace=True) laplace_accuracy.append(utils.accuracy(y_test, laplace_pred)) print('a)') print('\tGaussian - Accuracy: {:.2f} +- {:.2f}'.format(np.mean(gaussian_accuracy), np.std(gaussian_accuracy))) print('b)') print('\tDiscrete - Accuracy: {:.2f} +- {:.2f}'.format(np.mean(discrete_accuracy), np.std(discrete_accuracy))) print('c)') print('\tDiscrete (with Laplace) - Accuracy: {:.2f} +- {:.2f}'.format(np.mean(laplace_accuracy), np.std(laplace_accuracy))) exit()
def main(): # Building Phase data = import_data( "./dataset/crx_clean.data.txt" ) X, Y, X_train, X_test, Y_train, Y_test = split_dataset(data) clf_entropy = train_using_entropy(X_train, Y_train) # Operational Phase print("\n### SINGLE TRAIN-TEST SPLIT ###\n") Y_pred_entropy = prediction(X_test, clf_entropy) print_scores(Y_test, Y_pred_entropy) print("\n### CROSS VAL USING STRATIFIED K FOLD ###\n") fold_scores = cv_with_entropy(X, Y) print("Cross Validate: ", fold_scores) print("Best F1_score: ", max(fold_scores)*100) scorer = make_scorer(f1_score, pos_label='+') print("\n### GRID SEARCH CROSS VAL USING STRATIFIED K FOLD###\n") Y_pred_grid_search = grid_search_cv_DT(X_train, Y_train, X_test, Y_test, scorer) print_scores(Y_test, Y_pred_grid_search)
def objective(params): nneurons = params['nneurons'] if params['season'] == 'full_day': window = create_window_array(params['window'], season_lag=288) if params['season'] == 'half_day': window = create_window_array(params['window'], season_lag=168) else: window = create_window_array(params['window']) if not any(window) or nneurons < 2: return {'status': STATUS_FAIL} X_train, y_train, *_ = split_dataset( data, window, ratio=0.90, standardize=True) model = compile_model( nneurons, input_dim=sum(1 for x in window if x), loss_fn='mse', activation_fn=params['activation_function']) hist = model.fit( X_train, y_train, nb_epoch=50, validation_split=0.1, callbacks=[EarlyStopping(monitor='val_loss', patience=2)], verbose=0) return {'loss': hist.history['val_loss'][-1], 'status': STATUS_OK}
def main(args): #################### # Arguments gpu = args.gpu model_name = args.model initial_tree_sampling = args.initial_tree_sampling path_config = args.config data_augmentation = args.data_augmentation trial_name = args.name actiontype = args.actiontype max_epoch = args.max_epoch dev_size = args.dev_size # Check assert actiontype in ["train", "evaluate"] if actiontype == "train": assert max_epoch > 0 assert len(initial_tree_sampling.split("_")) == 3 for type_ in initial_tree_sampling.split("_"): assert type_ in ["X", "BU", "TD", "RB", "LB", "RB2"] assert initial_tree_sampling.split("_")[2] != "X" assert initial_tree_sampling.split("_")[1] != "RB2" assert initial_tree_sampling.split("_")[2] != "RB2" if trial_name is None or trial_name == "None": trial_name = utils.get_current_time() #################### # Path setting config = utils.Config(path_config) basename = "%s.%s.%s.aug_%s.%s" \ % (model_name, initial_tree_sampling, utils.get_basename_without_ext(path_config), data_augmentation, trial_name) if actiontype == "train": path_log = os.path.join(config.getpath("results"), basename + ".training.log") elif actiontype == "evaluate": path_log = os.path.join(config.getpath("results"), basename + ".evaluation.log") path_train = os.path.join(config.getpath("results"), basename + ".training.jsonl") path_valid = os.path.join(config.getpath("results"), basename + ".validation.jsonl") path_snapshot = os.path.join(config.getpath("results"), basename + ".model") path_pred = os.path.join(config.getpath("results"), basename + ".evaluation.ctrees") path_eval = os.path.join(config.getpath("results"), basename + ".evaluation.json") utils.set_logger(path_log) #################### # Random seed random_seed = trial_name random_seed = utils.hash_string(random_seed) random.seed(random_seed) np.random.seed(random_seed) cuda.cupy.random.seed(random_seed) #################### # Log so far utils.writelog("gpu=%d" % gpu) utils.writelog("model_name=%s" % model_name) utils.writelog("initial_tree_sampling=%s" % initial_tree_sampling) utils.writelog("path_config=%s" % path_config) utils.writelog("data_augmentation=%s" % data_augmentation) utils.writelog("trial_name=%s" % trial_name) utils.writelog("actiontype=%s" % actiontype) utils.writelog("max_epoch=%s" % max_epoch) utils.writelog("dev_size=%s" % dev_size) utils.writelog("path_log=%s" % path_log) utils.writelog("path_train=%s" % path_train) utils.writelog("path_valid=%s" % path_valid) utils.writelog("path_snapshot=%s" % path_snapshot) utils.writelog("path_pred=%s" % path_pred) utils.writelog("path_eval=%s" % path_eval) utils.writelog("random_seed=%d" % random_seed) #################### # Data preparation begin_time = time.time() train_dataset = dataloader.read_rstdt("train", relation_level="coarse-grained", with_root=False) test_dataset = dataloader.read_rstdt("test", relation_level="coarse-grained", with_root=False) vocab_word = utils.read_vocab( os.path.join(config.getpath("data"), "rstdt-vocab", "words.vocab.txt")) vocab_postag = utils.read_vocab( os.path.join(config.getpath("data"), "rstdt-vocab", "postags.vocab.txt")) vocab_deprel = utils.read_vocab( os.path.join(config.getpath("data"), "rstdt-vocab", "deprels.vocab.txt")) if data_augmentation: external_train_dataset = dataloader.read_ptbwsj_wo_rstdt( with_root=False) # Remove documents with only one leaf node external_train_dataset = utils.filter_dataset( external_train_dataset, condition=lambda data: len(data.edu_ids) > 1) end_time = time.time() utils.writelog("Loaded the corpus. %f [sec.]" % (end_time - begin_time)) #################### # Hyper parameters word_dim = config.getint("word_dim") postag_dim = config.getint("postag_dim") deprel_dim = config.getint("deprel_dim") lstm_dim = config.getint("lstm_dim") mlp_dim = config.getint("mlp_dim") n_init_epochs = config.getint("n_init_epochs") negative_size = config.getint("negative_size") batch_size = config.getint("batch_size") weight_decay = config.getfloat("weight_decay") gradient_clipping = config.getfloat("gradient_clipping") optimizer_name = config.getstr("optimizer_name") utils.writelog("word_dim=%d" % word_dim) utils.writelog("postag_dim=%d" % postag_dim) utils.writelog("deprel_dim=%d" % deprel_dim) utils.writelog("lstm_dim=%d" % lstm_dim) utils.writelog("mlp_dim=%d" % mlp_dim) utils.writelog("n_init_epochs=%d" % n_init_epochs) utils.writelog("negative_size=%d" % negative_size) utils.writelog("batch_size=%d" % batch_size) utils.writelog("weight_decay=%f" % weight_decay) utils.writelog("gradient_clipping=%f" % gradient_clipping) utils.writelog("optimizer_name=%s" % optimizer_name) #################### # Model preparation cuda.get_device(gpu).use() # Initialize a model utils.mkdir(os.path.join(config.getpath("data"), "caches")) path_embed = config.getpath("pretrained_word_embeddings") path_caches = os.path.join( config.getpath("data"), "caches", "cached." + os.path.basename(path_embed) + ".npy") if os.path.exists(path_caches): utils.writelog("Loading cached word embeddings ...") initialW = np.load(path_caches) else: initialW = utils.read_word_embedding_matrix(path=path_embed, dim=word_dim, vocab=vocab_word, scale=0.0) np.save(path_caches, initialW) if model_name == "spanbasedmodel": # Span-based model w/ template features template_feature_extractor = models.TemplateFeatureExtractor( dataset=train_dataset) utils.writelog("Template feature size=%d" % template_feature_extractor.feature_size) if actiontype == "train": for template in template_feature_extractor.templates: dim = template_feature_extractor.template2dim[template] utils.writelog("Template feature #%s %s" % (dim, template)) model = models.SpanBasedModel( vocab_word=vocab_word, vocab_postag=vocab_postag, vocab_deprel=vocab_deprel, word_dim=word_dim, postag_dim=postag_dim, deprel_dim=deprel_dim, lstm_dim=lstm_dim, mlp_dim=mlp_dim, initialW=initialW, template_feature_extractor=template_feature_extractor) elif model_name == "spanbasedmodel2": # Span-based model w/o template features model = models.SpanBasedModel2(vocab_word=vocab_word, vocab_postag=vocab_postag, vocab_deprel=vocab_deprel, word_dim=word_dim, postag_dim=postag_dim, deprel_dim=deprel_dim, lstm_dim=lstm_dim, mlp_dim=mlp_dim, initialW=initialW) else: raise ValueError("Invalid model_name=%s" % model_name) utils.writelog("Initialized the model ``%s''" % model_name) # Load pre-trained parameters if actiontype != "train": serializers.load_npz(path_snapshot, model) utils.writelog("Loaded trained parameters from %s" % path_snapshot) model.to_gpu(gpu) #################### # Decoder preparation decoder = decoders.IncrementalCKYDecoder() #################### # Initializer preparation sampler = treesamplers.TreeSampler(initial_tree_sampling.split("_")) #################### # Training / evaluation if actiontype == "train": with chainer.using_config("train", True): if dev_size > 0: # Training with cross validation train_dataset, dev_dataset = utils.split_dataset( dataset=train_dataset, n_dev=dev_size, seed=None) with open( os.path.join(config.getpath("results"), basename + ".valid_gold.ctrees"), "w") as f: for data in dev_dataset: f.write("%s\n" % " ".join(data.nary_sexp)) else: # Training with the full training set dev_dataset = None if data_augmentation: train_dataset = np.concatenate( [train_dataset, external_train_dataset], axis=0) train(model=model, decoder=decoder, sampler=sampler, max_epoch=max_epoch, n_init_epochs=n_init_epochs, negative_size=negative_size, batch_size=batch_size, weight_decay=weight_decay, gradient_clipping=gradient_clipping, optimizer_name=optimizer_name, train_dataset=train_dataset, dev_dataset=dev_dataset, path_train=path_train, path_valid=path_valid, path_snapshot=path_snapshot, path_pred=os.path.join(config.getpath("results"), basename + ".valid_pred.ctrees"), path_gold=os.path.join(config.getpath("results"), basename + ".valid_gold.ctrees")) elif actiontype == "evaluate": with chainer.using_config("train", False), chainer.no_backprop_mode(): # Test parse(model=model, decoder=decoder, dataset=test_dataset, path_pred=path_pred) scores = metrics.rst_parseval( pred_path=path_pred, gold_path=os.path.join(config.getpath("data"), "rstdt", "wsj", "test", "gold.labeled.nary.ctrees")) old_scores = metrics.old_rst_parseval( pred_path=path_pred, gold_path=os.path.join(config.getpath("data"), "rstdt", "wsj", "test", "gold.labeled.nary.ctrees")) out = { "Morey2018": { "Unlabeled Precision": scores["S"]["Precision"] * 100.0, "Precision_info": scores["S"]["Precision_info"], "Unlabeled Recall": scores["S"]["Recall"] * 100.0, "Recall_info": scores["S"]["Recall_info"], "Micro F1": scores["S"]["Micro F1"] * 100.0 }, "Marcu2000": { "Unlabeled Precision": old_scores["S"]["Precision"] * 100.0, "Precision_info": old_scores["S"]["Precision_info"], "Unlabeled Recall": old_scores["S"]["Recall"] * 100.0, "Recall_info": old_scores["S"]["Recall_info"], "Micro F1": old_scores["S"]["Micro F1"] * 100.0 } } utils.write_json(path_eval, out) utils.writelog(utils.pretty_format_dict(out)) utils.writelog("Done: %s" % basename)
def boston_housing(): """ Trains algorithms for the boston housing dataset """ # boston housing dataset is available in the sklearn library boston = load_boston() dimension = boston.data.shape[1] boston_data = np.column_stack((boston.data, boston.target)) split = 2 / 3 iterations = 20 gammas = np.linspace(math.pow(2, -40), math.pow(2, -26), 15) sigmas = np.linspace(math.pow(2, 7), math.pow(2, 13), 14) k_fold = 5 naive_results = np.zeros((iterations, 2)) single_results = np.zeros((iterations, dimension, 2)) all_results = np.zeros((iterations, 2)) k_results = np.zeros((iterations, 2)) for i in range(iterations): print("iterations i", i) training_set, testing_set = utils.split_dataset(boston_data, split) # naive _, train_mse, test_mse = lr.naive(training_set, testing_set) naive_results[i, 0] = train_mse naive_results[i, 1] = test_mse # single attribute for attr in range(dimension): _, train_mse, test_mse = lr.single_attribute(training_set, testing_set, attr) single_results[i, attr, 0] = train_mse single_results[i, attr, 1] = test_mse # all attributes _, train_mse, test_mse = lr.all_attributes(training_set, testing_set) all_results[i, 0] = train_mse all_results[i, 1] = test_mse # kernel _, train_mse, test_mse = rr.gaussian_kernel_cross_validation( training_set, testing_set, gammas, sigmas, k_fold) k_results[i, 0] = train_mse k_results[i, 1] = test_mse # display naive results print("Naive MSE train: %s +- %s" % ( np.mean(naive_results[:, 0]), np.std(naive_results[:, 0]))) print("Naive MSE test: %s +- %s" % ( np.mean(naive_results[:, 1]), np.std(naive_results[:, 1]))) # display single attribute results for attr in range(dimension): id = attr + 1 print("Linear (attribute %d) MSE train: %s +- %s" % ( id, np.mean(single_results[:, attr, 0]), np.std(single_results[:, attr, 0]))) print("Linear (attribute %d) MSE test: %s +- %s" % ( id, np.mean(single_results[:, attr, 1]), np.std(single_results[:, attr, 1]))) # display all attributes results print("Linear (all) MSE train: %s +- %s" % ( np.mean(all_results[:, 0]), np.std(all_results[:, 0]))) print("Linear (all) MSE test: %s +- %s" % ( np.mean(all_results[:, 1]), np.std(all_results[:, 1]))) # display kernel results print("Kernel MSE train: %s +- %s" % ( np.mean(k_results[:, 0]), np.std(k_results[:, 0]))) print("Kernel MSE test: %s +- %s" % ( np.mean(k_results[:, 1]), np.std(k_results[:, 1])))
self.__get_labels_neighborhood(row, distance) for row in X_classifier ] if __name__ == "__main__": PATH_FILE = ( "/home/nobrega/Dados/Documentos/Estudos/notes/dataset/knn_classification.csv" ) df = pd.read_csv(PATH_FILE) df.pop("id") y = df["class"].values y = y.reshape(len(y), 1) df.pop("class") X = df.values X_train, y_train, X_test, y_test = split_dataset(X, y, 0.7) n_neighbors = 15 knn = KNearestNeighbors(X_train, y_train, n_neighbors) for type_distance in DistanceTypes: try: y_hat = np.array(knn.predict(X_test, type_distance.value)) y_test = np.ndarray.flatten(np.array(y_test)) acc = sum(((y_hat == y_test) * 1.0)) / len(y_test) print(f"Type of Distance {type_distance.value} AcurĂ¡cia: {acc}") except: print(f"This distance type {type_distance.value} can't calculate")
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') keep_topK = 200 USE_HOLD_OUT = True # visualization of HOLD-OUT set # dir_prenms = "../results/preNMS" # =========================== Dataset ============================== # file path and make a list imgs_path = '../data/hw3_mycocodata_img_comp_zlib.h5' masks_path = '../data/hw3_mycocodata_mask_comp_zlib.h5' labels_path = "../data/hw3_mycocodata_labels_comp_zlib.npy" bboxes_path = "../data/hw3_mycocodata_bboxes_comp_zlib.npy" paths = [imgs_path, masks_path, labels_path, bboxes_path] dataset = BuildDataset(paths, augmentation=False) train_dataset, test_dataset = utils.split_dataset(dataset) # dataset # train_build_loader = BuildDataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8) # train_loader = train_build_loader.loader() test_build_loader = BuildDataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0) test_loader = test_build_loader.loader() # we will need the ImageList from torchvision from torchvision.models.detection.image_list import ImageList do_eval(test_loader, checkpoint_file,
def main(): #torch.manual_seed(42) # ------------ # args # ------------ parser = ArgumentParser() parser.add_argument('--auto_lr', type=U.str2bool, default=False, help="Auto lr finder") parser.add_argument('--learning_rate', type=float, default=10e-4) parser.add_argument('--scheduler', type=U.str2bool, default=False) parser.add_argument('--wd', type=float, default=2e-4) parser.add_argument('--moment', type=float, default=0.9) parser.add_argument('--batch_size', default=5, type=int) parser.add_argument('--n_epochs', default=10, type=int) parser.add_argument('--model', default='FCN', type=str, help="FCN or DLV3 model") parser.add_argument('--pretrained', default=False, type=U.str2bool, help="Use pretrained pytorch model") parser.add_argument('--eval_angle', default=True, type=U.str2bool,help=\ "If true, it'll eval the model with different angle input size") parser.add_argument('--rotate', default=False, type=U.str2bool, help="Use random rotation as data augmentation") parser.add_argument('--scale', default=True, type=U.str2bool, help="Use scale as data augmentation") parser.add_argument('--size_img', default=520, type=int, help="Size of input images") parser.add_argument('--size_crop', default=480, type=int, help="Size of crop image during training") parser.add_argument('--nw', default=0, type=int, help="Num workers for the data loader") parser.add_argument('--pm', default=True, type=U.str2bool, help="Pin memory for the dataloader") parser.add_argument('--gpu', default=0, type=int, help="Wich gpu to select for training") parser.add_argument('--benchmark', default=False, type=U.str2bool, help="enable or disable backends.cudnn") parser.add_argument('--split', default=False, type=U.str2bool, help="Split the dataset") parser.add_argument('--split_ratio', default=0.3, type=float, help="Amount of data we used for training") parser.add_argument('--dataroot_voc', default='/share/DEEPLEARNING/datasets/voc2012/', type=str) parser.add_argument('--dataroot_sbd', default='/share/DEEPLEARNING/datasets/sbd/', type=str) parser.add_argument('--model_name', type=str, help="what name to use for saving") parser.add_argument('--save_dir', default='/data/save_model', type=str) parser.add_argument('--save_all_ep', default=False, type=U.str2bool,help=\ "If true it'll save the model every epoch in save_dir") parser.add_argument('--save_best', default=False, type=U.str2bool, help="If true will only save the best epoch model") args = parser.parse_args() # ------------ # save # ------------ save_dir = U.create_save_directory(args.save_dir) print('model will be saved in', save_dir) U.save_hparams(args, save_dir) # ------------ # device # ------------ device = torch.device( "cuda:" + str(args.gpu) if torch.cuda.is_available() else "cpu") print("device used:", device) # ------------ # model # ------------ if args.model.upper() == 'FCN': model = models.segmentation.fcn_resnet101(pretrained=args.pretrained) elif args.model.upper() == 'DLV3': model = models.segmentation.deeplabv3_resnet101( pretrained=args.pretrained) else: raise Exception('model must be "FCN" or "DLV3"') model.to(device) # ------------ # data # ------------ if args.size_img < args.size_crop: raise Exception( 'Cannot have size of input images less than size of crop') size_img = (args.size_img, args.size_img) size_crop = (args.size_crop, args.size_crop) train_dataset_VOC = mdset.VOCSegmentation(args.dataroot_voc,year='2012', image_set='train', \ download=True,rotate=args.rotate,scale=args.scale,size_img=size_img,size_crop=size_crop) val_dataset_VOC = mdset.VOCSegmentation(args.dataroot_voc, year='2012', image_set='val', download=True) train_dataset_SBD = mdset.SBDataset(args.dataroot_sbd, image_set='train_noval',mode='segmentation',\ rotate=args.rotate,scale=args.scale,size_img=size_img,size_crop=size_crop) # Concatene dataset train_dataset = tud.ConcatDataset([train_dataset_VOC, train_dataset_SBD]) split = args.split if split == True: train_dataset = U.split_dataset(train_dataset, args.split_ratio) # Print len datasets print("There is", len(train_dataset), "images for training and", len(val_dataset_VOC), "for validation") dataloader_train = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size,num_workers=args.nw,\ pin_memory=args.pm,shuffle=True,drop_last=True)#,collate_fn=U.my_collate) dataloader_val = torch.utils.data.DataLoader(val_dataset_VOC,num_workers=args.nw,pin_memory=args.pm,\ batch_size=args.batch_size) # Decide which device we want to run on # ------------ # training # ------------ # Auto lr finding #if args.auto_lr==True: criterion = nn.CrossEntropyLoss( ignore_index=21) # On ignore la classe border. optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate, momentum=args.moment, weight_decay=args.wd) ev.train_fully_supervised(model=model,n_epochs=args.n_epochs,train_loader=dataloader_train,val_loader=dataloader_val,\ criterion=criterion,optimizer=optimizer,save_folder=save_dir,scheduler=args.scheduler,model_name=args.model_name,\ benchmark=args.benchmark, save_best=args.save_best,save_all_ep=args.save_all_ep,device=device,num_classes=21) # Final evaluation if args.eval_angle: d_iou = ev.eval_model_all_angle(model, args.size_img, args.dataroot_voc, train=True, device=device) U.save_eval_angle(d_iou, save_dir) d_iou = ev.eval_model_all_angle(model, args.size_img, args.dataroot_voc, train=False, device=device) U.save_eval_angle(d_iou, save_dir)
def train_dvna(A, P, verbose=False): n_epochs = 4000 train_ones_indices, train, val, test = u.split_dataset(A, seed=seed) A_train = u.prepare_train_matrix_dvne(A, train_ones_indices) # sample triplets # nbrs is a dict nbrs[i] -> {j1,j2,...,} nbrs = {} not_nbrs = {} all_nodes = {i for i in range(A.shape[0])} for ij in zip(train_ones_indices[0], train_ones_indices[1]): i, j = int(ij[0]), int(ij[1]) if i in nbrs.keys(): nbrs[i] = nbrs[i].union({j}) else: nbrs[i] = {j} if j in nbrs.keys(): nbrs[j] = nbrs[j].union({i}) else: nbrs[j] = {i} for i in nbrs.keys(): nbrs_set = nbrs[i] not_nbrs[i] = all_nodes - nbrs_set model = DVNE(n_features=A.shape[0]) model.to(device) opt = torch.optim.Adam(lr=0.001, params=model.parameters()) nonzero_ratio = A.sum() / (A.shape[0]**2) zero_ratio = 1 - nonzero_ratio A = A.to(device) P = P.to(device) A_train = A_train.to(device) model.n_samples = len(train) criterion = l.dvne_loss for e in range(n_epochs): t0 = time.time() triplets = u.sample_triplets(nbrs, not_nbrs, 300) i, j, k = triplets model.train() opt.zero_grad() out_i, mi, stdi = model.forward(A_train[i, :]) out_j, mj, stdj = model.forward(A_train[j, :]) out_k, mk, stdk = model.forward(A_train[k, :]) gt_i = P[i, :] gt_j = P[j, :] gt_k = P[k, :] out_reconstruction = torch.cat([out_i, out_j, out_k], dim=0).view(-1) gt = torch.cat([gt_i, gt_j, gt_k], dim=0).view(-1).to(device) a_gt = torch.cat([A[i, :], A[j, :], A[j, :]], dim=0).view(-1).to(device) x = torch.ones(gt.shape[0]).to(device) weights = torch.where(gt == 0.0, x * float(nonzero_ratio), x * float(zero_ratio)).to(device) loss_norm = weights.sum() / len(train) loss_weight = 0.6 # loss = criterion(out_reconstruction, gt_reconsturction, weight=weights) * loss_weight/ loss_norm l2 = criterion(gt, out_reconstruction) * loss_weight # loss = 0.0 w_ij = model.wasserstein((mi, stdi), (mj, stdj)) w_ik = model.wasserstein((mi, stdi), (mk, stdk)) l1 = l.energy_loss(w_ij, w_ik) loss = l1 + loss_weight * l2 loss.backward() opt.step() t1 = time.time() if verbose: if (e + 1) % 100 == 0: if len(val) > 0: with torch.no_grad(): val_loss = float( criterion( (model.forward(A_train)[0]).reshape(-1)[val], A.reshape(-1)[val].data)) val_auc = u.test_auc_dvna(model, A_train, A, val) else: val_auc = np.nan val_loss = np.nan print( "Iteration: {0}; train loss: {1:.4f}; val loss: {2:.4f}; val auc: {3:.4f}; time: {4:.4f}" .format(e + 1, loss, val_loss, val_auc, t1 - t0)) test_auc = u.test_auc_dvna(model, A_train, A, idx=test, test=True) # print("Test auc: ", test_auc) if dataset_name == 'cora' and visualize: with torch.no_grad(): encodings, mean, std = model.encode(P.to(device)) embeddings = torch.cat([mean, std], dim=1) dv.reduct_and_visualize(embeddings.cpu().numpy(), Y.argmax(axis=1)) train, val_test = next( Split(train_size=140, random_state=seed).split(embeddings, Y)) embeddings = embeddings.cpu() x_train, y_train = embeddings[train], Y[train] x_test, y_test = embeddings[val_test], Y[val_test] svm = SVC(C=10.0) svm.fit(x_train, y_train.argmax(axis=1)) y_predicted = svm.predict(x_test) print("SVM Accuracy: ", accuracy_score(y_predicted, y_test.argmax(axis=1))) return test_auc
def run(self, id, updated_fields, enable_service=False): share = self.datastore.get_by_id('shares', id) if not share: raise TaskException(errno.ENOENT, 'Share not found') if share['immutable']: raise TaskException(errno.EACCES, 'Cannot modify immutable share {0}.'.format(id)) if 'name' in updated_fields or 'type' in updated_fields: share.update(updated_fields) if self.datastore.exists( 'shares', ('id', '!=', id), ('type', '=', share['type']), ('name', '=', share['name']) ): raise TaskException(errno.EEXIST, 'Share {0} of type {1} already exists'.format( share['name'], share['type'] )) path_after_update = updated_fields.get('target_path', share['target_path']) type_after_update = updated_fields.get('target_type', share['target_type']) permissions = updated_fields.pop('permissions', None) share_path = self.dispatcher.call_sync('share.expand_path', path_after_update, type_after_update) if not os.path.exists(share_path): raise TaskException( errno.ENOENT, 'Selected share target {0} does not exist'.format(path_after_update) ) share = self.datastore.get_by_id('shares', id) remove_unchanged(updated_fields, share) path = self.dispatcher.call_sync('share.get_directory_path', share['id']) try: delete_config( path, '{0}-{1}'.format(share['type'], share['name']) ) except (OSError, ValueError): pass if 'type' in updated_fields: old_share_type = share['type'] new_share_type = self.dispatcher.call_sync('share.supported_types').get(updated_fields['type']) if share['target_type'] == 'DATASET': pool, dataset = split_dataset(share['target_path']) self.join_subtasks( self.run_subtask('volume.dataset.update', dataset, { 'permissions_type': new_share_type['perm_type'] }) ) share.update(updated_fields) self.run_subtask_sync('share.{0}.delete'.format(old_share_type), id) self.run_subtask_sync('share.{0}.create'.format(updated_fields['type']), share) else: self.run_subtask_sync('share.{0}.update'.format(share['type']), id, updated_fields) if permissions: path = self.dispatcher.call_sync('share.translate_path', id) self.run_subtask_sync('file.set_permissions', path, permissions) self.dispatcher.dispatch_event('share.changed', { 'operation': 'update', 'ids': [share['id']] }) updated_share = self.datastore.get_by_id('shares', id) path = self.dispatcher.call_sync('share.get_directory_path', updated_share['id']) try: save_config( path, '{0}-{1}'.format(updated_share['type'], updated_share['name']), updated_share ) except OSError as err: self.add_warning(TaskWarning(errno.ENXIO, 'Cannot save backup config file: {0}'.format(str(err)))) service_state = self.dispatcher.call_sync('service.query', [('name', '=', share['type'])], {'single': True}) if service_state['state'] != 'RUNNING': if enable_service: config = service_state['config'] config['enable'] = True self.run_subtask_sync('service.update', service_state['id'], {'config': config}) else: self.add_warning(TaskWarning( errno.ENXIO, "Share has been updated but the service {0} is not currently running " "Please enable the {0} service.".format(share['type']) ))
def run(self, id, updated_fields, enable_service=False): share = self.datastore.get_by_id('shares', id) if not share: raise TaskException(errno.ENOENT, 'Share not found') if share['immutable']: raise TaskException(errno.EACCES, 'Cannot modify immutable share {0}.'.format(id)) if 'name' in updated_fields or 'type' in updated_fields: share.update(updated_fields) if self.datastore.exists( 'shares', ('id', '!=', id), ('type', '=', share['type']), ('name', '=', share['name']) ): raise TaskException(errno.EEXIST, 'Share {0} of type {1} already exists'.format( share['name'], share['type'] )) path_after_update = updated_fields.get('target_path', share['target_path']) type_after_update = updated_fields.get('target_type', share['target_type']) permissions = updated_fields.pop('permissions', None) share_path = self.dispatcher.call_sync('share.expand_path', path_after_update, type_after_update) if type_after_update in ('DIRECTORY', 'FILE'): pool_mountpoints = tuple(self.dispatcher.call_sync('volume.query', [], {'select': 'mountpoint'})) if not path_after_update.startswith(pool_mountpoints): raise TaskException(errno.EINVAL, "Provided directory or file has to reside within user defined ZFS pool") if not os.path.exists(share_path): raise TaskException( errno.ENOENT, 'Selected share target {0} does not exist'.format(path_after_update) ) share = self.datastore.get_by_id('shares', id) remove_unchanged(updated_fields, share) path = self.dispatcher.call_sync('share.get_directory_path', share['id']) try: delete_config( path, '{0}-{1}'.format(share['type'], share['name']) ) except (OSError, ValueError): pass if 'type' in updated_fields: old_share_type = share['type'] new_share_type = self.dispatcher.call_sync('share.supported_types').get(updated_fields['type']) if share['target_type'] == 'DATASET': pool, dataset = split_dataset(share['target_path']) self.join_subtasks( self.run_subtask('volume.dataset.update', dataset, { 'permissions_type': new_share_type['perm_type'] }) ) share.update(updated_fields) self.run_subtask_sync('share.{0}.delete'.format(old_share_type), id) self.run_subtask_sync('share.{0}.create'.format(updated_fields['type']), share) else: self.run_subtask_sync('share.{0}.update'.format(share['type']), id, updated_fields) if permissions: path = self.dispatcher.call_sync('share.translate_path', id) self.run_subtask_sync('file.set_permissions', path, permissions) self.dispatcher.dispatch_event('share.changed', { 'operation': 'update', 'ids': [share['id']] }) updated_share = self.datastore.get_by_id('shares', id) path = self.dispatcher.call_sync('share.get_directory_path', updated_share['id']) try: save_config( path, '{0}-{1}'.format(updated_share['type'], updated_share['name']), updated_share, file_perms=0o600 ) except OSError as err: self.add_warning(TaskWarning(errno.ENXIO, 'Cannot save backup config file: {0}'.format(str(err)))) service_state = self.dispatcher.call_sync('service.query', [('name', '=', share['type'])], {'single': True}) if service_state['state'] != 'RUNNING': if enable_service: config = service_state['config'] config['enable'] = True self.run_subtask_sync('service.update', service_state['id'], {'config': config}) else: self.add_warning(TaskWarning( errno.ENXIO, "Share has been updated but the service {0} is not currently running " "Please enable the {0} service.".format(share['type']) ))
import utils from pylearn2.utils import serial import h5py import numpy as np import sys if __name__ == "__main__": conf_file = sys.argv[1] if len(sys.argv) > 1 else None conf = utils.get_config(conf_file) paths = utils.get_paths() region_size = conf['region_size'] region_stride = conf['region_stride'] train_rows, valid_rows, test_rows = utils.split_dataset( utils.get_filtered_rows(), conf['valid_percent'], conf['test_percent'], rng=conf['rng_seed']) rowsdict = {'train': train_rows, 'valid': valid_rows, 'test': test_rows} nsamples = {} prefixes = ['s_', 'i_', 't_'] # Feature names' prefixes for subset, subrows in rowsdict.iteritems(): X = None y = [] feats = [] for row in subrows: samples = utils.get_samples_from_image( row, oversampling=(subset == 'train' and conf['oversampling'])) print "%i samples to %s taken from %s" % ( len(samples), subset, row['image_filename']) if len(samples) == 0:
from torchvision import transforms import os import argparse from utils import split_dataset, train, validate from grayscale import Grayscale from colorize import Colorize device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") ap = argparse.ArgumentParser() ap.add_argument("-i", "--image", required=True, help="path to input image") ap.add_argument("-e", "--epochs", type=int, default=100, help="# of epochs") args = vars(ap.parse_args()) # split the landscape dataset to train and validation folders split_dataset(args["image"]) # Training train_transforms = transforms.Compose( [transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip()]) train_folder = Grayscale('images/train', train_transforms) train_loader = torch.utils.data.DataLoader(train_folder, batch_size=64, shuffle=True) # Validation val_transforms = transforms.Compose( [transforms.Resize(256), transforms.CenterCrop(224)]) val_folder = Grayscale('images/val', val_transforms)
for i in range(0, len(rows), chunkSize): offset = min(i + chunkSize, len(rows)) f_chunk, y_chunk = fe_extraction.get_feats_from_cnn( rows[i:offset], model) if feats is None: feats = f_chunk y = y_chunk else: feats = np.vstack((feats, f_chunk)) y = np.hstack((y, y_chunk)) segm_ids = np.asarray([int(row['segmentation_id']) for row in rows]) features = np.hstack((features, feats)) train_rows, valid_rows, test_rows = utils.split_dataset( utils.get_filtered_rows(), conf['valid_percent'], conf['test_percent'], rng=conf['rng_seed']) rows = train_rows + valid_rows patients = utils.rows_to_patients(rows) for i in range(n_runs): train_rows, empty_rows, valid_rows = utils.split_dataset( rows, valid_percent=0, test_percent=0.2, rng=rng, patients=patients) X_train, y_train = get_features(train_rows, features, segm_ids) X_valid, y_valid = get_features(valid_rows, features, segm_ids) print 'train: %i, valid: %i' % (X_train.shape[0], X_valid.shape[0]) if scale_feats: min_max_scaler = preprocessing.MinMaxScaler() X_train = min_max_scaler.fit_transform(X_train) X_valid = min_max_scaler.transform(X_valid)