def main(): X_tr, X_te, y_tr, y_te = read_dataset() model = build_model() model.fit(X_tr, y_tr) print("Train train score", model.score(X_tr, y_tr)) print("Train test score ", model.score(X_te, y_te)) print("Train set") print(classification_report(model.predict(X_tr), y_tr)) print("Test set") print(classification_report(model.predict(X_te), y_te)) build_model()
def lsq_pos_l1_penalty(matrix, rhs, cost_multiplier, weights_0): """ min 2-norm (matrix*w - rhs)** + 1-norm(cost_multiplier*(w-w0)) s.t. e'w = 1 w >= 0 """ # define model model = mModel.build_model('lsqSparse') # introduce n non-negative weight variables weights = mModel.weights(model, "weights", n=matrix.shape[1], lb=0.0) # e'*w = 1 mBound.equal(model, Expr.sum(weights), 1.0) # sum of squared residuals v = mMath.l2_norm_squared(model, "2-norm(res)**", __residual(matrix, rhs, weights)) print matrix.shape[1] print weights_0 print weights # \Gamma*(w - w0), p is an expression p = mMath.mat_vec_prod(cost_multiplier, Expr.sub(weights, weights_0)) t = mMath.l1_norm(model, 'abs(weights)', p) # Minimise v + t mModel.minimise(model, __sum_weighted(1.0, v, 1.0, t)) return np.array(weights.level())
def test_main_model(data): X_tr, X_te = data model = build_model() model.fit(X_tr) predictions = model.predict(X_te) revenue_in_2_weeks = predictions.groupby("company_name").sum() print(revenue_in_2_weeks)
def main(): model = build_model() X_tr, X_te = read_dataset(n_days=14) model.fit(X_tr) predictions = model.predict(X_te) revenue_in_2_weeks = predictions.groupby("company_name").sum() print(revenue_in_2_weeks) revenue_in_2_weeks.to_csv('predictions.csv', index=False)
def main(data_path, week): data = prepare_dataset(data_path) (X_tr, y_tr), (X_te, y_te) = train_last_week_split(data, week=week, full_training=False) clf = build_model() clf.fit(X_tr, y_tr) submission = X_te.loc[:, ["i", "j"]] submission["prediction"] = clf.predict_proba(X_te)[:, 1] submission.to_csv("prediction.csv", index=False)
def markowitz(exp_ret, covariance_mat, aversion): # define model model = mModel.build_model("mean_var") # set of n weights (unconstrained) weights = mModel.weights(model, "weights", n=len(exp_ret)) mBound.equal(model, Expr.sum(weights), 1.0) # standard deviation induced by covariance matrix var = mMath.variance(model, "var", weights, covariance_mat) mModel.maximise(model=model, expr=Expr.sub(Expr.dot(exp_ret, weights), Expr.mul(aversion, var))) return np.array(weights.level())
def markowitz_riskobjective(exp_ret, covariance_mat, bound): # define model model = mModel.build_model("mean_var") # set of n weights (unconstrained) weights = mModel.weights(model, "weights", n=len(exp_ret)) # standard deviation induced by covariance matrix stdev = mMath.stdev(model, "std", weights, covariance_mat) # impose a bound on this standard deviation mBound.upper(model, stdev, bound) mModel.maximise(model=model, expr=Expr.dot(exp_ret, weights)) return np.array(weights.level())
def lasso(matrix, rhs, lamb): """ min 2-norm (matrix*w - rhs)^2 + lamb * 1-norm(w) """ # define model model = mModel.build_model('lasso') # introduce variables and constraints weights = mModel.weights(model, "weights", matrix.shape[1]) v = mMath.l2_norm_squared(model, "2-norm(res)**", __residual(matrix, rhs, weights)) t = mMath.l1_norm(model, "1-norm(w)", weights) # Minimise 1.0*v + lambda * t mModel.minimise(model=model, expr=__sum_weighted(c1=1.0, expr1=v, c2=lamb, expr2=t)) return np.array(weights.level())
def lsq_ls(matrix, rhs): """ min 2-norm (matrix*w - rhs)^2 s.t. e'w = 1 """ # define model model = mModel.build_model('lsqPos') # introduce n non-negative weight variables weights = mModel.weights(model, "weights", n=matrix.shape[1]) # e'*w = 1 mBound.equal(model, Expr.sum(weights), 1.0) v = mMath.l2_norm(model, "2-norm(res)", expr=__residual(matrix, rhs, weights)) # minimization of the residual mModel.minimise(model=model, expr=v) return np.array(weights.level())
def main(): path = "/home/alisher/Documents/data/brats/MICCAI_BraTS_2019_Data_Training/HGG/BraTS19_2013_10_1" input_shape = (4, 80, 96, 64) seqs = ["flair", "t1", "t2", "t1ce", "seg"] data_paths = [{}] for name in os.listdir(path): for seq in seqs: if seq in name: data_paths[0][seq] = f"{path}/{name}" break output_channels = 3 data = np.empty((len(data_paths[:4]), ) + input_shape, dtype=np.float32) labels = np.empty((len(data_paths[:4]), output_channels) + input_shape[1:], dtype=np.uint8) for i, imgs in enumerate(data_paths): try: data[i] = np.array( [preprocess(nib.load(imgs[m]), input_shape[1:]) for m in seqs], dtype=np.float32) labels[i] = preprocess_label(read_img(imgs['seg']), input_shape[1:])[None, ...] except Exception as e: print( f'Something went wrong with {imgs["t1"]}, skipping...\n Exception:\n{str(e)}' ) continue config, _ = get_config(os.path.abspath('config.json')) if os.path.exists(config.model_path): print("Pretrained model is loaded") model = load_model(config.model_path) else: print("New model was initialized") model = build_model(input_shape=input_shape, output_channels=config.label_num) model.fit(data, [labels, data], batch_size=1, epochs=1)
dev_images1 = dev_images1.astype('float32') / 255 # general categoricla variable # notice I am add "s" at train_label, to distinguish from earlier numpy. train_labels = to_categorical(train_label_np) dev_labels = to_categorical(dev_label_np) train_labels1 = to_categorical(train_label_np1) dev_labels1 = to_categorical(dev_label_np1) train_labels_merged = merge_labels(train_labels, train_labels1) dev_labels_merged = merge_labels(dev_labels, dev_labels1) sample_size = dev_labels_merged.shape[0] # sample size for i in range(sample_size): print(train_labels_merged[i, :], ",", train_label_np[i], ",", train_label_np1[i]) params = {} params['height'] = height params['width'] = width params['channel'] = channel model = build_model(True, params) history = train_model(model, train_labels_merged, train_images, dev_labels_merged, dev_images) # print the graph of learning history for diagnostic purpose. print_plot_keras_metrics(history)
def initial_fit(num_timesteps, num_targets, train_percent=.93, num_tweets=300): print("started init fit") dir_path = os.path.dirname(os.path.abspath(__file__)) #clear contents of log files open(os.path.join(dir_path, 'logs/context_prices.txt'), 'w').close() open(os.path.join(dir_path, 'logs/actuals.txt'), 'w').close() open(os.path.join(dir_path, 'logs/predictions.txt'), 'w').close() open(os.path.join(dir_path, 'logs/history.txt'), 'w').close() open(os.path.join(dir_path, 'logs/proxy_log.txt'), 'w').close() data = get_historical(num_tweets, from_date="") X_train, y_train, X_test, y_test, ref = load_data( data, num_timesteps, num_targets=num_targets, train_percent=train_percent ) #TODO: make higher percentage of training when this goes into "prod" # store recent data so that we can get a live prediction recent_reference = [] recent_data = data[-num_timesteps:, 1:] recent_data = normalize_timestep(recent_data, recent_reference) print(" X_train", X_train.shape) print(" y_train", y_train.shape) print(" X_test", X_test.shape) print(" y_test", y_test.shape) model = build_model([9, num_timesteps, num_targets]) #train the model print("TRAINING") model.fit(X_train, y_train, batch_size=512, epochs=600, validation_split=0.1, verbose=2) save_model(model) trainScore = model.evaluate(X_train, y_train, verbose=100) print('Train Score: %.2f MSE (%.2f RMSE) (%.2f)' % (trainScore[0], math.sqrt(trainScore[0]), trainScore[1])) testScore = model.evaluate(X_test, y_test, verbose=100) print('Test Score: %.2f MSE (%.2f RMSE) (%.2f)' % (testScore[0], math.sqrt(testScore[0]), testScore[1])) #make predictions print("PREDICTING") p = model.predict(X_test) recent_data = [ recent_data ] # One-sample predictions need list wrapper. Argument must be 3d. recent_data = np.asarray(recent_data) future = model.predict(recent_data) # document results in file print("WRITING TO LOG") file = open(os.path.join(dir_path, "logs/log_initial.txt"), "w") for i in range(0, len(X_train)): for s in range(0, num_timesteps): file.write(str(X_train[i][s]) + "\n") file.write("Target: " + str(y_train[i]) + "\n") file.write("\n") for i in range(0, len(X_test)): for s in range(0, num_timesteps): file.write(str(X_test[i][s]) + "\n") file.write("Target: " + str(y_test[i]) + "\n") file.write("Prediction: " + str(p[i]) + "\n") file.write("\n") file.close() # de-normalize print("DENORMALIZING") for i in range(0, len(p)): p[i] = (p[i] + 1) * ref[round(.9 * len(ref) + i)] y_test[i] = (y_test[i] + 1) * ref[round(.9 * len(ref) + i)] future[0] = (future[0] + 1) * recent_reference[0] recent_data[0] = (recent_data[0] + 1) * recent_reference[0] file = open(os.path.join(dir_path, "logs/predictions.txt"), "a") file.write(str(future[0][0]) + "\n") file.close() # plot historical predictions print("PLOTTING") for i in range(0, len(p)): if i % (num_targets * 2) == 0: plot_index = i #for filling plot indexes plot_indexes = [] plot_values = p[i] for j in range(0, num_targets): plot_indexes.append(plot_index) plot_index += 1 plt.plot(plot_indexes, plot_values, color="red") # plot historical actual plt.plot(y_test[:, 0], color='blue', label='Actual') # actual price history # plot recent prices plot_indexes = [len(y_test) - 1] plot_values = [y_test[-1, 0]] plot_index = None for i in range(0, len(recent_data[0])): plot_values.append(recent_data[0][i][0]) plot_index = len(y_test) + i plot_indexes.append(len(y_test) + i) plt.plot(plot_indexes, plot_values, color='blue') # plot future predictions plot_indexes = [plot_index] plot_values = [recent_data[0][-1][0]] for i in range(0, len(future[0])): plot_index += 1 plot_values.append(future[0][i]) plot_indexes.append(plot_index) plt.plot(plot_indexes, plot_values, color="red", label="Prediction") #show/save plot print("SENDING EMAILS") plt.legend(loc="upper left") plt.title("ETH Price Predictions") plt.xlabel("Hours") plt.ylabel("Price ($)") filename = str(arrow.utcnow().format("YYYY-MM-DD")) plt.savefig(os.path.join(dir_path, "graphs/" + filename)) #plt.show() plt.close() send_email() return
print(xx,"\n") #print(xxx) y = sigmoid(float(y.rstrip())) x_train.append(xx) #x_train.append(x) y_train.append(y) x_train = np.array(x_train) y_train = np.array(y_train) x_train =x_train.reshape(248,1,-1) #print("feat:" ,a,b,c , "lab:" , y , "\n") model = build_model(input_shape=x_train.shape) model.train(x_train,y_train,epochs=500) x_predict = '2017-5-6' x_predict = x_predict.split('-') predict = [] for x in x_predict: predict.append(sigmoid(int(x))) predict = np.array(predict) predict =predict.reshape(1,3,1) print(predict.shape) res = model.predict(x_train) plt.scatter(range(248),res,c='r') plt.scatter(range(248),y_train,c='g') plt.show()
def test_model_pipeline(data): X, y = xy(data) model = build_model() model.fit(X, y)
model_dir = args.output_path + '/models' log_dir = '../logs' tb_dir = args.output_path + '/logs' _clear_logs(log_dir, tb_dir) x_train, x_test, y_train, y_test = _load_data(args.input_path) x_train, y_train = training_xform(x_train, y_train) tf.config.threading.set_inter_op_parallelism_threads = 0 tf.config.threading.set_intra_op_parallelism_threads = 0 save_prefix = model_dir + '/' checkpoint_path = os.path.join(save_prefix, 'cp') hp = HypParams() model = build_model(hp) early_stopping_cb = K.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.00001, restore_best_weights=True, verbose=1, patience=7) checkpoint_cb = K.callbacks.ModelCheckpoint(checkpoint_path, save_best_only=True, verbose=1, monitor='val_loss') lr_scheduler_cb = K.callbacks.ReduceLROnPlateau(monitor='val_loss', min_delta=0.0001, factor=0.5, patience=2, verbose=1)
def test_handles_model(): build_model()
def main(): for_test = False # preprocessing step logger = Logger(path=os.path.abspath('logs/'), name="trainining_process") config, _ = get_config(os.path.abspath('config.json')) # data_helper = DataHelper(config, logger) # missing_dict, history = data_helper.check_missing() # s = pd.Series(missing_dict, name = "Modalities") # s.to_csv(os.path.abspath("missing_two.csv")) # preprocessing step # for (index, id) in enumerate(history.keys()): # if (len(history[id]) > 1): # logger.info(f"{id} - {history[id]}") input_shape = (CHANNELS_NUM, ) + tuple(config.image_shape) # print(mri_image.get_data()) data_generator = DataGenerator(config) main_path = config.main_path hdf_path = f"{main_path}/data/{config.data_set}" data_file = tables.open_file(hdf_path, 'r') train_indices, validaiton_indices = data_generator.validation_split( f"{main_path}/data/train_keys.csv", f"{main_path}/data/validation_kes.csv") train_indices = list(train_indices) validaiton_indices = list(validaiton_indices) # train_generator, validation_generator, train_steps, validation_steps = trainer.get_generators(data_file, train_indices,validaiton_indices, config.patch_shape, config.patch_start_offset, config.batch_size) # logger.info("Generators are created") params = { 'n_labels': config.label_num, 'labels': config.labels, 'dim': tuple(config.image_shape), 'batch_size': 1, 'shuffle': True } logger.info("Generators are creating") train_generator = DataGeneratorNew(train_indices, data_file, **params) validation_generator = DataGeneratorNew(validaiton_indices, data_file, **params) logger.info("Generators are ready") logger.info("Model is loading") if os.path.exists(config.model_path) and for_test: is_compile = not for_test model = load_model( config.model_path, compile=is_compile ) #custom_objects={'loss_gt_':loss_gt(), 'loss_VAE_':loss_VAE, 'dice_coefficient':dice_coefficient}) logger.info("Pretrained model is loaded") else: logger.info("New model was initialized") model = build_model(input_shape=input_shape, output_channels=config.label_num) logger.info("Training model is started") if not for_test: model.fit_generator( generator=train_generator, epochs=config.epochs, validation_data=validation_generator, callbacks=get_callbacks( config.model_path, initial_learning_rate=config.initial_learning_rate, learning_rate_drop=config.learning_rate_drop)) else: # some validation index validation_case = validaiton_indices[0] (y, replication) = model.predict( data_file.root.data[validation_case][np.newaxis]) replication_nii = nib.Nifti1Image(replication[0][0], affine=np.eye(4)) label_nii = nib.Nifti1Image(y[0][0], affine=np.eye(4)) original_nill = nib.Nifti1Image( data_file.root.data[validation_case][0], affine=np.eye(4)) nib.save(label_nii, f'/home/alisher/Desktop/label.nii.gz') nib.save(original_nill, f'/home/alisher/Desktop/original.nii.gz') nib.save(replication_nii, f'/home/alisher/Desktop/replication.nii.gz') data_file.close()