restarts = 1
quality_metric = 'AIC'
output_file = "./results.txt"
alpha, beta = 1., 1.
lambdaU = 1./10.
lambdaV = 1./10.
priors = { 'alpha':alpha, 'beta':beta, 'lambdaU':lambdaU, 'lambdaV':lambdaV }
# Load in the CCLE EC50 dataset
R,M = load_ccle(ic50=False)
# Run the cross-validation framework
#random.seed(42)
#numpy.random.seed(9000)
nested_crossval = LineSearchCrossValidation(
classifier=bnmf_gibbs_optimised,
R=R,
M=M,
values_K=K_range,
folds=no_folds,
priors=priors,
init_UV=init_UV,
iterations=iterations,
restarts=restarts,
quality_metric=quality_metric,
file_performance=output_file
)
nested_crossval.run(burn_in=burn_in,thinning=thinning)
random.seed(42)
numpy.random.seed(9000)
nested_crossval = LineSearchCrossValidation(
classifier=bnmf_vb_optimised,
R=X_min,
M=M,
values_K=K_range,
folds=no_folds,
priors=priors,
init_UV=init_UV,
iterations=iterations,
restarts=restarts,
quality_metric=quality_metric,
file_performance=output_file
)
nested_crossval.run()
"""
All model fits for fold 1, metric AIC: [255811.96901730512, 252136.60278205923, 250856.47342754889, 250226.77983334119, 248494.30974350378].
Best K for fold 1: 10.
Performance: {'R^2': 0.8062985415424776, 'MSE': 2.2509307850276188, 'Rp': 0.89865828998196062}.
All model fits for fold 2, metric AIC: [254648.23575785581, 252738.04961239398, 251251.00158825511, 249894.09518978855, 249008.72134400008].
Best K for fold 2: 10.
Performance: {'R^2': 0.8034479975799519, 'MSE': 2.2875509346902207, 'Rp': 0.8971303533111219}.
All model fits for fold 3, metric AIC: [254815.81149370887, 252028.20607615853, 251536.61512286673, 249416.79922148222, 249128.1772425485].
Best K for fold 3: 10.
Performance: {'R^2': 0.8053582521009495, 'MSE': 2.3186747240876673, 'Rp': 0.89811845194979079}.
All model fits for fold 4, metric AIC: [253723.76692031277, 253550.89846148193, 251558.85284609973, 249711.78828414652, 249128.15196378558].
minimum_TN = 0.1
# Load in the Sanger dataset
(_,X_min,M,_,_,_,_) = load_gdsc(standardised=standardised)
# Run the cross-validation framework
#random.seed(42)
#numpy.random.seed(9000)
nested_crossval = LineSearchCrossValidation(
classifier=nmf_icm,
R=X_min,
M=M,
values_K=K_range,
folds=no_folds,
priors=priors,
init_UV=init_UV,
iterations=iterations,
restarts=restarts,
quality_metric=quality_metric,
file_performance=output_file
)
nested_crossval.run(minimum_TN=minimum_TN)
"""
all_MSE = [3.5039148405029135, 9.0622730084824674, 3.7009069757338917, 3.3451246835265178, 3.1147595748400358, 3.9037354439533258, 13.991970030783968, 3.1814210224127897, 3.2677197491020404, 12.460551868851933]
all_R2 = [0.7072309782081623, 0.2162669348625822, 0.6853079551313846, 0.7144108917311998, 0.7341480430315861, 0.6671037956836574, -0.17013019643779437, 0.7288988508164431, 0.7201731755424339, -0.07478035943340289]
Average MSE: 5.953237719818989 +- 16.165927731904752
Average R^2: 0.49286300691362522 +- 0.11663176700952635
"""
# Run the cross-validation framework
random.seed(42)
numpy.random.seed(9000)
nested_crossval = LineSearchCrossValidation(classifier=bnmf_vb_optimised,
R=X_min,
M=M,
values_K=K_range,
folds=no_folds,
priors=priors,
init_UV=init_UV,
iterations=iterations,
restarts=restarts,
quality_metric=quality_metric,
file_performance=output_file)
nested_crossval.run()
"""
All model fits for fold 1, metric AIC: [255811.96901730512, 252136.60278205923, 250856.47342754889, 250226.77983334119, 248494.30974350378].
Best K for fold 1: 10.
Performance: {'R^2': 0.8062985415424776, 'MSE': 2.2509307850276188, 'Rp': 0.89865828998196062}.
All model fits for fold 2, metric AIC: [254648.23575785581, 252738.04961239398, 251251.00158825511, 249894.09518978855, 249008.72134400008].
Best K for fold 2: 10.
Performance: {'R^2': 0.8034479975799519, 'MSE': 2.2875509346902207, 'Rp': 0.8971303533111219}.
All model fits for fold 3, metric AIC: [254815.81149370887, 252028.20607615853, 251536.61512286673, 249416.79922148222, 249128.1772425485].
Best K for fold 3: 10.
Performance: {'R^2': 0.8053582521009495, 'MSE': 2.3186747240876673, 'Rp': 0.89811845194979079}.
All model fits for fold 4, metric AIC: [253723.76692031277, 253550.89846148193, 251558.85284609973, 249711.78828414652, 249128.15196378558].
Best K for fold 4: 10.
random.seed(42)
numpy.random.seed(9000)
nested_crossval = LineSearchCrossValidation(
classifier=bnmf_gibbs_optimised,
R=X_min,
M=M,
values_K=K_range,
folds=no_folds,
priors=priors,
init_UV=init_UV,
iterations=iterations,
restarts=restarts,
quality_metric=quality_metric,
file_performance=output_file
)
nested_crossval.run(burn_in=burn_in,thinning=thinning)
"""
All model fits for fold 1, metric AIC: [259448.77057675872, 255676.62051582715, 255523.47957909582, 254241.92682732039, 252679.74096095277].
Best K for fold 1: 10.
Performance: {'R^2': 0.8060286774889585, 'MSE': 2.2540667722869601, 'Rp': 0.89804470049443852}.
All model fits for fold 2, metric AIC: [257600.37492492507, 257018.22482604941, 255614.94666957104, 253635.59735767939, 253954.46612192781].
Best K for fold 2: 9.
Performance: {'R^2': 0.8055058118215439, 'MSE': 2.2636012682721058, 'Rp': 0.8978242230542316}.
All model fits for fold 3, metric AIC: [256887.89870918918, 257936.53215861766, 256188.7339930259, 255221.85440717341, 253671.36609325348].
Best K for fold 3: 10.
Performance: {'R^2': 0.8031286514017, 'MSE': 2.3452348985721208, 'Rp': 0.89656801110008155}.
All model fits for fold 4, metric AIC: [259802.08723315704, 257908.12596277907, 253929.13649055792, 254928.65295304605, 253336.49697324197].
random.seed(42)
numpy.random.seed(9000)
nested_crossval = LineSearchCrossValidation(
classifier=nmf_icm,
R=X_min,
M=M,
values_K=K_range,
folds=no_folds,
priors=priors,
init_UV=init_UV,
iterations=iterations,
restarts=restarts,
quality_metric=quality_metric,
file_performance=output_file
)
nested_crossval.run(minimum_TN=minimum_TN)
"""
All model fits for fold 1, metric AIC: [252997.91066856537, 250747.87405923707, 249171.33422154275, 248348.40470946211, 246802.03487854672].
Best K for fold 1: 10.
Performance: {'R^2': 0.807748962293107, 'MSE': 2.2340759985699807, 'Rp': 0.89958671281823332}.
All model fits for fold 2, metric AIC: [252993.4545215781, 250747.87405923707, 249171.33422154275, 248348.40470946211, 246802.03487854672].
Best K for fold 2: 10.
Performance: {'R^2': 0.8075286864183611, 'MSE': 2.2400582434352443, 'Rp': 0.89953229833400639}.
All model fits for fold 3, metric AIC: [252998.07360170182, 250747.87405923707, 249171.33422154275, 248348.40470946211, 246802.03487854672].
Best K for fold 3: 10.
Performance: {'R^2': 0.8043624727400118, 'MSE': 2.3305369708045012, 'Rp': 0.89777844572303478}.
All model fits for fold 4, metric AIC: [252998.33798114926, 250747.87405923707, 249171.33422154275, 248348.40470946211, 246802.03487854672].
