def setUpClass(cls):
os.chdir(cls._rscript_folder)
if not is_r_package_installed("reshape2"):
cls._output = "R reshape2 not found: SKIP ... "
elif not is_r_package_installed("deSolve"):
cls._output = "R deSolve not found: SKIP ... "
elif not is_r_package_installed("minpack.lm"):
cls._output = "R minpack.lm not found: SKIP ... "
def test_sim_r(self):
if not is_r_package_installed("reshape2"):
sys.stdout.write("R reshape2 not found: SKIP ... ")
sys.stdout.flush()
elif not is_r_package_installed("deSolve"):
sys.stdout.write("R deSolve not found: SKIP ... ")
sys.stdout.flush()
elif not is_r_package_installed("minpack.lm"):
sys.stdout.write("R minpack.lm not found: SKIP ... ")
sys.stdout.flush()
else:
self.assertEqual(sbpipe(simulate="sim_simple_reacts.yaml", quiet=True), 0)
def analyse_data(cls,
model,
knock_down_only,
outputdir,
sim_data_folder,
sim_plots_folder,
runs,
local_cpus,
percent_levels,
min_level,
max_level,
levels_number,
homogeneous_lines,
cluster="local",
xaxis_label='',
yaxis_label=''):
"""
The second pipeline step: data analysis.
:param model: the model name
:param knock_down_only: True for knock down simulation, false if also scanning over expression.
:param outputdir: the directory containing the results
:param sim_data_folder: the folder containing the simulated data sets
:param sim_plots_folder: the folder containing the generated plots
:param runs: the number of simulations
:param local_cpus: the number of cpus
:param percent_levels: True if the levels are percents.
:param min_level: the minimum level
:param max_level: the maximum level
:param levels_number: the number of levels
:param homogeneous_lines: True if generated line style should be homogeneous
:param cluster: local, lsf for Load Sharing Facility, sge for Sun Grid Engine.
:param xaxis_label: the name of the x axis (e.g. Time [min])
:param yaxis_label: the name of the y axis (e.g. Level [a.u.])
:return: True if the task was completed successfully, False otherwise.
"""
# some control
if not os.path.exists(os.path.join(outputdir, sim_data_folder)):
logger.error("input_dir " +
os.path.join(outputdir, sim_data_folder) +
" does not exist. Generate some data first.")
return False
#if float(min_level) < 0:
# logger.error("min_level MUST BE non negative. Please, check your configuration file.")
# return False
if float(max_level) < 0:
logger.error(
"max_level MUST BE non negative. Please, check your configuration file."
)
return False
if float(max_level) <= float(min_level):
logger.error(
"min_level MUST BE lower than max_level. Please, check your configuration file."
)
return False
if int(local_cpus) < 1:
logger.error(
"variable local_cpus must be greater than 0. Please, check your configuration file."
)
return False
if int(runs) < 1:
logger.error(
"variable runs must be greater than 0. Please, check your configuration file."
)
return False
if int(levels_number) < 1:
logger.error(
"variable levels_number must be greater than 0. Please, check your configuration file."
)
return False
if percent_levels and float(max_level) < 100:
logger.error(
"max_level cannot be less than 100 (=ctrl) if option `percent_levels` is True. "
"Please, check your configuration file.")
return False
# folder preparation
refresh(os.path.join(outputdir, sim_plots_folder),
os.path.splitext(model)[0])
str_to_replace = get_rand_alphanum_str(10)
# requires devtools::install_github("pdp10/sbpiper")
if not is_r_package_installed('sbpiper'):
logger.critical('R package `sbpiper` was not found. Abort.')
return False
command = 'R --quiet -e \'library(sbpiper); sbpiper_ps1(\"' + model + \
'\", \"' + str(knock_down_only).upper() + \
'\", \"' + os.path.join(outputdir, sim_data_folder) + \
'\", \"' + os.path.join(outputdir, sim_plots_folder) + \
'\", \"' + str_to_replace + \
'\", \"' + str(percent_levels).upper() + \
'\", \"' + str(min_level) + \
'\", \"' + str(max_level) + \
'\", \"' + str(levels_number) + \
'\", \"' + str(homogeneous_lines).upper()
# we replace \\ with / otherwise subprocess complains on windows systems.
command = command.replace('\\', '\\\\')
# We do this to make sure that characters like [ or ] don't cause troubles.
command += '\", \"' + xaxis_label + \
'\", \"' + yaxis_label + \
'\")\''
if not parcomp(command, str_to_replace, outputdir, cluster, runs,
local_cpus, False):
return False
if len(
glob.glob(
os.path.join(outputdir, sim_plots_folder,
os.path.splitext(model)[0] + '*.pdf'))) == 0:
return False
return True
def analyse_data(cls,
model,
scanned_par1,
scanned_par2,
inputdir,
outputdir,
cluster='local',
local_cpus=1,
runs=1):
"""
The second pipeline step: data analysis.
:param model: the model name
:param scanned_par1: the first scanned parameter
:param scanned_par2: the second scanned parameter
:param inputdir: the directory containing the simulated data sets to process
:param outputdir: the directory to store the performed analysis
:param cluster: local, lsf for Load Sharing Facility, sge for Sun Grid Engine.
:param local_cpus: the number of CPU.
:param runs: the number of model simulation
:return: True if the task was completed successfully, False otherwise.
"""
if not os.path.exists(inputdir):
logger.error("input_dir " + inputdir +
" does not exist. Generate some data first.")
return False
if runs < 1:
logger.error(
"variable `runs` must be greater than 0. Please, check your configuration file."
)
return False
if int(local_cpus) < 1:
logger.error(
"variable local_cpus must be greater than 0. Please, check your configuration file."
)
return False
# folder preparation
refresh(outputdir, os.path.splitext(model)[0])
str_to_replace = get_rand_alphanum_str(10)
# requires devtools::install_github("pdp10/sbpiper")
if not is_r_package_installed('sbpiper'):
logger.critical('R package `sbpiper` was not found. Abort.')
return False
command = 'R --quiet -e \'library(sbpiper); sbpiper_ps2(\"' + model + \
'\", \"' + scanned_par1 + '\", \"' + scanned_par2 + \
'\", \"' + inputdir + \
'\", \"' + outputdir + \
'\", \"' + str_to_replace
# we replace \\ with / otherwise subprocess complains on windows systems.
command = command.replace('\\', '\\\\')
# We do this to make sure that characters like [ or ] don't cause troubles.
command += '\")\''
if not parcomp(command, str_to_replace, outputdir, cluster, int(runs),
int(local_cpus), False):
return False
if len(
glob.glob(
os.path.join(outputdir,
os.path.splitext(model)[0] + '*.pdf'))) == 0:
return False
return True
def analyse_data(cls,
simulator,
model,
inputdir,
outputdir,
fileout_final_estims,
fileout_all_estims,
fileout_param_estim_best_fits_details,
fileout_param_estim_details,
fileout_param_estim_summary,
sim_plots_dir,
best_fits_percent,
data_point_num,
cluster='local',
plot_2d_66cl_corr=False,
plot_2d_95cl_corr=False,
plot_2d_99cl_corr=False,
logspace=True,
scientific_notation=True):
"""
The second pipeline step: data analysis.
:param simulator: the name of the simulator (e.g. Copasi)
:param model: the model name
:param inputdir: the directory containing the simulation data
:param outputdir: the directory to store the results
:param fileout_final_estims: the name of the file containing final parameter sets with the objective value
:param fileout_all_estims: the name of the file containing all the parameter sets with the objective value
:param fileout_param_estim_best_fits_details: the file containing the statistics for the best fits analysis
:param fileout_param_estim_details: the file containing the statistics for the PLE analysis
:param fileout_param_estim_summary: the name of the file containing the summary for the parameter estimation
:param sim_plots_dir: the directory of the simulation plots
:param best_fits_percent: the percent to consider for the best fits
:param data_point_num: the number of data points
:param cluster: local, lsf for Load Sharing Facility, sge for Sun Grid Engine.
:param plot_2d_66cl_corr: True if 2 dim plots for the parameter sets within 66% should be plotted
:param plot_2d_95cl_corr: True if 2 dim plots for the parameter sets within 95% should be plotted
:param plot_2d_99cl_corr: True if 2 dim plots for the parameter sets within 99% should be plotted
:param logspace: True if parameters should be plotted in log space
:param scientific_notation: True if axis labels should be plotted in scientific notation
:return: True if the task was completed successfully, False otherwise.
"""
if not os.path.exists(inputdir) or not os.listdir(inputdir):
logger.error(
"inputdir " + inputdir +
" does not exist or is empty. Generate some data first.")
return False
if int(best_fits_percent) < 1 or int(best_fits_percent) > 100:
logger.error(
"variable `best_fits_percent` must be in (0, 100]. Please, check your configuration file."
)
return False
if int(data_point_num) < 0:
logger.error(
"variable `data_point_num` must be >= 0. To visualise thresholds, `data_point_num` must be "
"greater than the number of estimated parameters. Please, check your configuration file."
)
return False
refresh(sim_plots_dir, os.path.splitext(model)[0])
logger.info("Collect results:")
# Collect and summarises the parameter estimation results
try:
sim = cls.get_simul_obj(simulator)
files_num = sim.get_best_fits(inputdir, outputdir,
fileout_final_estims)
sim.get_all_fits(inputdir, outputdir, fileout_all_estims)
logger.info('Files retrieved: ' + str(files_num))
if files_num == 0:
return False
except Exception as e:
logger.error("simulator: " + simulator + " not found.")
import traceback
logger.debug(traceback.format_exc())
return False
# we don't replace any string in files. So let's use a substring which won't even be in any file.
str_to_replace = get_rand_alphanum_str(10)
logger.info("\n")
logger.info("Fits analysis:")
# requires devtools::install_github("pdp10/sbpiper")
if not is_r_package_installed('sbpiper'):
logger.critical('R package `sbpiper` was not found. Abort.')
return False
command = 'R --quiet -e \'library(sbpiper); sbpiper_pe(\"' + model + \
'\", \"' + os.path.join(outputdir, fileout_final_estims) + \
'\", \"' + os.path.join(outputdir, fileout_all_estims) + \
'\", \"' + sim_plots_dir + \
'\", ' + str(data_point_num) + \
', \"' + os.path.join(outputdir, fileout_param_estim_best_fits_details) + \
'\", \"' + os.path.join(outputdir, fileout_param_estim_details) + \
'\", \"' + os.path.join(outputdir, fileout_param_estim_summary) + \
'\", ' + str(best_fits_percent) + \
', ' + str(plot_2d_66cl_corr).upper() + \
', ' + str(plot_2d_95cl_corr).upper() + \
', ' + str(plot_2d_99cl_corr).upper() + \
', ' + str(logspace).upper() + \
', ' + str(scientific_notation).upper()
# we replace \\ with / otherwise subprocess complains on windows systems.
command = command.replace('\\', '\\\\')
# We do this to make sure that characters like [ or ] don't cause troubles.
command += ')\''
if not parcomp(command, str_to_replace, outputdir, cluster, 1, 1,
False):
return False
if len(
glob.glob(
os.path.join(sim_plots_dir,
os.path.splitext(model)[0] + '*.pdf'))) == 0:
return False
return True
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from sbpipe.utils.parcomp import run_cmd
from sbpipe.utils.dependencies import is_r_package_installed
if not is_r_package_installed('sbpiper'):
raise Exception('R package `sbpiper` was not found. Abort.')
def ps1_analyse_plot(model_name, inhibition_only, inputdir, outputdir, repeat,
percent_levels, min_level, max_level, levels_number,
xaxis_label, yaxis_label):
"""
Plot model single parameter scan time courses (Python wrapper).
:param model_name: the model name without extension
:param inhibition_only: true if the scanning only decreases the variable amount (inhibition only)
:param inputdir: the input directory containing the simulated data
:param outputdir: the output directory that will contain the simulated plots
:param repeat: the simulation number
:param percent_levels: true if scanning levels are in percent
def analyse_data(cls,
simulator,
model,
inputdir,
outputdir,
sim_plots_dir,
exp_dataset,
plot_exp_dataset,
exp_dataset_alpha=1.0,
cluster="local",
local_cpus=2,
xaxis_label='',
yaxis_label=''):
"""
The second pipeline step: data analysis.
:param simulator: the name of the simulator (e.g. Copasi)
:param model: the model name
:param inputdir: the directory containing the data to analyse
:param outputdir: the output directory containing the results
:param sim_plots_dir: the directory to save the plots
:param exp_dataset: the full path of the experimental data set
:param plot_exp_dataset: True if the experimental data set should also be plotted
:param exp_dataset_alpha: the alpha level for the data set
:param cluster: local, lsf for Load Sharing Facility, sge for Sun Grid Engine.
:param local_cpus: the number of CPUs.
:param xaxis_label: the label for the x axis (e.g. Time [min])
:param yaxis_label: the label for the y axis (e.g. Level [a.u.])
:return: True if the task was completed successfully, False otherwise.
"""
if not os.path.exists(inputdir):
logger.error("inputdir " + inputdir +
" does not exist. Generate some data first.")
return False
if float(exp_dataset_alpha) > 1.0 or float(exp_dataset_alpha) < 0.0:
logger.warning(
"variable exp_dataset_alpha must be in [0,1]. Please, check your configuration file."
)
exp_dataset_alpha = 1.0
sim_data_by_var_dir = os.path.join(outputdir, "simulate_data_by_var")
# folder preparation
refresh(sim_plots_dir, os.path.splitext(model)[0])
refresh(sim_data_by_var_dir, os.path.splitext(model)[0])
# Read the columns to process
try:
sim = cls.get_simul_obj(simulator)
except TypeError as e:
logger.error("simulator: " + simulator + " not found.")
logger.debug(traceback.format_exc())
return False
try:
columns = sim.get_sim_columns(inputdir)
except Exception as e:
logger.error(str(e))
logger.debug(traceback.format_exc())
return False
str_to_replace = 'COLUMN_TO_REPLACE'
logger.info("Analysing generated simulations:")
# requires devtools::install_github("pdp10/sbpiper")
if not is_r_package_installed('sbpiper'):
logger.critical('R package `sbpiper` was not found. Abort.')
return False
command = 'R --quiet -e \'library(sbpiper); sbpiper_sim(\"' + model + \
'\", \"' + inputdir + '\", \"' + sim_plots_dir + \
'\", \"' + os.path.join(outputdir, 'sim_stats_' + model + '_' + str_to_replace + '.csv') + \
'\", \"' + os.path.join(sim_data_by_var_dir, model + '_' + str_to_replace + '.csv') + \
'\", \"' + exp_dataset + \
'\", ' + str(plot_exp_dataset).upper() + \
', \"' + str(exp_dataset_alpha)
# we replace \\ with / otherwise subprocess complains on windows systems.
command = command.replace('\\', '\\\\')
# We do this to make sure that characters like [ or ] don't cause troubles.
command += '\", \"' + xaxis_label + \
'\", \"' + yaxis_label + \
'\", \"' + str_to_replace + \
'\")\''
if not parcomp(command, str_to_replace, outputdir, cluster, 1,
local_cpus, False, columns):
return False
if len(
glob.glob(
os.path.join(sim_plots_dir,
os.path.splitext(model)[0] + '*.pdf'))) == 0:
return False
return True
def test_stoch_sim_r_cox_ingersoll_ross_process(self):
if not is_r_package_installed("sde"):
sys.stdout.write("R sde not found: SKIP ... ")
sys.stdout.flush()
else:
self.assertEqual(sbpipe(simulate="sde_cox_ingersoll_ross_process.yaml", quiet=True), 0)
def test_stoch_sim_r_periodic_drift(self):
if not is_r_package_installed("sde"):
sys.stdout.write("R sde not found: SKIP ... ")
sys.stdout.flush()
else:
self.assertEqual(sbpipe(simulate="sde_periodic_drift.yaml", quiet=True), 0)
def test_sim_r_pde_lotka_volterra(self):
if not is_r_package_installed("deSolve"):
sys.stdout.write("R deSolve not found: SKIP ... ")
sys.stdout.flush()
else:
self.assertEqual(sbpipe(simulate="2Dpde_lotka_volterra.yaml", quiet=True), 0)