def _run_par_comput(self,
model,
inputdir,
outputdir,
cluster="local",
local_cpus=1,
runs=1,
output_msg=False):
__doc__ = Simul._run_par_comput.__doc__
if self._language is None:
logger.error(self._language_not_found_msg)
return False
model_group = self._get_model_group(model)
# run in parallel
# To make things simple, the last 10 character of groupid are extracted and reversed.
# This string will be likely different from groupid and is the string to replace with
# the iteration number.
str_to_replace = self._groupid[10::-1]
opts = " "
if self._options:
opts = " " + self._options + " "
command = self._language + opts + os.path.join(inputdir, model) + \
" " + model_group + str_to_replace + ".csv"
command = command.replace('\\', '\\\\')
if not parcomp(command, str_to_replace, outputdir, cluster, runs,
local_cpus, output_msg):
return False
if not self._move_reports('.', outputdir, model, self._groupid):
return False
return True
def _run_par_comput(self, inputdir, model, outputdir, cluster="local", local_cpus=1, runs=1, output_msg=False):
__doc__ = Simul._run_par_comput.__doc__
if self._copasi is None:
logger.error(self._copasi_not_found_msg)
return False
model_group = self._get_model_group(model)
# replicate the models
for i in range(1, runs + 1):
shutil.copyfile(os.path.join(inputdir, model), os.path.join(inputdir, model_group) + str(i) + ".cps")
replace_str_in_file(os.path.join(inputdir, model_group) + str(i) + ".cps",
os.path.splitext(model)[0] + ".csv",
model_group + str(i) + ".csv")
replace_str_in_file(os.path.join(inputdir, model_group) + str(i) + ".cps",
os.path.splitext(model)[0] + ".txt",
model_group + str(i) + ".csv")
replace_str_in_file(os.path.join(inputdir, model_group) + str(i) + ".cps",
os.path.splitext(model)[0] + ".tsv",
model_group + str(i) + ".csv")
replace_str_in_file(os.path.join(inputdir, model_group) + str(i) + ".cps",
os.path.splitext(model)[0] + ".dat",
model_group + str(i) + ".csv")
# run copasi in parallel
# To make things simple, the last 10 character of groupid are extracted and reversed.
# This string will be likely different from groupid and is the string to replace with
# the iteration number.
str_to_replace = self._groupid[10::-1]
command = self._copasi + " " + os.path.join(inputdir, model_group + str_to_replace + ".cps")
command = command.replace('\\', '\\\\')
if not parcomp(command, str_to_replace, outputdir, cluster, runs, local_cpus, output_msg):
return False
if not self._move_reports(inputdir, outputdir, model, self._groupid):
return False
return True
def analyse_data(cls,
model,
knock_down_only,
outputdir,
sim_data_folder,
sim_plots_folder,
runs,
local_cpus,
percent_levels,
min_level,
max_level,
levels_number,
homogeneous_lines,
cluster="local",
xaxis_label='',
yaxis_label=''):
"""
The second pipeline step: data analysis.
:param model: the model name
:param knock_down_only: True for knock down simulation, false if also scanning over expression.
:param outputdir: the directory containing the results
:param sim_data_folder: the folder containing the simulated data sets
:param sim_plots_folder: the folder containing the generated plots
:param runs: the number of simulations
:param local_cpus: the number of cpus
:param percent_levels: True if the levels are percents.
:param min_level: the minimum level
:param max_level: the maximum level
:param levels_number: the number of levels
:param homogeneous_lines: True if generated line style should be homogeneous
:param cluster: local, lsf for Load Sharing Facility, sge for Sun Grid Engine.
:param xaxis_label: the name of the x axis (e.g. Time [min])
:param yaxis_label: the name of the y axis (e.g. Level [a.u.])
:return: True if the task was completed successfully, False otherwise.
"""
# some control
if not os.path.exists(os.path.join(outputdir, sim_data_folder)):
logger.error("input_dir " +
os.path.join(outputdir, sim_data_folder) +
" does not exist. Generate some data first.")
return False
#if float(min_level) < 0:
# logger.error("min_level MUST BE non negative. Please, check your configuration file.")
# return False
if float(max_level) < 0:
logger.error(
"max_level MUST BE non negative. Please, check your configuration file."
)
return False
if float(max_level) <= float(min_level):
logger.error(
"min_level MUST BE lower than max_level. Please, check your configuration file."
)
return False
if int(local_cpus) < 1:
logger.error(
"variable local_cpus must be greater than 0. Please, check your configuration file."
)
return False
if int(runs) < 1:
logger.error(
"variable runs must be greater than 0. Please, check your configuration file."
)
return False
if int(levels_number) < 1:
logger.error(
"variable levels_number must be greater than 0. Please, check your configuration file."
)
return False
if percent_levels and float(max_level) < 100:
logger.error(
"max_level cannot be less than 100 (=ctrl) if option `percent_levels` is True. "
"Please, check your configuration file.")
return False
# folder preparation
refresh(os.path.join(outputdir, sim_plots_folder),
os.path.splitext(model)[0])
str_to_replace = get_rand_alphanum_str(10)
# requires devtools::install_github("pdp10/sbpiper")
if not is_r_package_installed('sbpiper'):
logger.critical('R package `sbpiper` was not found. Abort.')
return False
command = 'R --quiet -e \'library(sbpiper); sbpiper_ps1(\"' + model + \
'\", \"' + str(knock_down_only).upper() + \
'\", \"' + os.path.join(outputdir, sim_data_folder) + \
'\", \"' + os.path.join(outputdir, sim_plots_folder) + \
'\", \"' + str_to_replace + \
'\", \"' + str(percent_levels).upper() + \
'\", \"' + str(min_level) + \
'\", \"' + str(max_level) + \
'\", \"' + str(levels_number) + \
'\", \"' + str(homogeneous_lines).upper()
# we replace \\ with / otherwise subprocess complains on windows systems.
command = command.replace('\\', '\\\\')
# We do this to make sure that characters like [ or ] don't cause troubles.
command += '\", \"' + xaxis_label + \
'\", \"' + yaxis_label + \
'\")\''
if not parcomp(command, str_to_replace, outputdir, cluster, runs,
local_cpus, False):
return False
if len(
glob.glob(
os.path.join(outputdir, sim_plots_folder,
os.path.splitext(model)[0] + '*.pdf'))) == 0:
return False
return True
def analyse_data(cls,
model,
scanned_par1,
scanned_par2,
inputdir,
outputdir,
cluster='local',
local_cpus=1,
runs=1):
"""
The second pipeline step: data analysis.
:param model: the model name
:param scanned_par1: the first scanned parameter
:param scanned_par2: the second scanned parameter
:param inputdir: the directory containing the simulated data sets to process
:param outputdir: the directory to store the performed analysis
:param cluster: local, lsf for Load Sharing Facility, sge for Sun Grid Engine.
:param local_cpus: the number of CPU.
:param runs: the number of model simulation
:return: True if the task was completed successfully, False otherwise.
"""
if not os.path.exists(inputdir):
logger.error("input_dir " + inputdir +
" does not exist. Generate some data first.")
return False
if runs < 1:
logger.error(
"variable `runs` must be greater than 0. Please, check your configuration file."
)
return False
if int(local_cpus) < 1:
logger.error(
"variable local_cpus must be greater than 0. Please, check your configuration file."
)
return False
# folder preparation
refresh(outputdir, os.path.splitext(model)[0])
str_to_replace = get_rand_alphanum_str(10)
# requires devtools::install_github("pdp10/sbpiper")
if not is_r_package_installed('sbpiper'):
logger.critical('R package `sbpiper` was not found. Abort.')
return False
command = 'R --quiet -e \'library(sbpiper); sbpiper_ps2(\"' + model + \
'\", \"' + scanned_par1 + '\", \"' + scanned_par2 + \
'\", \"' + inputdir + \
'\", \"' + outputdir + \
'\", \"' + str_to_replace
# we replace \\ with / otherwise subprocess complains on windows systems.
command = command.replace('\\', '\\\\')
# We do this to make sure that characters like [ or ] don't cause troubles.
command += '\")\''
if not parcomp(command, str_to_replace, outputdir, cluster, int(runs),
int(local_cpus), False):
return False
if len(
glob.glob(
os.path.join(outputdir,
os.path.splitext(model)[0] + '*.pdf'))) == 0:
return False
return True
def analyse_data(cls,
simulator,
model,
inputdir,
outputdir,
fileout_final_estims,
fileout_all_estims,
fileout_param_estim_best_fits_details,
fileout_param_estim_details,
fileout_param_estim_summary,
sim_plots_dir,
best_fits_percent,
data_point_num,
cluster='local',
plot_2d_66cl_corr=False,
plot_2d_95cl_corr=False,
plot_2d_99cl_corr=False,
logspace=True,
scientific_notation=True):
"""
The second pipeline step: data analysis.
:param simulator: the name of the simulator (e.g. Copasi)
:param model: the model name
:param inputdir: the directory containing the simulation data
:param outputdir: the directory to store the results
:param fileout_final_estims: the name of the file containing final parameter sets with the objective value
:param fileout_all_estims: the name of the file containing all the parameter sets with the objective value
:param fileout_param_estim_best_fits_details: the file containing the statistics for the best fits analysis
:param fileout_param_estim_details: the file containing the statistics for the PLE analysis
:param fileout_param_estim_summary: the name of the file containing the summary for the parameter estimation
:param sim_plots_dir: the directory of the simulation plots
:param best_fits_percent: the percent to consider for the best fits
:param data_point_num: the number of data points
:param cluster: local, lsf for Load Sharing Facility, sge for Sun Grid Engine.
:param plot_2d_66cl_corr: True if 2 dim plots for the parameter sets within 66% should be plotted
:param plot_2d_95cl_corr: True if 2 dim plots for the parameter sets within 95% should be plotted
:param plot_2d_99cl_corr: True if 2 dim plots for the parameter sets within 99% should be plotted
:param logspace: True if parameters should be plotted in log space
:param scientific_notation: True if axis labels should be plotted in scientific notation
:return: True if the task was completed successfully, False otherwise.
"""
if not os.path.exists(inputdir) or not os.listdir(inputdir):
logger.error(
"inputdir " + inputdir +
" does not exist or is empty. Generate some data first.")
return False
if int(best_fits_percent) < 1 or int(best_fits_percent) > 100:
logger.error(
"variable `best_fits_percent` must be in (0, 100]. Please, check your configuration file."
)
return False
if int(data_point_num) < 0:
logger.error(
"variable `data_point_num` must be >= 0. To visualise thresholds, `data_point_num` must be "
"greater than the number of estimated parameters. Please, check your configuration file."
)
return False
refresh(sim_plots_dir, os.path.splitext(model)[0])
logger.info("Collect results:")
# Collect and summarises the parameter estimation results
try:
sim = cls.get_simul_obj(simulator)
files_num = sim.get_best_fits(inputdir, outputdir,
fileout_final_estims)
sim.get_all_fits(inputdir, outputdir, fileout_all_estims)
logger.info('Files retrieved: ' + str(files_num))
if files_num == 0:
return False
except Exception as e:
logger.error("simulator: " + simulator + " not found.")
import traceback
logger.debug(traceback.format_exc())
return False
# we don't replace any string in files. So let's use a substring which won't even be in any file.
str_to_replace = get_rand_alphanum_str(10)
logger.info("\n")
logger.info("Fits analysis:")
# requires devtools::install_github("pdp10/sbpiper")
if not is_r_package_installed('sbpiper'):
logger.critical('R package `sbpiper` was not found. Abort.')
return False
command = 'R --quiet -e \'library(sbpiper); sbpiper_pe(\"' + model + \
'\", \"' + os.path.join(outputdir, fileout_final_estims) + \
'\", \"' + os.path.join(outputdir, fileout_all_estims) + \
'\", \"' + sim_plots_dir + \
'\", ' + str(data_point_num) + \
', \"' + os.path.join(outputdir, fileout_param_estim_best_fits_details) + \
'\", \"' + os.path.join(outputdir, fileout_param_estim_details) + \
'\", \"' + os.path.join(outputdir, fileout_param_estim_summary) + \
'\", ' + str(best_fits_percent) + \
', ' + str(plot_2d_66cl_corr).upper() + \
', ' + str(plot_2d_95cl_corr).upper() + \
', ' + str(plot_2d_99cl_corr).upper() + \
', ' + str(logspace).upper() + \
', ' + str(scientific_notation).upper()
# we replace \\ with / otherwise subprocess complains on windows systems.
command = command.replace('\\', '\\\\')
# We do this to make sure that characters like [ or ] don't cause troubles.
command += ')\''
if not parcomp(command, str_to_replace, outputdir, cluster, 1, 1,
False):
return False
if len(
glob.glob(
os.path.join(sim_plots_dir,
os.path.splitext(model)[0] + '*.pdf'))) == 0:
return False
return True
def analyse_data(cls,
simulator,
model,
inputdir,
outputdir,
sim_plots_dir,
exp_dataset,
plot_exp_dataset,
exp_dataset_alpha=1.0,
cluster="local",
local_cpus=2,
xaxis_label='',
yaxis_label=''):
"""
The second pipeline step: data analysis.
:param simulator: the name of the simulator (e.g. Copasi)
:param model: the model name
:param inputdir: the directory containing the data to analyse
:param outputdir: the output directory containing the results
:param sim_plots_dir: the directory to save the plots
:param exp_dataset: the full path of the experimental data set
:param plot_exp_dataset: True if the experimental data set should also be plotted
:param exp_dataset_alpha: the alpha level for the data set
:param cluster: local, lsf for Load Sharing Facility, sge for Sun Grid Engine.
:param local_cpus: the number of CPUs.
:param xaxis_label: the label for the x axis (e.g. Time [min])
:param yaxis_label: the label for the y axis (e.g. Level [a.u.])
:return: True if the task was completed successfully, False otherwise.
"""
if not os.path.exists(inputdir):
logger.error("inputdir " + inputdir +
" does not exist. Generate some data first.")
return False
if float(exp_dataset_alpha) > 1.0 or float(exp_dataset_alpha) < 0.0:
logger.warning(
"variable exp_dataset_alpha must be in [0,1]. Please, check your configuration file."
)
exp_dataset_alpha = 1.0
sim_data_by_var_dir = os.path.join(outputdir, "simulate_data_by_var")
# folder preparation
refresh(sim_plots_dir, os.path.splitext(model)[0])
refresh(sim_data_by_var_dir, os.path.splitext(model)[0])
# Read the columns to process
try:
sim = cls.get_simul_obj(simulator)
except TypeError as e:
logger.error("simulator: " + simulator + " not found.")
logger.debug(traceback.format_exc())
return False
try:
columns = sim.get_sim_columns(inputdir)
except Exception as e:
logger.error(str(e))
logger.debug(traceback.format_exc())
return False
str_to_replace = 'COLUMN_TO_REPLACE'
logger.info("Analysing generated simulations:")
# requires devtools::install_github("pdp10/sbpiper")
if not is_r_package_installed('sbpiper'):
logger.critical('R package `sbpiper` was not found. Abort.')
return False
command = 'R --quiet -e \'library(sbpiper); sbpiper_sim(\"' + model + \
'\", \"' + inputdir + '\", \"' + sim_plots_dir + \
'\", \"' + os.path.join(outputdir, 'sim_stats_' + model + '_' + str_to_replace + '.csv') + \
'\", \"' + os.path.join(sim_data_by_var_dir, model + '_' + str_to_replace + '.csv') + \
'\", \"' + exp_dataset + \
'\", ' + str(plot_exp_dataset).upper() + \
', \"' + str(exp_dataset_alpha)
# we replace \\ with / otherwise subprocess complains on windows systems.
command = command.replace('\\', '\\\\')
# We do this to make sure that characters like [ or ] don't cause troubles.
command += '\", \"' + xaxis_label + \
'\", \"' + yaxis_label + \
'\", \"' + str_to_replace + \
'\")\''
if not parcomp(command, str_to_replace, outputdir, cluster, 1,
local_cpus, False, columns):
return False
if len(
glob.glob(
os.path.join(sim_plots_dir,
os.path.splitext(model)[0] + '*.pdf'))) == 0:
return False
return True