def test_nonexistent_env_settings_file(
self, tmpdir, minimal_project_conf_path,
env_config_filepath, envconf_filename):
""" Project doesn't require default environment settings. """
# Create name to nonexistent file based on true default file.
envconf_dirpath, _ = os.path.split(env_config_filepath)
misnamed_envconf = os.path.join(envconf_dirpath, envconf_filename)
# Create and add log message handler for expected errors.
logfile = tmpdir.join("project-error-messages.log").strpath
expected_error_message_handler = logging.FileHandler(logfile, mode='w')
expected_error_message_handler.setLevel(logging.ERROR)
looper.models._LOGGER.handlers.append(expected_error_message_handler)
# Create Project, expecting to generate error messages.
project = Project(minimal_project_conf_path,
default_compute=misnamed_envconf)
# Remove the temporary message handler.
del looper.models._LOGGER.handlers[-1]
# Ensure nulls for all relevant Project attributes.
self._assert_null_compute_environment(project)
# We should have two error messages, describing the exception caught
# during default environment parsing and that it couldn't be set.
with open(logfile, 'r') as messages:
exception_messages = messages.readlines()
try:
assert 2 == len(exception_messages)
except AssertionError:
print("Exception messages: {}".format(exception_messages))
raise
def test_single_sample(self, tmpdir, path_proj_conf_file,
which_sample_index):
""" Single Sample is perfectly valid for Project and sheet. """
# Pull out the values for the current sample.
values = DATA[which_sample_index]
# Write the annotations.
anns_path = os.path.join(tmpdir.strpath, NAME_ANNOTATIONS_FILE)
with open(anns_path, 'w') as anns_file:
anns_file.write("{}\n".format(",".join(COLUMNS)))
anns_file.write("{}\n".format(",".join([str(v) for v in values])))
# Build the sheet.
p = Project(path_proj_conf_file)
sheet = p.build_sheet()
# It should be a single-row DataFrame.
assert isinstance(sheet, pd.DataFrame)
assert 1 == len(sheet)
assert 1 == p.num_samples
# There will be additional values added from the Project,
# but the core data values will have remained the same.
sample = list(p.samples)[0]
for attr, exp_val in zip(COLUMNS, values):
obs_val = getattr(sample, attr)
try:
assert exp_val == obs_val
except AssertionError as e:
try:
assert exp_val == int(obs_val)
except AssertionError:
raise e
def project(request, tmpdir, env_config_filepath):
""" Provide requesting test case with a basic Project instance. """
# Write just the sample names as the annotations.
annotations_filename = "anns-fill.csv"
anns_path = tmpdir.join(annotations_filename).strpath
num_samples = request.getfixturevalue("num_samples")
df = pd.DataFrame(
OrderedDict([("sample_name",
["sample{}".format(i) for i in range(num_samples)]),
("data", range(num_samples))]))
with open(anns_path, 'w') as anns_file:
df.to_csv(anns_file, sep="\t", index=False)
# Create the Project config data.
config_data = {"metadata": {SAMPLE_ANNOTATIONS_KEY: annotations_filename}}
if request.getfixturevalue(request.cls.CONFIG_DATA_PATHS_HOOK):
config_data["paths"] = {}
paths_dest = config_data["paths"]
else:
paths_dest = config_data["metadata"]
# Add the paths data to the Project config.
for path_name, path in PATH_BY_TYPE.items():
paths_dest[path_name] = os.path.join(tmpdir.strpath, path)
# Write the Project config file.
conf_path = tmpdir.join("proj-conf.yaml").strpath
with open(conf_path, 'w') as conf_file:
yaml.safe_dump(config_data, conf_file)
return Project(conf_path, default_compute=env_config_filepath)
def test_no_default_env_settings_provided(
self, minimal_project_conf_path,
explicit_null, compute_env_attname):
""" Project doesn't require default environment settings. """
kwargs = {"default_compute": None} if explicit_null else {}
project = Project(minimal_project_conf_path, **kwargs)
observed_attribute = getattr(project, compute_env_attname)
expected_attribute = \
self.default_compute_settings(project)[compute_env_attname]
if compute_env_attname == "compute":
# 'compute' refers to a section in the default environment
# settings file and also to a Project attribute. A Project
# instance selects just one of the options in the 'compute'
# section of the file as the value for its 'compute' attribute.
expected_attribute = expected_attribute["default"]
observed_attribute = _compute_paths_to_names(observed_attribute)
elif compute_env_attname == "environment":
envs_with_reduced_filepaths = \
_env_paths_to_names(observed_attribute["compute"])
observed_attribute = AttributeDict(
{"compute": envs_with_reduced_filepaths})
assert expected_attribute == observed_attribute
def test_minimal_configuration_name_inference(
self, tmpdir, minimal_project_conf_path, env_config_filepath):
""" Project infers name from where its configuration lives. """
project = Project(minimal_project_conf_path,
default_compute=env_config_filepath)
_, expected_name = os.path.split(tmpdir.strpath)
assert expected_name == project.name
def main():
args = parse_arguments()
# Start project object
prj = Project(args.project_config_file)
if "trackhubs" not in prj:
raise ValueError(
"Project configuration does not have a trackhub section.")
if "trackhub_dir" not in prj.trackhubs:
raise ValueError(
"Project trackhub configuration does not have a trackhub_dir attribute."
)
# Setup paths and hub files
bigwig_dir = os.path.join(prj.trackhubs.trackhub_dir)
track_hub = os.path.join(bigwig_dir, "hub.txt")
genomes_hub = os.path.join(bigwig_dir, "genomes.txt")
open(genomes_hub, 'w').write("")
# Setup attributes
proj_name = prj[
'project_name'] if "project_name" in prj else os.path.basename(
prj['paths']['output_dir'])
proj_desc = prj[
'project_description'] if "project_description" in prj else proj_name
user_email = prj['email'] if "email" in prj else ""
# In the future there will be more actions than this
make_ucsc_trackhub(args, prj, track_hub, bigwig_dir, genomes_hub,
proj_name, proj_desc, user_email)
track_file = os.path.join(bigwig_dir, "igv", "index.html")
track_url = os.path.join(prj['trackhubs']['url'], "igv")
make_igv_tracklink(prj, track_file, track_url)
def test_sample_name_availability(
self, path_project_conf, path_sample_anns, lazy):
""" Sample names always available on Project. """
with open(path_sample_anns, 'r') as anns_file:
expected_sample_names = \
[l.split(",")[0] for l in anns_file.readlines()[1:] if l]
p = Project(path_project_conf, defer_sample_construction=lazy)
assert expected_sample_names == list(p.sample_names)
def project(self, tmpdir, minimal_project_conf_path):
""" Create a Project with base/default environment. """
# Write base/default environment data to disk.
env_config_filename = "env-conf.yaml"
env_config_filepath = tmpdir.join(env_config_filename).strpath
with open(env_config_filepath, 'w') as env_conf:
yaml.safe_dump(self.ENVIRONMENT_CONFIG_DATA, env_conf)
return Project(minimal_project_conf_path,
default_compute=env_config_filepath)
def test_samples_are_generic(path_anns_file, path_proj_conf_file):
""" Regardless of protocol, Samples for sheet are generic. """
# Annotations filepath fixture is also writes that file, so
# it's needed even though that return value isn't used locally.
p = Project(path_proj_conf_file)
assert len(SAMPLE_NAMES) == p.num_samples
samples = list(p.samples)
assert p.num_samples == len(samples)
assert all([Sample is type(s) for s in samples])
def interactive(prj_lines=PROJECT_CONFIG_LINES,
iface_lines=PIPELINE_INTERFACE_CONFIG_LINES,
merge_table_lines=MERGE_TABLE_LINES,
annotation_lines=SAMPLE_ANNOTATION_LINES,
project_kwargs=None,
logger_kwargs=None):
"""
Create Project and PipelineInterface instances from default or given data.
This is intended to provide easy access to instances of fundamental looper
object for interactive test-authorship-motivated work in an iPython
interpreter or Notebook. Test authorship is simplified if we provide
easy access to viable instances of these objects.
:param Iterable[str] prj_lines: project config lines
:param Iterable[str] iface_lines: pipeline interface config lines
:param Iterable[str] merge_table_lines: lines for a merge table file
:param Iterable[str] annotation_lines: lines for a sample annotations file
:param str | int loglevel: level at which to attend to log messages
:param dict project_kwargs: keyword arguments for Project constructor
:param dict logger_kwargs: keyword arguments for logging configuration
:param bool devmode: whether logging should be done in development mode;
this implies a more verbose level setting and a more information-rich
template for logging message formatting
:param str logfile: path to file to which to write logging messages
:return Project, PipelineInterface: one Project and one PipelineInterface,
"""
# Establish logging for interactive session.
looper_logger_kwargs = {"level": "DEBUG"}
looper_logger_kwargs.update(logger_kwargs or {})
setup_looper_logger(**looper_logger_kwargs)
# TODO: don't work with tempfiles once ctors tolerate Iterable.
dirpath = tempfile.mkdtemp()
path_conf_file = _write_temp(prj_lines,
dirpath=dirpath,
fname=P_CONFIG_FILENAME)
path_iface_file = _write_temp(iface_lines,
dirpath=dirpath,
fname="pipeline_interface.yaml")
path_merge_table_file = _write_temp(merge_table_lines,
dirpath=dirpath,
fname=MERGE_TABLE_FILENAME)
path_sample_annotation_file = _write_temp(annotation_lines,
dirpath=dirpath,
fname=ANNOTATIONS_FILENAME)
prj = Project(path_conf_file, **(project_kwargs or {}))
iface = PipelineInterface(path_iface_file)
for path in [
path_conf_file, path_iface_file, path_merge_table_file,
path_sample_annotation_file
]:
os.unlink(path)
return prj, iface
def test_counting_samples_doesnt_create_samples(
self, sample_annotation_lines,
path_project_conf, path_sample_anns):
""" User can ask about sample count without creating samples. """
# We're not parameterized in terms of Sample creation laziness here
# because a piece of the test's essence is Sample collection absence.
p = Project(path_project_conf, defer_sample_construction=True)
assert p._samples is None
expected_sample_count = sum(1 for _ in sample_annotation_lines) - 1
assert expected_sample_count == p.num_samples
assert p._samples is None
def test_lacks_sample_annotations(
self, project_config_data, env_config_filepath, tmpdir):
""" Lack of sample annotations precludes Project construction. """
# Remove sample annotations KV pair from config data for this test.
del project_config_data["metadata"][SAMPLE_ANNOTATIONS_KEY]
# Write the config and assert the expected exception for Project ctor.
conf_path = _write_project_config(
project_config_data, dirpath=tmpdir.strpath)
with pytest.raises(_MissingMetadataException):
Project(conf_path, default_compute=env_config_filepath)
def test_multiple_samples(self, protocols, path_anns_file,
path_proj_conf_file):
""" Project also processes multiple Sample fine. """
p = Project(path_proj_conf_file)
# Total sample count is constant.
assert len(SAMPLE_NAMES) == sum(1 for _ in p.samples)
# But the sheet permits filtering to specific protocol(s).
exp_num_samples = len(SAMPLE_NAMES) if not protocols else \
sum(sum(1 for l in LIBRARIES if l == p) for p in protocols)
sheet = p.build_sheet(*protocols)
assert exp_num_samples == len(sheet)
if protocols:
fuzzy_protos = {alpha_cased(p) for p in protocols}
for _, sample_data in sheet.iterrows():
assert alpha_cased(sample_data.library) in fuzzy_protos
def observed_argstring_elements(
self, confdata, pipeline, confpath, envpath):
"""
Write config, build project, and validate argstring for pipeline.
:param dict confdata: project configuration data
:param str pipeline: name of pipeline for which to build argstring
:param str confpath: where to write project config file
:param str envpath: pointer to default environment file
:return Iterable[str] argstring components
"""
conf_file_path = _write_project_config(confdata, dirpath=confpath)
# Subvert requirement for sample annotations file.
with mock.patch("looper.models.check_sheet"):
project = Project(conf_file_path, default_compute=envpath)
argstring = project.get_arg_string(pipeline)
return argstring.split(" ")
def create_project(
self, project_config_data, default_env_path, case_type, dirpath):
"""
For a test case, determine expectations and create Project instance.
:param dict project_config_data: the actual data to write to the
Project configuration file
:param str default_env_path: path to the default environment config
file to pass to Project constructor
:param str case_type: type of test case to execute; this determines
how to specify the derived columns in the config file
:param str dirpath: path in which to write config file
:return (Iterable[str], Project): collection of names of derived
columns to expect, along with Project instance with which to test
"""
# Ensure valid parameterization.
if case_type not in self.DERIVED_COLUMNS_CASE_TYPES:
raise ValueError(
"Unexpected derived_columns case type: '{}' (known={})".
format(case_type, self.DERIVED_COLUMNS_CASE_TYPES))
# Parameterization specifies expectation and explicit specification.
expected_derived_columns = copy.copy(Project.DERIVED_COLUMNS_DEFAULT)
if case_type == "implicit":
# Negative control; ensure config data lacks derived columns.
assert "derived_columns" not in project_config_data
else:
explicit_derived_columns = \
copy.copy(self.ADDITIONAL_DERIVED_COLUMNS)
expected_derived_columns.extend(self.ADDITIONAL_DERIVED_COLUMNS)
# Determine explicit inclusion of default derived columns.
if case_type == "intersection":
explicit_derived_columns.extend(
Project.DERIVED_COLUMNS_DEFAULT)
project_config_data["derived_columns"] = explicit_derived_columns
# Write the config and build the Project.
conf_file_path = _write_project_config(
project_config_data, dirpath=dirpath)
with mock.patch("looper.models.check_sheet"):
project = Project(conf_file_path, default_compute=default_env_path)
return expected_derived_columns, project
def test_sample_creation_laziness(
self, path_project_conf, path_sample_anns, lazy):
""" Project offers control over whether to create base Sample(s). """
p = Project(path_project_conf, defer_sample_construction=lazy)
if lazy:
# Samples should remain null during lazy Project construction.
assert p._samples is None
else:
# Eager Project construction builds Sample objects.
assert p._samples is not None
with open(path_sample_anns, 'r') as anns_file:
anns_file_lines = anns_file.readlines()
# Sum excludes the header line.
num_samples_expected = sum(1 for l in anns_file_lines[1:] if l)
assert num_samples_expected == len(p._samples)
assert all([Sample == type(s) for s in p._samples])
def test_no_merge_table_in_config(
self, tmpdir, spec_type, lazy, proj_conf_data, path_sample_anns):
""" Merge table attribute remains null if config lacks merge_table. """
metadata = proj_conf_data["metadata"]
try:
assert "merge_table" in metadata
except AssertionError:
print("Project metadata section lacks 'merge_table'")
print("All config data: {}".format(proj_conf_data))
print("Config metadata section: {}".format(metadata))
raise
if spec_type == "as_null":
metadata["merge_table"] = None
elif spec_type == "missing":
del metadata["merge_table"]
else:
raise ValueError("Unknown way to specify no merge table: {}".
format(spec_type))
path_config_file = os.path.join(tmpdir.strpath, "project_config.yaml")
with open(path_config_file, 'w') as conf_file:
yaml.safe_dump(proj_conf_data, conf_file)
p = Project(path_config_file, defer_sample_construction=lazy)
assert p.merge_table is None
sep="\t", header=None, skiprows=1,
names=["gene_name", "chr", "strand", sample.name]).set_index("gene_name")[sample.name]
except IOError:
print("Sample {} is missing.".format(sample.name))
continue
# add gene index
if first:
expr = pd.DataFrame(counts)
first = False
else:
expr[sample.name] = counts
return expr
prj = Project(os.path.join("metadata", "config.yaml"))
prj.add_sample_sheet()
prj.paths.results_dir = os.path.join("results")
# get guide annotation
guide_annotation = pd.read_csv(os.path.join("metadata", "guide_annotation.csv"))
# Gather gRNA assignment info across all samples used
for experiment, rows in prj.sheet.df.groupby(['experiment']):
# merge gRNA data
reads = pd.DataFrame()
scores = pd.DataFrame()
assignment = pd.DataFrame()
for sample_name in rows["sample_name"]:
def main():
parser = ArgumentParser(prog="ngs_analysis_recipe",
description="NGS analysis recipe.")
parser = add_args(parser)
args = parser.parse_args()
# args = parser.parse_args('-t ATAC-seq metadata/project_config.yaml'.split(" "))
# Start project
print(
"Starting looper project with project configuration file: '{}'".format(
args.config_file))
prj = Project(args.config_file)
print("Changing directory to project root directory: '{}'.".format(
prj.metadata.output_dir))
os.chdir(prj.metadata.output_dir)
if args.pass_qc:
print(
"Filtering samples out which didn't pass QC as specified in sample annotation in column 'pass_qc'"
)
prj._samples = [
s for s in prj._samples
if s.pass_qc not in ['0', 0, 'False', False]
]
print("Setting location of sample files dependent on sample types.")
for sample in prj.samples:
if hasattr(sample, "protocol"):
sample.library = sample.protocol
if sample.library in ["ATAC-seq", "ChIP-seq", "ChIPmentation"]:
sample.mapped = os.path.join(sample.paths.sample_root, "mapped",
sample.name + ".trimmed.bowtie2.bam")
sample.filtered = os.path.join(
sample.paths.sample_root, "mapped",
sample.name + ".trimmed.bowtie2.filtered.bam")
sample.peaks = os.path.join(sample.paths.sample_root, "peaks",
sample.name + "_peaks.narrowPeak")
elif sample.library == "RNA-seq":
sample.bitseq_counts = os.path.join(
sample.paths.sample_root,
"bowtie1_{}".format(sample.transcriptome), "bitSeq",
sample.name + ".counts")
# ANALYSIS
if args.data_type is None:
print(
"Type of analysis not specified. Will run independent analysis for all types of data in the sample annotation sheet."
)
data_types = sorted(list(set([s.library for s in prj.samples])))
print("Sample data types: '{}'.".format(",".join(data_types)))
else:
print(
"Type of analysis specified. Will run only analysis for samples of type '{}'."
.format(args.data_type))
data_types = [args.data_type]
print("Sample data types: '{}'.".format(",".join(data_types)))
if args.name is None:
print(
"Analysis name not specified, will use name in project configuration file: '{}'."
.format(prj.project_name))
args.name = prj.project_name
for data_type in data_types:
print("Starting analysis for samples of type: '{}'.".format(data_type))
samples = [s for s in prj.samples if (s.library == data_type)]
if len(samples) > 0:
print("Samples under consideration: '{}'. ".format(",".join(
[s.name for s in samples])) +
"Total of {} samples.".format(len([s.name
for s in samples])))
else:
raise ValueError(
"There were no valid samples for this analysis type!")
if data_type in ["ATAC-seq"]:
print("Initializing ATAC-seq analysis")
analysis = ATACSeqAnalysis(name=args.name + "_atacseq",
prj=prj,
samples=samples,
results_dir=args.results_dir)
elif data_type in ["ChIP-seq"]:
print("Initializing ChIP-seq analysis")
analysis = ChIPSeqAnalysis(name=args.name + "_chipseq",
prj=prj,
samples=samples,
results_dir=args.results_dir)
elif data_type in ["RNA-seq"]:
print("Initializing RNA-seq analysis")
analysis = RNASeqAnalysis(name=args.name + "_rnaseq",
prj=prj,
samples=samples,
results_dir=args.results_dir)
if hasattr(prj, "sample_attributes"):
print(
"Using sample attributes from project configuration file: '{}'."
.format(",".join(prj.sample_attributes)))
sample_attributes = prj.sample_attributes
else:
print(
"Project configuration file does not contain a 'sample_attributes' section."
)
print("Sample annotation will be minimal!")
sample_attributes = ['sample_name']
if hasattr(prj, "group_attributes"):
print(
"Using group attributes from project configuration file: '{}'."
.format(",".join(prj.group_attributes)))
group_attributes = prj.group_attributes
else:
print(
"Project configuration file does not contain a 'group_attributes' section."
)
print("Group-wise labeling of samples will not be possible!")
group_attributes = ['sample_name']
return main_analysis_pipeline(analysis,
data_type=data_type,
sample_attributes=sample_attributes,
plotting_attributes=group_attributes,
alpha=args.alpha,
abs_fold_change=args.abs_fold_change)
def test_minimal_configuration_doesnt_fail(
self, minimal_project_conf_path, env_config_filepath):
""" Project ctor requires minimal config and default environment. """
Project(config_file=minimal_project_conf_path,
default_compute=env_config_filepath)
def test_minimal_configuration_output_dir(
self, tmpdir, minimal_project_conf_path, env_config_filepath):
""" Project infers output path from its configuration location. """
project = Project(minimal_project_conf_path,
default_compute=env_config_filepath)
assert tmpdir.strpath == project.output_dir
def test_no_samples(self, path_empty_project):
""" Lack of Samples is unproblematic. """
p = Project(path_empty_project)
assert 0 == p.num_samples
assert [] == list(p.samples)
interactions_TF['interaction_score'] >= 1]
interactions_TF_filtered.to_csv(os.path.join(
foots_dir, label + ".piq.TF-TF_interactions.filtered.tsv"),
sep="\t",
index=False)
# INIT
# Get path configuration
data_dir = os.path.join('.', "data")
scratch_dir = os.path.join("/scratch/users/arendeiro/piq")
results_dir = os.path.join('.', "results")
plots_dir = os.path.join(results_dir, "plots")
# Start project
prj = Project("metadata/project_config.yaml")
prj.add_sample_sheet()
# annotated samples with a few more things:
prj.samples = annotate_samples(prj.samples, prj.sheet.df.columns.tolist())
samples = prj.samples
# MOTIFS FOR PIQ
motifs_file = "data/external/jaspar_human_motifs.txt"
n_motifs = 366
# prepare motifs for footprinting (done once)
cmds = piq_prepare_motifs(motifs_file, n_motifs)
for cmd in cmds:
cmd2 = tk.slurmHeader("PIQ_preparemotifs",
os.path.join("/home/arendeiro/",
def test_no_samples(self, protocols, delimiter, path_empty_project):
""" Lack of Samples is unproblematic for the sheet build. """
# Regardless of protocol(s), the sheet should be empty.
p = Project(path_empty_project)
sheet = p.build_sheet(*protocols)
assert sheet.empty
def test_empty_project(self, path_empty_project):
""" It's unproblematic to create a Project that lacks samples. """
Project(path_empty_project)