def test_background_pairs(self):
proc = Process.objects.create(
type="data:test:process:",
slug="test-process",
contributor=self.contributor,
entity_type="sample",
entity_descriptor_schema="sample",
)
proc2 = Process.objects.create(
type="data:test:process2:",
slug="test-process2",
contributor=self.contributor,
entity_type="sample",
entity_descriptor_schema="sample",
input_schema=[{
"name": "src",
"type": "data:test:process:"
}],
)
DescriptorSchema.objects.create(slug="sample",
version="1.0.0",
contributor=self.contributor)
collection = Collection.objects.create(name="Test collection",
contributor=self.contributor)
data = []
data2 = []
for i in range(6):
data.append(
Data.objects.create(
name="Data {}".format(i),
contributor=self.contributor,
process=proc,
status=Data.STATUS_DONE,
))
data2.append(
Data.objects.create(
name="Data2 {}".format(i),
contributor=self.contributor,
process=proc2,
status=Data.STATUS_DONE,
input={"src": data[-1].id},
))
rel_type_background = RelationType.objects.get(name="background")
background = Relation.objects.create(
contributor=self.contributor,
collection=collection,
type=rel_type_background,
category="Background",
)
RelationPartition.objects.create(relation=background,
entity=data2[0].entity,
label="background")
RelationPartition.objects.create(relation=background,
entity=data2[1].entity,
label="case")
RelationPartition.objects.create(relation=background,
entity=data2[3].entity,
label="case")
RelationPartition.objects.create(relation=background,
entity=data2[4].entity,
label="case")
def load_templates(template_name):
if template_name == "background_pairs":
return "{{ some_data|background_pairs }}"
env = Environment(loader=FunctionLoader(load_templates))
env.filters["background_pairs"] = background_pairs
background_pairs_template = env.get_template("background_pairs")
self.assertEqual(
background_pairs_template.render(
some_data=[{
"__id": d.id,
"__type": d.process.type
} for d in data]),
"[({1}, {0}), ({2}, None), ({3}, {0}), ({4}, {0}), ({5}, None)]".
format(data[0].id, data[1].id, data[2].id, data[3].id, data[4].id,
data[5].id),
)
self.assertEqual(
background_pairs_template.render(
some_data=[{
"__id": d.id,
"__type": d.process.type
} for d in data2]),
"[({1}, {0}), ({2}, None), ({3}, {0}), ({4}, {0}), ({5}, None)]".
format(
data2[0].id,
data2[1].id,
data2[2].id,
data2[3].id,
data2[4].id,
data2[5].id,
),
)
# Test list must be given
with self.assertRaises(ValueError):
background_pairs(42)
def test_background_pairs(self):
proc = Process.objects.create(type='data:test:process:',
slug='test-process',
contributor=self.contributor,
entity_type='sample',
entity_descriptor_schema='sample')
proc2 = Process.objects.create(type='data:test:process2:',
slug='test-process2',
contributor=self.contributor,
entity_type='sample',
entity_descriptor_schema='sample',
input_schema=[{
'name':
'src',
'type':
'data:test:process:'
}])
DescriptorSchema.objects.create(slug='sample',
version='1.0.0',
contributor=self.contributor)
collection = Collection.objects.create(name='Test collection',
contributor=self.contributor)
data = []
data2 = []
for i in range(6):
data.append(
Data.objects.create(
name='Data {}'.format(i),
contributor=self.contributor,
process=proc,
status=Data.STATUS_DONE,
))
data2.append(
Data.objects.create(name='Data2 {}'.format(i),
contributor=self.contributor,
process=proc2,
status=Data.STATUS_DONE,
input={'src': data[-1].id}))
rel_type_background = RelationType.objects.get(name='background')
background = Relation.objects.create(contributor=self.contributor,
collection=collection,
type=rel_type_background,
category='Background')
RelationPartition.objects.create(relation=background,
entity=data2[0].entity_set.first(),
label='background')
RelationPartition.objects.create(relation=background,
entity=data2[1].entity_set.first(),
label='case')
RelationPartition.objects.create(relation=background,
entity=data2[3].entity_set.first(),
label='case')
RelationPartition.objects.create(relation=background,
entity=data2[4].entity_set.first(),
label='case')
def load_templates(template_name):
if template_name == 'background_pairs':
return '{{ some_data|background_pairs }}'
env = Environment(loader=FunctionLoader(load_templates))
env.filters['background_pairs'] = background_pairs
background_pairs_template = env.get_template('background_pairs')
self.assertEqual(
background_pairs_template.render(
some_data=[{
'__id': d.id,
'__type': d.process.type
} for d in data]),
'[({1}, {0}), ({2}, None), ({3}, {0}), ({4}, {0}), ({5}, None)]'.
format(data[0].id, data[1].id, data[2].id, data[3].id, data[4].id,
data[5].id))
self.assertEqual(
background_pairs_template.render(
some_data=[{
'__id': d.id,
'__type': d.process.type
} for d in data2]),
'[({1}, {0}), ({2}, None), ({3}, {0}), ({4}, {0}), ({5}, None)]'.
format(data2[0].id, data2[1].id, data2[2].id, data2[3].id,
data2[4].id, data2[5].id))
# Test list must be given
with self.assertRaises(ValueError):
background_pairs(42)
def test_macs2_batch(self):
with self.preparation_stage():
inputs = {
"src": "macs2/input/SRR5675973_chr17.bam",
"species": "H**o sapiens",
"build": "hg19",
}
case_1 = self.run_process("upload-bam", inputs)
inputs = {
"src": "macs2/input/SRR5675974_chr17.bam",
"species": "H**o sapiens",
"build": "hg19",
}
background_1 = self.run_process("upload-bam", inputs)
inputs = {
"src": "macs2/input/SRR5675975_chr17.bam",
"species": "H**o sapiens",
"build": "hg19",
}
case_2 = self.run_process("upload-bam", inputs)
inputs = {
"src": "macs2/input/SRR5675976_chr17.bam",
"species": "H**o sapiens",
"build": "hg19",
}
background_2 = self.run_process("upload-bam", inputs)
inputs = {
"src": "macs2/input/SRR5675973_chr17.bam",
"species": "H**o sapiens",
"build": "hg19",
}
case_wo_background = self.run_process("upload-bam", inputs)
promoters = self.run_process(
"upload-bed",
{
"src": "macs2/input/promoter_regions.bed",
"species": "H**o sapiens",
"build": "hg19",
},
)
collection = Collection.objects.create(
name="Test collection", contributor=self.contributor)
rel_type_background = RelationType.objects.get(name="background")
background1 = Relation.objects.create(
contributor=self.contributor,
collection=collection,
type=rel_type_background,
category="Background",
)
background2 = Relation.objects.create(
contributor=self.contributor,
collection=collection,
type=rel_type_background,
category="Background2",
)
RelationPartition.objects.create(relation=background1,
entity=case_1.entity,
label="case")
RelationPartition.objects.create(relation=background1,
entity=background_1.entity,
label="background")
RelationPartition.objects.create(relation=background2,
entity=case_2.entity,
label="case")
RelationPartition.objects.create(relation=background2,
entity=background_2.entity,
label="background")
self.assertEqual(
background_pairs([
{
"__id": case_1.id,
"__type": case_1.process.type
},
{
"__id": background_1.id,
"__type": background_1.process.type
},
{
"__id": case_2.id,
"__type": case_2.process.type
},
{
"__id": background_2.id,
"__type": background_2.process.type
},
{
"__id": case_wo_background.id,
"__type": case_wo_background.process.type,
},
]),
[
(case_1.id, background_1.id),
(case_2.id, background_2.id),
(case_wo_background.id, None),
],
)
self.run_process(
"macs2-batch",
{
"alignments": [
case_1.id,
background_1.id,
case_2.id,
background_2.id,
case_wo_background.id,
],
"promoter":
promoters.id,
"tagalign":
True,
},
)
for data in Data.objects.all():
self.assertStatus(data, Data.STATUS_DONE)
macs2 = Data.objects.filter(process__slug="macs2-callpeak").last()
self.assertFields(macs2, "species", "H**o sapiens")
self.assertFields(macs2, "build", "hg19")
self.assertFile(macs2, "case_prepeak_qc",
"macs2/output/batch_case_prepeak_qc.txt")
self.assertFile(macs2, "chip_qc",
"macs2/output/batch_case_postpeak_qc_report.txt")
self.assertFileExists(macs2, "called_peaks")
self.assertFileExists(macs2, "narrow_peaks")
self.assertFileExists(macs2, "narrow_peaks_bigbed_igv_ucsc")
self.assertFileExists(macs2, "summits")
self.assertFileExists(macs2, "summits_tbi_jbrowse")
self.assertFileExists(macs2, "summits_bigbed_igv_ucsc")
self.assertFileExists(macs2, "treat_pileup")
self.assertFileExists(macs2, "treat_pileup_bigwig")
self.assertFileExists(macs2, "control_lambda")
self.assertFileExists(macs2, "control_lambda_bigwig")
def test_macs2_rose2_batch(self):
with self.preparation_stage():
inputs = {
"src": "macs2/input/SRR5675973_chr17.bam",
"species": "H**o sapiens",
"build": "hg19",
}
case_1 = self.run_process("upload-bam", inputs)
inputs = {
"src": "macs2/input/SRR5675974_chr17.bam",
"species": "H**o sapiens",
"build": "hg19",
}
background_1 = self.run_process("upload-bam", inputs)
promoters = self.run_process(
"upload-bed",
{
"src": "macs2/input/promoter_regions.bed",
"species": "H**o sapiens",
"build": "hg19",
},
)
collection = Collection.objects.create(
name="Test collection", contributor=self.contributor)
rel_type_background = RelationType.objects.get(name="background")
background = Relation.objects.create(
contributor=self.contributor,
collection=collection,
type=rel_type_background,
category="Background",
)
RelationPartition.objects.create(relation=background,
entity=case_1.entity,
label="case")
RelationPartition.objects.create(relation=background,
entity=background_1.entity,
label="background")
self.assertEqual(
background_pairs([
{
"__id": case_1.id,
"__type": case_1.process.type
},
{
"__id": background_1.id,
"__type": background_1.process.type
},
]),
[(case_1.id, background_1.id)],
)
self.run_process(
"macs2-rose2-batch",
{
"alignments": [case_1.id, background_1.id],
"promoter": promoters.id
},
)
for data in Data.objects.all():
self.assertStatus(data, Data.STATUS_DONE)
rose2 = Data.objects.filter(process__slug="rose2").last()
# remove changing lines from the rose2 output
def filter_created(line):
return line.startswith(b"#Created")
self.assertFile(
rose2,
"all_enhancers",
"macs2/output/rose2_enhancer_table.txt",
file_filter=filter_created,
)
def test_macs2_rose2_batch(self):
with self.preparation_stage():
inputs = {
'src': 'macs2/input/SRR5675973_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
case_1 = self.run_process("upload-bam", inputs)
inputs = {
'src': 'macs2/input/SRR5675974_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
background_1 = self.run_process('upload-bam', inputs)
promoters = self.run_process(
'upload-bed', {
'src': 'macs2/input/promoter_regions.bed',
'species': 'H**o sapiens',
'build': 'hg19',
})
collection = Collection.objects.create(
name='Test collection', contributor=self.contributor)
rel_type_background = RelationType.objects.get(name='background')
background = Relation.objects.create(contributor=self.contributor,
collection=collection,
type=rel_type_background,
category='Background')
RelationPartition.objects.create(relation=background,
entity=case_1.entity,
label='case')
RelationPartition.objects.create(relation=background,
entity=background_1.entity,
label='background')
self.assertEqual(
background_pairs([
{
'__id': case_1.id,
'__type': case_1.process.type
},
{
'__id': background_1.id,
'__type': background_1.process.type
},
]), [(case_1.id, background_1.id)])
self.run_process(
'macs2-rose2-batch', {
'alignments': [case_1.id, background_1.id],
'promoter': promoters.id,
'tagalign': True,
})
for data in Data.objects.all():
self.assertStatus(data, Data.STATUS_DONE)
rose2 = Data.objects.last()
# remove changing lines from the rose2 output
def filter_created(line):
return line.startswith(b'#Created')
self.assertFile(rose2,
'all_enhancers',
'macs2/output/rose2_enhancer_table.txt',
file_filter=filter_created)
def test_macs2_batch(self):
with self.preparation_stage():
inputs = {
'src': 'macs2/input/SRR5675973_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
case_1 = self.run_process("upload-bam", inputs)
inputs = {
'src': 'macs2/input/SRR5675974_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
background_1 = self.run_process('upload-bam', inputs)
inputs = {
'src': 'macs2/input/SRR5675975_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
case_2 = self.run_process('upload-bam', inputs)
inputs = {
'src': 'macs2/input/SRR5675976_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
background_2 = self.run_process('upload-bam', inputs)
inputs = {
'src': 'macs2/input/SRR5675973_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
case_wo_background = self.run_process('upload-bam', inputs)
promoters = self.run_process(
'upload-bed', {
'src': 'macs2/input/promoter_regions.bed',
'species': 'H**o sapiens',
'build': 'hg19',
})
collection = Collection.objects.create(
name='Test collection', contributor=self.contributor)
rel_type_background = RelationType.objects.get(name='background')
background1 = Relation.objects.create(contributor=self.contributor,
collection=collection,
type=rel_type_background,
category='Background')
background2 = Relation.objects.create(contributor=self.contributor,
collection=collection,
type=rel_type_background,
category='Background2')
RelationPartition.objects.create(relation=background1,
entity=case_1.entity,
label='case')
RelationPartition.objects.create(relation=background1,
entity=background_1.entity,
label='background')
RelationPartition.objects.create(relation=background2,
entity=case_2.entity,
label='case')
RelationPartition.objects.create(relation=background2,
entity=background_2.entity,
label='background')
self.assertEqual(
background_pairs([
{
'__id': case_1.id,
'__type': case_1.process.type
},
{
'__id': background_1.id,
'__type': background_1.process.type
},
{
'__id': case_2.id,
'__type': case_2.process.type
},
{
'__id': background_2.id,
'__type': background_2.process.type
},
{
'__id': case_wo_background.id,
'__type': case_wo_background.process.type
},
]),
[(case_1.id, background_1.id), (case_2.id, background_2.id),
(case_wo_background.id, None)])
self.run_process(
'macs2-batch', {
'alignments': [
case_1.id, background_1.id, case_2.id, background_2.id,
case_wo_background.id
],
'promoter':
promoters.id,
'tagalign':
True,
})
for data in Data.objects.all():
self.assertStatus(data, Data.STATUS_DONE)
macs2 = Data.objects.last()
self.assertFields(macs2, 'species', 'H**o sapiens')
self.assertFields(macs2, 'build', 'hg19')
self.assertFile(macs2, 'case_prepeak_qc',
'macs2/output/batch_case_prepeak_qc.txt')
self.assertFile(macs2, 'chip_qc',
'macs2/output/batch_case_postpeak_qc_report.txt')
self.assertFileExists(macs2, 'called_peaks')
self.assertFileExists(macs2, 'narrow_peaks')
self.assertFileExists(macs2, 'narrow_peaks_bigbed_igv_ucsc')
self.assertFileExists(macs2, 'summits')
self.assertFileExists(macs2, 'summits_tbi_jbrowse')
self.assertFileExists(macs2, 'summits_bigbed_igv_ucsc')
self.assertFileExists(macs2, 'treat_pileup')
self.assertFileExists(macs2, 'treat_pileup_bigwig')
self.assertFileExists(macs2, 'control_lambda')
self.assertFileExists(macs2, 'control_lambda_bigwig')
def test_macs2_rose2_batch(self):
with self.preparation_stage():
inputs = {
'src': 'macs2/input/SRR5675973_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
case_1 = self.run_process("upload-bam", inputs)
inputs = {
'src': 'macs2/input/SRR5675974_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
background_1 = self.run_process('upload-bam', inputs)
promoters = self.run_process('upload-bed', {
'src': 'macs2/input/promoter_regions.bed',
'species': 'H**o sapiens',
'build': 'hg19',
})
collection = Collection.objects.create(
name='Test collection',
contributor=self.contributor
)
rel_type_background = RelationType.objects.get(name='background')
background = Relation.objects.create(
contributor=self.contributor,
collection=collection,
type=rel_type_background,
category='Background'
)
RelationPartition.objects.create(relation=background, entity=case_1.entity_set.first(), label='case')
RelationPartition.objects.create(
relation=background,
entity=background_1.entity_set.first(),
label='background'
)
self.assertEqual(
background_pairs([
{'__id': case_1.id, '__type': case_1.process.type},
{'__id': background_1.id, '__type': background_1.process.type},
]),
[(case_1.id, background_1.id)]
)
self.run_process(
'macs2-rose2-batch', {
'alignments': [case_1.id, background_1.id],
'promoter': promoters.id,
'tagalign': True,
}
)
for data in Data.objects.all():
self.assertStatus(data, Data.STATUS_DONE)
rose2 = Data.objects.last()
# remove changing lines from the rose2 output
def filter_created(line):
return line.startswith(b'#Created')
self.assertFile(
rose2,
'all_enhancers',
'macs2/output/rose2_enhancer_table.txt',
file_filter=filter_created
)
def test_macs2_batch(self):
with self.preparation_stage():
inputs = {
'src': 'macs2/input/SRR5675973_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
case_1 = self.run_process("upload-bam", inputs)
inputs = {
'src': 'macs2/input/SRR5675974_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
background_1 = self.run_process('upload-bam', inputs)
inputs = {
'src': 'macs2/input/SRR5675975_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
case_2 = self.run_process('upload-bam', inputs)
inputs = {
'src': 'macs2/input/SRR5675976_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
background_2 = self.run_process('upload-bam', inputs)
inputs = {
'src': 'macs2/input/SRR5675973_chr17.bam',
'species': 'H**o sapiens',
'build': 'hg19',
}
case_wo_background = self.run_process('upload-bam', inputs)
promoters = self.run_process('upload-bed', {
'src': 'macs2/input/promoter_regions.bed',
'species': 'H**o sapiens',
'build': 'hg19',
})
collection = Collection.objects.create(
name='Test collection',
contributor=self.contributor
)
rel_type_background = RelationType.objects.get(name='background')
background1 = Relation.objects.create(
contributor=self.contributor,
collection=collection,
type=rel_type_background,
category='Background'
)
background2 = Relation.objects.create(
contributor=self.contributor,
collection=collection,
type=rel_type_background,
category='Background2'
)
RelationPartition.objects.create(relation=background1, entity=case_1.entity_set.first(), label='case')
RelationPartition.objects.create(
relation=background1,
entity=background_1.entity_set.first(),
label='background'
)
RelationPartition.objects.create(relation=background2, entity=case_2.entity_set.first(), label='case')
RelationPartition.objects.create(
relation=background2,
entity=background_2.entity_set.first(),
label='background'
)
self.assertEqual(
background_pairs([
{'__id': case_1.id, '__type': case_1.process.type},
{'__id': background_1.id, '__type': background_1.process.type},
{'__id': case_2.id, '__type': case_2.process.type},
{'__id': background_2.id, '__type': background_2.process.type},
{'__id': case_wo_background.id, '__type': case_wo_background.process.type},
]),
[(case_1.id, background_1.id), (case_2.id, background_2.id), (case_wo_background.id, None)]
)
self.run_process(
'macs2-batch', {
'alignments': [case_1.id, background_1.id, case_2.id, background_2.id, case_wo_background.id],
'promoter': promoters.id,
'tagalign': True,
}
)
for data in Data.objects.all():
self.assertStatus(data, Data.STATUS_DONE)
macs2 = Data.objects.last()
self.assertFields(macs2, 'species', 'H**o sapiens')
self.assertFields(macs2, 'build', 'hg19')
self.assertFile(macs2, 'case_prepeak_qc', 'macs2/output/batch_case_prepeak_qc.txt')
self.assertFile(macs2, 'chip_qc', 'macs2/output/batch_case_postpeak_qc_report.txt')
self.assertFileExists(macs2, 'called_peaks')
self.assertFileExists(macs2, 'narrow_peaks')
self.assertFileExists(macs2, 'narrow_peaks_bigbed_igv_ucsc')
self.assertFileExists(macs2, 'summits')
self.assertFileExists(macs2, 'summits_tbi_jbrowse')
self.assertFileExists(macs2, 'summits_bigbed_igv_ucsc')
self.assertFileExists(macs2, 'treat_pileup')
self.assertFileExists(macs2, 'treat_pileup_bigwig')
self.assertFileExists(macs2, 'control_lambda')
self.assertFileExists(macs2, 'control_lambda_bigwig')
def test_background_pairs(self):
proc = Process.objects.create(
type='data:test:process:',
slug='test-process',
contributor=self.contributor,
entity_type='sample',
entity_descriptor_schema='sample'
)
proc2 = Process.objects.create(
type='data:test:process2:',
slug='test-process2',
contributor=self.contributor,
entity_type='sample',
entity_descriptor_schema='sample',
input_schema=[{'name': 'src', 'type': 'data:test:process:'}]
)
DescriptorSchema.objects.create(
slug='sample',
version='1.0.0',
contributor=self.contributor
)
collection = Collection.objects.create(
name='Test collection',
contributor=self.contributor
)
data = []
data2 = []
for i in range(6):
data.append(Data.objects.create(
name='Data {}'.format(i),
contributor=self.contributor,
process=proc,
status=Data.STATUS_DONE,
))
data2.append(Data.objects.create(
name='Data2 {}'.format(i),
contributor=self.contributor,
process=proc2,
status=Data.STATUS_DONE,
input={'src': data[-1].id}
))
rel_type_background = RelationType.objects.get(name='background')
background = Relation.objects.create(
contributor=self.contributor,
collection=collection,
type=rel_type_background,
category='Background'
)
RelationPartition.objects.create(relation=background, entity=data2[0].entity_set.first(), label='background')
RelationPartition.objects.create(relation=background, entity=data2[1].entity_set.first(), label='case')
RelationPartition.objects.create(relation=background, entity=data2[3].entity_set.first(), label='case')
RelationPartition.objects.create(relation=background, entity=data2[4].entity_set.first(), label='case')
def load_templates(template_name):
if template_name == 'background_pairs':
return '{{ some_data|background_pairs }}'
env = Environment(loader=FunctionLoader(load_templates))
env.filters['background_pairs'] = background_pairs
background_pairs_template = env.get_template('background_pairs')
self.assertEqual(
background_pairs_template.render(some_data=[{'__id': d.id, '__type': d.process.type} for d in data]),
'[({1}, {0}), ({2}, None), ({3}, {0}), ({4}, {0}), ({5}, None)]'.format(
data[0].id,
data[1].id,
data[2].id,
data[3].id,
data[4].id,
data[5].id
)
)
self.assertEqual(
background_pairs_template.render(some_data=[{'__id': d.id, '__type': d.process.type} for d in data2]),
'[({1}, {0}), ({2}, None), ({3}, {0}), ({4}, {0}), ({5}, None)]'.format(
data2[0].id,
data2[1].id,
data2[2].id,
data2[3].id,
data2[4].id,
data2[5].id
)
)
# Test list must be given
with self.assertRaises(ValueError):
background_pairs(42)
