def setUp(self):
self.user = User.objects.create_user(username="******",
password="******",
email='*****@*****.**')
self.storage = Storage(name="test", type=StorageType.LOCAL)
self.storage.save()
self.file_group = FileGroup(name="Test Files", storage=self.storage)
self.file_group.save()
self.file_type_fasta = FileType(name='fasta')
self.file_type_fasta.save()
self.file_type_bam = FileType(name='bam')
self.file_type_bam.save()
self.file_type_fastq = FileType(name='fastq')
self.file_type_fastq.save()
def setUp(self):
self.storage = Storage(name="test", type=StorageType.LOCAL)
self.storage.save()
self.file_group = FileGroup(name="Test Files", storage=self.storage)
self.file_group.save()
self.pipeline = Pipeline(
name="pipeline_name",
github="http://pipeline.github.com",
version="v1.0",
entrypoint="pipeline.cwl",
output_file_group=self.file_group,
output_directory="/path/to/outputs",
)
self.pipeline.save()
self.run = Run(run_type=0, app=self.pipeline, status=RunStatus.CREATING, notify_for_outputs=[])
self.run.save()
self.file_type_unknown = FileType(name="unknown")
self.file_type_unknown.save()
self.file1 = File(
**{
"file_name": "FASTQ_L002_R1_001.fastq.gz",
"path": "/FASTQ/FASTQ_L002_R1_001.fastq.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
}
)
self.file1.save()
self.file2 = File(
**{
"file_name": "FASTQ_L002_R2_001.fastq.gz",
"path": "/FASTQ/FASTQ_L002_R2_001.fastq.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
}
)
self.file2.save()
self.file3 = File(
**{
"file_name": "FASTQ_L006_R1_001.fastq.gz",
"path": "/FASTQ/FASTQ_L006_R1_001.fastq.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
}
)
self.file3.save()
self.file4 = File(
**{
"file_name": "FASTQ_L006_R2_001.fastq.gz",
"path": "/FASTQ/FASTQ_L006_R2_001.fastq.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
}
)
self.file4.save()
file_list = [
{
"file_name": "GRCm38.fasta",
"path": "/resources/genomes/GRCm38/GRCm38.fasta",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "b37.fasta",
"path": "/resources/genomes/GRCh37/fasta/b37.fasta",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "hapmap_3.3.b37.vcf",
"path": "/resources/request_files/hapmap/hapmap_3.3.b37.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
"path": "/resources/genomes/GRCh37/facets_snps/dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "human.hg19.excl.tsv",
"path": "/resources/genomes/GRCh37/delly/human.hg19.excl.tsv",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "hotspot-list-union-v1-v2.maf",
"path": "/resources/qc_resources/hotspot-list-union-v1-v2.maf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "AgilentExon_v2_FP_tiling_intervals.intervals",
"path": "/resources/genomic_resources/targets/AgilentExon_v2/b37/AgilentExon_v2_FP_tiling_intervals.intervals",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "AgilentExon_v2_FP_tiling_genotypes.txt",
"path": "/resources/genomic_resources/targets/AgilentExon_v2/b37/AgilentExon_v2_FP_tiling_genotypes.txt",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "AgilentExon_v2_b37_baits.ilist",
"path": "/resources/genomic_resources/targets/AgilentExon_v2/b37/AgilentExon_v2_b37_baits.ilist",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "AgilentExon_v2_b37_targets.ilist",
"path": "/resources/genomic_resources/targets/AgilentExon_v2/b37/AgilentExon_v2_b37_targets.ilist",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "refGene_b37.sorted.txt",
"path": "/resources/request_files/refseq/refGene_b37.sorted.txt",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "ExAC_nonTCGA.r0.3.1.sites.vep.vcf.gz",
"path": "/resources/vep/cache/ExAC_nonTCGA.r0.3.1.sites.vep.vcf.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "CosmicCodingMuts_v67_b37_20131024__NDS.vcf",
"path": "/resources/request_files/cosmic/CosmicCodingMuts_v67_b37_20131024__NDS.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "s_C_006537_N001_d.Group0.rg.md.abra.printreads.bam",
"path": "/resources/curated_bams/IDT_Exome_v1_FP_b37/s_C_006537_N001_d.Group0.rg.md.abra.printreads.bam",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "s_C_006284_N002_d.Group3.rg.md.abra.printreads.bam",
"path": "/resources/curated_bams/IDT_Exome_v1_FP_b37/s_C_006284_N002_d.Group3.rg.md.abra.printreads.bam",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "Mills_and_1000G_gold_standard.indels.b37.vcf",
"path": "/resources/request_files/indels_1000g/Mills_and_1000G_gold_standard.indels.b37.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "dbsnp_138.b37.excluding_sites_after_129.vcf",
"path": "/resources/request_files/dbsnp/dbsnp_138.b37.excluding_sites_after_129.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "1000G_phase1.snps.high_confidence.b37.vcf",
"path": "/resources/request_files/snps_1000g/1000G_phase1.snps.high_confidence.b37.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
]
for f in file_list:
self.file = File(**f)
self.file.save()
self.outputs = {
"maf": {
"size": 68273211,
"class": "File",
"nameext": ".maf",
"basename": "test_1.muts.maf",
"checksum": "sha1$0ccf4718a717f5a991607561af0b5bf989caf1e4",
"location": "file:///outputs/test_1.muts.maf",
"nameroot": "test_1.muts.maf",
},
"bams": [
{
"size": 56524168530,
"class": "File",
"nameext": ".bam",
"basename": "test_1.printreads.bam",
"checksum": "sha1$e4c05e8b3e7c1d682640e690f71536e22cb63802",
"location": "file:///output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam",
"nameroot": "test_1.printreads.bam",
"secondaryFiles": [
{
"size": 7214992,
"class": "File",
"nameext": ".bai",
"basename": "test_1.rg.md.abra.printreads.bai",
"checksum": "sha1$79299d55657a0226206a4bf3ddaba854ae11f9f1",
"location": "file:///output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai",
"nameroot": "test_1.rg.md.abra.printreads.bai",
}
],
},
{
"size": 6163808009,
"class": "File",
"nameext": ".bam",
"basename": "test_2.rg.md.abra.printreads.bam",
"checksum": "sha1$15ddc908c3ece551d331e78806d3ac19569174c3",
"location": "file:///output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam",
"nameroot": "test_2.rg.md.abra.printreads",
"secondaryFiles": [
{
"size": 6235920,
"class": "File",
"nameext": ".bai",
"basename": "test_2.rg.md.abra.printreads.bai",
"checksum": "sha1$6fd2fc0ce6b42253cbac89d31e8eddb169d65605",
"location": "file:///output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai",
"nameroot": "test_2.rg.md.abra.printreads",
}
],
},
],
}
class CWLRunObjectTest(APITestCase):
fixtures = [
"beagle_etl.operator.json",
"runner.operator_run.json",
"runner.operator_trigger.json",
]
def setUp(self):
self.storage = Storage(name="test", type=StorageType.LOCAL)
self.storage.save()
self.file_group = FileGroup(name="Test Files", storage=self.storage)
self.file_group.save()
self.pipeline = Pipeline(
name="pipeline_name",
github="http://pipeline.github.com",
version="v1.0",
entrypoint="pipeline.cwl",
output_file_group=self.file_group,
output_directory="/path/to/outputs",
)
self.pipeline.save()
self.run = Run(run_type=0, app=self.pipeline, status=RunStatus.CREATING, notify_for_outputs=[])
self.run.save()
self.file_type_unknown = FileType(name="unknown")
self.file_type_unknown.save()
self.file1 = File(
**{
"file_name": "FASTQ_L002_R1_001.fastq.gz",
"path": "/FASTQ/FASTQ_L002_R1_001.fastq.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
}
)
self.file1.save()
self.file2 = File(
**{
"file_name": "FASTQ_L002_R2_001.fastq.gz",
"path": "/FASTQ/FASTQ_L002_R2_001.fastq.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
}
)
self.file2.save()
self.file3 = File(
**{
"file_name": "FASTQ_L006_R1_001.fastq.gz",
"path": "/FASTQ/FASTQ_L006_R1_001.fastq.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
}
)
self.file3.save()
self.file4 = File(
**{
"file_name": "FASTQ_L006_R2_001.fastq.gz",
"path": "/FASTQ/FASTQ_L006_R2_001.fastq.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
}
)
self.file4.save()
file_list = [
{
"file_name": "GRCm38.fasta",
"path": "/resources/genomes/GRCm38/GRCm38.fasta",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "b37.fasta",
"path": "/resources/genomes/GRCh37/fasta/b37.fasta",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "hapmap_3.3.b37.vcf",
"path": "/resources/request_files/hapmap/hapmap_3.3.b37.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
"path": "/resources/genomes/GRCh37/facets_snps/dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "human.hg19.excl.tsv",
"path": "/resources/genomes/GRCh37/delly/human.hg19.excl.tsv",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "hotspot-list-union-v1-v2.maf",
"path": "/resources/qc_resources/hotspot-list-union-v1-v2.maf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "AgilentExon_v2_FP_tiling_intervals.intervals",
"path": "/resources/genomic_resources/targets/AgilentExon_v2/b37/AgilentExon_v2_FP_tiling_intervals.intervals",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "AgilentExon_v2_FP_tiling_genotypes.txt",
"path": "/resources/genomic_resources/targets/AgilentExon_v2/b37/AgilentExon_v2_FP_tiling_genotypes.txt",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "AgilentExon_v2_b37_baits.ilist",
"path": "/resources/genomic_resources/targets/AgilentExon_v2/b37/AgilentExon_v2_b37_baits.ilist",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "AgilentExon_v2_b37_targets.ilist",
"path": "/resources/genomic_resources/targets/AgilentExon_v2/b37/AgilentExon_v2_b37_targets.ilist",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "refGene_b37.sorted.txt",
"path": "/resources/request_files/refseq/refGene_b37.sorted.txt",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "ExAC_nonTCGA.r0.3.1.sites.vep.vcf.gz",
"path": "/resources/vep/cache/ExAC_nonTCGA.r0.3.1.sites.vep.vcf.gz",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "CosmicCodingMuts_v67_b37_20131024__NDS.vcf",
"path": "/resources/request_files/cosmic/CosmicCodingMuts_v67_b37_20131024__NDS.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "s_C_006537_N001_d.Group0.rg.md.abra.printreads.bam",
"path": "/resources/curated_bams/IDT_Exome_v1_FP_b37/s_C_006537_N001_d.Group0.rg.md.abra.printreads.bam",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "s_C_006284_N002_d.Group3.rg.md.abra.printreads.bam",
"path": "/resources/curated_bams/IDT_Exome_v1_FP_b37/s_C_006284_N002_d.Group3.rg.md.abra.printreads.bam",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "Mills_and_1000G_gold_standard.indels.b37.vcf",
"path": "/resources/request_files/indels_1000g/Mills_and_1000G_gold_standard.indels.b37.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "dbsnp_138.b37.excluding_sites_after_129.vcf",
"path": "/resources/request_files/dbsnp/dbsnp_138.b37.excluding_sites_after_129.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
{
"file_name": "1000G_phase1.snps.high_confidence.b37.vcf",
"path": "/resources/request_files/snps_1000g/1000G_phase1.snps.high_confidence.b37.vcf",
"size": 1234,
"file_group": self.file_group,
"file_type": self.file_type_unknown,
},
]
for f in file_list:
self.file = File(**f)
self.file.save()
self.outputs = {
"maf": {
"size": 68273211,
"class": "File",
"nameext": ".maf",
"basename": "test_1.muts.maf",
"checksum": "sha1$0ccf4718a717f5a991607561af0b5bf989caf1e4",
"location": "file:///outputs/test_1.muts.maf",
"nameroot": "test_1.muts.maf",
},
"bams": [
{
"size": 56524168530,
"class": "File",
"nameext": ".bam",
"basename": "test_1.printreads.bam",
"checksum": "sha1$e4c05e8b3e7c1d682640e690f71536e22cb63802",
"location": "file:///output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam",
"nameroot": "test_1.printreads.bam",
"secondaryFiles": [
{
"size": 7214992,
"class": "File",
"nameext": ".bai",
"basename": "test_1.rg.md.abra.printreads.bai",
"checksum": "sha1$79299d55657a0226206a4bf3ddaba854ae11f9f1",
"location": "file:///output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai",
"nameroot": "test_1.rg.md.abra.printreads.bai",
}
],
},
{
"size": 6163808009,
"class": "File",
"nameext": ".bam",
"basename": "test_2.rg.md.abra.printreads.bam",
"checksum": "sha1$15ddc908c3ece551d331e78806d3ac19569174c3",
"location": "file:///output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam",
"nameroot": "test_2.rg.md.abra.printreads",
"secondaryFiles": [
{
"size": 6235920,
"class": "File",
"nameext": ".bai",
"basename": "test_2.rg.md.abra.printreads.bai",
"checksum": "sha1$6fd2fc0ce6b42253cbac89d31e8eddb169d65605",
"location": "file:///output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai",
"nameroot": "test_2.rg.md.abra.printreads",
}
],
},
],
}
@patch("runner.pipeline.pipeline_cache.PipelineCache.get_pipeline")
def test_run_creation_from_cwl(self, mock_get_pipeline):
with open("runner/tests/run/pair-workflow.cwl", "r") as f:
app = json.load(f)
with open("runner/tests/run/inputs.json", "r") as f:
inputs = json.load(f)
mock_get_pipeline.return_value = app
run = RunObjectFactory.from_definition(str(self.run.id), inputs)
run.ready()
for inp in run.inputs:
if inp.name == "pair":
self.assertEqual(inp.db_value[0]["R1"][0]["location"], "bid://%s" % str(self.file1.id))
self.assertEqual(inp.value[0]["R1"][0]["path"], self.file1.path)
self.assertEqual(inp.db_value[0]["R2"][0]["location"], "bid://%s" % str(self.file2.id))
self.assertEqual(inp.value[0]["R2"][0]["path"], self.file2.path)
self.assertEqual(inp.db_value[1]["R1"][0]["location"], "bid://%s" % str(self.file3.id))
self.assertEqual(inp.value[1]["R1"][0]["path"], self.file3.path)
self.assertEqual(inp.db_value[1]["R2"][0]["location"], "bid://%s" % str(self.file4.id))
self.assertEqual(inp.value[1]["R2"][0]["path"], self.file4.path)
@patch("runner.pipeline.pipeline_cache.PipelineCache.get_pipeline")
def test_run_to_db(self, mock_get_pipeline):
with open("runner/tests/run/pair-workflow.cwl", "r") as f:
app = json.load(f)
with open("runner/tests/run/inputs.json", "r") as f:
inputs = json.load(f)
mock_get_pipeline.return_value = app
run = RunObjectFactory.from_definition(str(self.run.id), inputs)
run.to_db()
try:
run_obj = Run.objects.get(id=run.run_id)
except Run.DoesNotExist as e:
pass
self.assertEqual(str(run_obj.id), run.run_id)
@patch("notifier.tasks.send_notification.delay")
@patch("lib.memcache_lock.memcache_task_lock")
@patch("runner.pipeline.pipeline_cache.PipelineCache.get_pipeline")
@patch("file_system.tasks.populate_job_group_notifier_metadata.delay")
def test_run_complete_job(
self, mock_populate_job_group_notifier, mock_get_pipeline, memcache_task_lock, send_notification
):
with open("runner/tests/run/pair-workflow.cwl", "r") as f:
app = json.load(f)
with open("runner/tests/run/inputs.json", "r") as f:
inputs = json.load(f)
mock_populate_job_group_notifier.return_value = None
mock_get_pipeline.return_value = app
memcache_task_lock.return_value = True
send_notification.return_value = False
run = RunObjectFactory.from_definition(str(self.run.id), inputs)
run.to_db()
operator_run = OperatorRun.objects.first()
operator_run.runs.add(run.run_obj)
num_completed_runs = operator_run.num_completed_runs
complete_job(run.run_id, self.outputs)
operator_run.refresh_from_db()
self.assertEqual(operator_run.num_completed_runs, num_completed_runs + 1)
run_obj = RunObjectFactory.from_db(run.run_id)
file_obj = File.objects.filter(path=self.outputs["maf"]["location"].replace("file://", "")).first()
run_obj.to_db()
for out in run_obj.outputs:
if out.name == "maf":
self.assertEqual(out.value["location"], self.outputs["maf"]["location"])
self.assertEqual(FileProcessor.get_bid_from_file(file_obj), out.db_value["location"])
port = Port.objects.filter(run_id=run_obj.run_id, name="bams").first()
self.assertEqual(len(port.files.all()), 4)
expected_result = (
"/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam",
"/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai",
"/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam",
"/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai",
)
self.assertTrue(port.files.all()[0].path in expected_result)
self.assertTrue(port.files.all()[1].path in expected_result)
self.assertTrue(port.files.all()[2].path in expected_result)
self.assertTrue(port.files.all()[3].path in expected_result)
@patch("runner.models.Run.set_for_restart")
@patch("notifier.tasks.send_notification.delay")
@patch("lib.memcache_lock.memcache_task_lock")
@patch("runner.pipeline.pipeline_cache.PipelineCache.get_pipeline")
def test_run_fail_job(self, mock_get_pipeline, memcache_task_lock, send_notification, set_for_restart):
with open("runner/tests/run/pair-workflow.cwl", "r") as f:
app = json.load(f)
with open("runner/tests/run/inputs.json", "r") as f:
inputs = json.load(f)
set_for_restart.return_value = None
mock_get_pipeline.return_value = app
memcache_task_lock.return_value = True
send_notification.return_value = False
run = RunObjectFactory.from_definition(str(self.run.id), inputs)
run.to_db()
operator_run = OperatorRun.objects.first()
operator_run.runs.add(run.run_obj)
num_failed_runs = operator_run.num_failed_runs
fail_job(run.run_id, {"details": "Error has happened"})
operator_run.refresh_from_db()
self.assertEqual(operator_run.num_failed_runs, num_failed_runs + 1)
run_obj = RunObjectFactory.from_db(run.run_id)
self.assertEqual(run_obj.message, {"details": "Error has happened"})
@patch("runner.models.Run.set_for_restart")
@patch("notifier.tasks.send_notification.delay")
@patch("lib.memcache_lock.memcache_task_lock")
@patch("runner.pipeline.pipeline_cache.PipelineCache.get_pipeline")
def test_multiple_failed_on_same_job(
self, mock_get_pipeline, memcache_task_lock, send_notification, set_for_restart
):
with open("runner/tests/run/pair-workflow.cwl", "r") as f:
app = json.load(f)
with open("runner/tests/run/inputs.json", "r") as f:
inputs = json.load(f)
set_for_restart.return_value = None
memcache_task_lock.return_value = True
send_notification.return_value = False
mock_get_pipeline.return_value = app
run = RunObjectFactory.from_definition(str(self.run.id), inputs)
run.to_db()
operator_run = OperatorRun.objects.first()
operator_run.runs.add(run.run_obj)
num_failed_runs = operator_run.num_failed_runs
fail_job(run.run_id, {"details": "Error has happened"})
fail_job(run.run_id, {"details": "Error has happened"})
fail_job(run.run_id, {"details": "Error has happened"})
operator_run.refresh_from_db()
self.assertEqual(operator_run.num_failed_runs, num_failed_runs + 1)
def setUp(self):
self.storage = Storage(name="test", type=StorageType.LOCAL)
self.storage.save()
self.file_group = FileGroup(name="Test Files", storage=self.storage)
self.file_group.save()
self.file_type_fastq = FileType(name='fastq')
self.file_type_fastq.save()
self.file_ext_fastq_1 = FileExtension(extension='fastq',
file_type=self.file_type_fastq)
self.file_ext_fastq_1.save()
self.file_ext_fastq_2 = FileExtension(extension='fastq.gz',
file_type=self.file_type_fastq)
self.file_ext_fastq_2.save()
self.file_type_vcf = FileType(name='vcf')
self.file_type_vcf.save()
self.file_type_txt = FileType(name='txt')
self.file_type_txt.save()
self.file_type_tsv = FileType(name='tsv')
self.file_type_tsv.save()
self.file_type_maf = FileType(name='maf')
self.file_type_maf.save()
self.file_type_unknown = FileType(name='unknown')
self.file_type_unknown.save()
self.file1 = File(file_name="S16_R1_001.fastq.gz",
path="/path/to/file/S16_R1_001.fastq.gz",
file_type=self.file_type_fastq,
size=5966546453,
file_group=self.file_group)
self.file1.save()
self.file2 = File(file_name="S16_R2_001.fastq.gz",
path="/path/to/file/S16_R2_001.fastq.gz",
file_type=self.file_type_fastq,
size=5832468368,
file_group=self.file_group)
self.file2.save()
self.file3 = File(file_name="P-S12_R1_001.fastq.gz",
path="/path/to/file/Sample_P/P-S12_R1_001.fastq.gz",
file_type=self.file_type_fastq,
size=3576965127,
file_group=self.file_group)
self.file3.save()
self.file4 = File(file_name="P-S12_R2_001.fastq.gz",
path="/path/to/file/Sample_P/P-S12_R2_001.fastq.gz",
file_type=self.file_type_fastq,
size=3592299152,
file_group=self.file_group)
self.file4.save()
self.file5 = File(file_name="refGene_b37.sorted.txt",
path="/path/to/file/refGene_b37.sorted.txt",
file_type=self.file_type_fastq,
size=359229,
file_group=self.file_group)
self.file5.save()
self.file6 = File(
file_name=
"dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
path=
"/path/to/file/dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
file_type=self.file_type_vcf,
size=359228,
file_group=self.file_group)
self.file6.save()
self.file7 = File(file_name="FP_tiling_genotypes.txt",
path="/path/to/file/FP_tiling_genotypes.txt",
file_type=self.file_type_vcf,
size=359228,
file_group=self.file_group)
self.file7.save()
self.file8 = File(file_name="hotspot-list-union-v1-v2.txt",
path="/path/to/file/hotspot-list-union-v1-v2.txt",
file_type=self.file_type_vcf,
size=359228,
file_group=self.file_group)
self.file8.save()
self.file9 = File(file_name="human.hg19.excl.tsv",
path="/path/to/file/human.hg19.excl.tsv",
file_type=self.file_type_tsv,
size=359228,
file_group=self.file_group)
self.file9.save()
self.file10 = File(
file_name="IDT_Exome_v1_FP_b37_baits.ilist",
path="/path/to/file/IDT_Exome_v1_FP_b37_baits.ilist",
file_type=self.file_type_tsv,
size=359228,
file_group=self.file_group)
self.file10.save()
self.file11 = File(file_name="hotspot-list-union-v1-v2.maf",
path="/path/to/file/hotspot-list-union-v1-v2.maf",
file_type=self.file_type_maf,
size=359228,
file_group=self.file_group)
self.file11.save()
self.file12 = File(
file_name="IDT_Exome_v1_FP_b37_targets.ilist",
path="/path/to/file/IDT_Exome_v1_FP_b37_targets.ilist",
file_type=self.file_type_maf,
size=359228,
file_group=self.file_group)
self.file12.save()
self.file13 = File(file_name="FP_tiling_intervals.intervals",
path="/path/to/file/FP_tiling_intervals.intervals",
file_type=self.file_type_maf,
size=359228,
file_group=self.file_group)
self.file13.save()
self.file14 = File(file_name="refGene_b37.sorted.txt.sec",
path="/path/to/file/refGene_b37.sorted.txt.sec",
file_type=self.file_type_maf,
size=359228,
file_group=self.file_group)
self.file14.save()
class ProcessorTest(APITestCase):
def setUp(self):
self.storage = Storage(name="test", type=StorageType.LOCAL)
self.storage.save()
self.file_group = FileGroup(name="Test Files", storage=self.storage)
self.file_group.save()
self.file_type_fastq = FileType(name='fastq')
self.file_type_fastq.save()
self.file_ext_fastq_1 = FileExtension(extension='fastq',
file_type=self.file_type_fastq)
self.file_ext_fastq_1.save()
self.file_ext_fastq_2 = FileExtension(extension='fastq.gz',
file_type=self.file_type_fastq)
self.file_ext_fastq_2.save()
self.file_type_vcf = FileType(name='vcf')
self.file_type_vcf.save()
self.file_type_txt = FileType(name='txt')
self.file_type_txt.save()
self.file_type_tsv = FileType(name='tsv')
self.file_type_tsv.save()
self.file_type_maf = FileType(name='maf')
self.file_type_maf.save()
self.file_type_unknown = FileType(name='unknown')
self.file_type_unknown.save()
self.file1 = File(file_name="S16_R1_001.fastq.gz",
path="/path/to/file/S16_R1_001.fastq.gz",
file_type=self.file_type_fastq,
size=5966546453,
file_group=self.file_group)
self.file1.save()
self.file2 = File(file_name="S16_R2_001.fastq.gz",
path="/path/to/file/S16_R2_001.fastq.gz",
file_type=self.file_type_fastq,
size=5832468368,
file_group=self.file_group)
self.file2.save()
self.file3 = File(file_name="P-S12_R1_001.fastq.gz",
path="/path/to/file/Sample_P/P-S12_R1_001.fastq.gz",
file_type=self.file_type_fastq,
size=3576965127,
file_group=self.file_group)
self.file3.save()
self.file4 = File(file_name="P-S12_R2_001.fastq.gz",
path="/path/to/file/Sample_P/P-S12_R2_001.fastq.gz",
file_type=self.file_type_fastq,
size=3592299152,
file_group=self.file_group)
self.file4.save()
self.file5 = File(file_name="refGene_b37.sorted.txt",
path="/path/to/file/refGene_b37.sorted.txt",
file_type=self.file_type_fastq,
size=359229,
file_group=self.file_group)
self.file5.save()
self.file6 = File(
file_name=
"dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
path=
"/path/to/file/dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
file_type=self.file_type_vcf,
size=359228,
file_group=self.file_group)
self.file6.save()
self.file7 = File(file_name="FP_tiling_genotypes.txt",
path="/path/to/file/FP_tiling_genotypes.txt",
file_type=self.file_type_vcf,
size=359228,
file_group=self.file_group)
self.file7.save()
self.file8 = File(file_name="hotspot-list-union-v1-v2.txt",
path="/path/to/file/hotspot-list-union-v1-v2.txt",
file_type=self.file_type_vcf,
size=359228,
file_group=self.file_group)
self.file8.save()
self.file9 = File(file_name="human.hg19.excl.tsv",
path="/path/to/file/human.hg19.excl.tsv",
file_type=self.file_type_tsv,
size=359228,
file_group=self.file_group)
self.file9.save()
self.file10 = File(
file_name="IDT_Exome_v1_FP_b37_baits.ilist",
path="/path/to/file/IDT_Exome_v1_FP_b37_baits.ilist",
file_type=self.file_type_tsv,
size=359228,
file_group=self.file_group)
self.file10.save()
self.file11 = File(file_name="hotspot-list-union-v1-v2.maf",
path="/path/to/file/hotspot-list-union-v1-v2.maf",
file_type=self.file_type_maf,
size=359228,
file_group=self.file_group)
self.file11.save()
self.file12 = File(
file_name="IDT_Exome_v1_FP_b37_targets.ilist",
path="/path/to/file/IDT_Exome_v1_FP_b37_targets.ilist",
file_type=self.file_type_maf,
size=359228,
file_group=self.file_group)
self.file12.save()
self.file13 = File(file_name="FP_tiling_intervals.intervals",
path="/path/to/file/FP_tiling_intervals.intervals",
file_type=self.file_type_maf,
size=359228,
file_group=self.file_group)
self.file13.save()
self.file14 = File(file_name="refGene_b37.sorted.txt.sec",
path="/path/to/file/refGene_b37.sorted.txt.sec",
file_type=self.file_type_maf,
size=359228,
file_group=self.file_group)
self.file14.save()
def test_convert_list_to_bid(self):
port_value_list = [{
"CN":
"MSKCC",
"ID":
"TEST_ID_1",
"LB":
"TEST_LB_1",
"PL":
"Illumina",
"PU": ["TEST_PU_1"],
"R1": [{
"location": "juno:///path/to/file/S16_R1_001.fastq.gz",
"size": 5966546453,
"class": "File"
}],
"R2": [{
"location": "juno:///path/to/file/S16_R2_001.fastq.gz",
"size": 5832468368,
"class": "File"
}],
"bam": [],
"zR1": [],
"zR2": [],
"RG_ID": ["TEST_RG_ID_1"],
"adapter":
"AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG",
"adapter2":
"AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT",
"bwa_output":
"bwa_output_1.bam"
}, {
"CN":
"MSKCC",
"ID":
"TEST_ID_2",
"LB":
"TEST_LB_2",
"PL":
"Illumina",
"PU": ["TEST_PU_2"],
"R1": [{
"location":
"juno:///path/to/file/Sample_P/P-S12_R1_001.fastq.gz",
"size": 3576965127,
"class": "File"
}],
"R2": [{
"location":
"juno:///path/to/file/Sample_P/P-S12_R2_001.fastq.gz",
"size": 3592299152,
"class": "File"
}],
"bam": [],
"zR1": [],
"zR2": [],
"RG_ID": ["TEST_RG_ID_2"],
"adapter":
"AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG",
"adapter2":
"AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT",
"bwa_output":
"bwa_output_2.bam"
}]
result = PortProcessor.process_files(port_value_list,
PortAction.CONVERT_TO_BID)
difference = diff(port_value_list, result)
difference = list(difference)
self.assertEqual(difference[0][0], 'change')
self.assertEqual(difference[0][1][0], 0)
self.assertEqual(difference[0][1][1], 'R1')
self.assertEqual(difference[0][1][2], 0)
self.assertEqual(difference[0][1][3], 'location')
self.assertEqual(difference[0][2][0], 'juno://%s' % self.file1.path)
self.assertEqual(difference[0][2][1], 'bid://%s' % str(self.file1.id))
self.assertEqual(difference[0][0], 'change')
self.assertEqual(difference[1][1][0], 0)
self.assertEqual(difference[1][1][1], 'R2')
self.assertEqual(difference[1][1][2], 0)
self.assertEqual(difference[1][1][3], 'location')
self.assertEqual(difference[1][2][0], 'juno://%s' % self.file2.path)
self.assertEqual(difference[1][2][1], 'bid://%s' % str(self.file2.id))
self.assertEqual(difference[2][1][0], 1)
self.assertEqual(difference[2][1][1], 'R1')
self.assertEqual(difference[2][1][2], 0)
self.assertEqual(difference[2][1][3], 'location')
self.assertEqual(difference[2][2][0], 'juno://%s' % self.file3.path)
self.assertEqual(difference[2][2][1], 'bid://%s' % str(self.file3.id))
self.assertEqual(difference[3][1][0], 1)
self.assertEqual(difference[3][1][1], 'R2')
self.assertEqual(difference[3][1][2], 0)
self.assertEqual(difference[3][1][3], 'location')
self.assertEqual(difference[3][2][0], 'juno://%s' % self.file4.path)
self.assertEqual(difference[3][2][1], 'bid://%s' % str(self.file4.id))
def test_convert_dict_to_bid(self):
port_value_dict = {
"test_data":
"string_value",
"test_data_none":
None,
"test_data_int":
2,
"refseq": {
"location":
"juno:///path/to/file/refGene_b37.sorted.txt",
"class":
"File",
"secondaryFiles": [{
"class":
"File",
"location":
"juno:///path/to/file/refGene_b37.sorted.txt.sec"
}]
},
"double_list_test": [[{
"location":
"juno:///path/to/file/dbsnp_137.b37__RmDupsClean__plusPseudo50__DROP_SORT.vcf.gz",
"class": "File"
}]],
"conpair_markers_bed":
"string_value_bed",
"double_nested_port_list": [{
"nested_port_list_1": [{
"bait_intervals_1": {
"location":
"juno:///path/to/file/IDT_Exome_v1_FP_b37_baits.ilist",
"class": "File"
},
"bait_intervals_2": {
"location":
"juno:///path/to/file/IDT_Exome_v1_FP_b37_baits.ilist",
"class": "File"
}
}]
}]
}
file_list = []
result = PortProcessor.process_files(port_value_dict,
PortAction.CONVERT_TO_BID,
file_list=file_list)
difference = diff(port_value_dict, result)
difference = list(difference)
self.assertEqual(difference[0][0], 'change')
self.assertEqual(difference[0][1], 'refseq.location')
self.assertEqual(difference[0][2][0], 'juno://%s' % self.file5.path)
self.assertEqual(difference[0][2][1], 'bid://%s' % str(self.file5.id))
self.assertEqual(difference[1][0], 'change')
self.assertEqual(difference[1][1][0], 'refseq')
self.assertEqual(difference[1][1][1], 'secondaryFiles')
self.assertEqual(difference[1][1][2], 0)
self.assertEqual(difference[1][1][3], 'location')
self.assertEqual(difference[1][2][0], 'juno://%s' % self.file14.path)
self.assertEqual(difference[1][2][1], 'bid://%s' % str(self.file14.id))
self.assertEqual(difference[2][0], 'change')
self.assertEqual(difference[2][1][0], 'double_list_test')
self.assertEqual(difference[2][1][1], 0)
self.assertEqual(difference[2][1][2], 0)
self.assertEqual(difference[2][1][3], 'location')
self.assertEqual(difference[2][2][0], 'juno://%s' % self.file6.path)
self.assertEqual(difference[2][2][1], 'bid://%s' % str(self.file6.id))
self.assertEqual(difference[3][0], 'change')
self.assertEqual(difference[3][1][0], 'double_nested_port_list')
self.assertEqual(difference[3][1][1], 0)
self.assertEqual(difference[3][1][2], 'nested_port_list_1')
self.assertEqual(difference[3][1][3], 0)
self.assertEqual(difference[3][1][4], 'bait_intervals_1')
self.assertEqual(difference[3][1][5], 'location')
self.assertEqual(difference[3][2][0], 'juno://%s' % self.file10.path)
self.assertEqual(difference[3][2][1], 'bid://%s' % self.file10.id)
self.assertEqual(difference[4][0], 'change')
self.assertEqual(difference[4][1][0], 'double_nested_port_list')
self.assertEqual(difference[4][1][1], 0)
self.assertEqual(difference[4][1][2], 'nested_port_list_1')
self.assertEqual(difference[4][1][3], 0)
self.assertEqual(difference[4][1][4], 'bait_intervals_2')
self.assertEqual(difference[4][1][5], 'location')
self.assertEqual(difference[4][2][0], 'juno://%s' % self.file10.path)
self.assertEqual(difference[4][2][1], 'bid://%s' % self.file10.id)
self.assertEqual(len(file_list), 5)
def test_create_file_setting_proper_file_type_based_on_extension(self):
file_obj = FileProcessor.create_file_obj(
'file:///path/to/file.fastq.gz', 123345,
'sha1$calculated checksum', str(self.file_group.id), {})
self.assertEqual(file_obj.file_type, self.file_type_fastq)
def test_create_file_type_unknown(self):
file_obj = FileProcessor.create_file_obj(
'file:///path/to/file.unknown_data_type', 123345,
'sha1$calculated checksum', str(self.file_group.id), {})
self.assertEqual(file_obj.file_type, self.file_type_unknown)
def test_create_file_obj_bad_file_group(self):
file_group_id = str(uuid.uuid4())
with self.assertRaises(Exception) as context:
file_obj = FileProcessor.create_file_obj(
'file:///path/to/file.unknown_data_type', 123345,
'sha1$calculated checksum', file_group_id, {})
self.assertTrue('Invalid FileGroup id: %s' %
file_group_id in context.exception)
def test_cwl_format(self):
refseq_port = {
"refseq": {
"location":
"juno:///path/to/file/refGene_b37.sorted.txt",
"class":
"File",
"secondaryFiles": [{
"class":
"File",
"location":
"juno:///path/to/file/refGene_b37.sorted.txt.sec"
}]
}
}
refseq_port_cwl_format = {
"refseq": {
"class":
"File",
"size":
359229,
"basename":
"refGene_b37.sorted.txt",
"nameext":
".txt",
"nameroot":
"refGene_b37.sorted",
"path":
"/path/to/file/refGene_b37.sorted.txt",
"secondaryFiles": [{
"class":
"File",
"size":
359228,
"basename":
"refGene_b37.sorted.txt.sec",
"nameext":
".sec",
"nameroot":
"refGene_b37.sorted.txt",
"path":
"/path/to/file/refGene_b37.sorted.txt.sec"
}]
}
}
cwl_format_port = PortProcessor.process_files(
refseq_port, PortAction.CONVERT_TO_CWL_FORMAT)
self.assertEqual(cwl_format_port, refseq_port_cwl_format)
class FileTest(APITestCase):
def setUp(self):
self.user = User.objects.create_user(username="******",
password="******",
email='*****@*****.**')
self.storage = Storage(name="test", type=StorageType.LOCAL)
self.storage.save()
self.file_group = FileGroup(name="Test Files", storage=self.storage)
self.file_group.save()
self.file_type_fasta = FileType(name='fasta')
self.file_type_fasta.save()
self.file_type_bam = FileType(name='bam')
self.file_type_bam.save()
self.file_type_fastq = FileType(name='fastq')
self.file_type_fastq.save()
def _create_single_file(self, path, file_type, group_id, request_id,
sample_id):
file_type_obj = FileType.objects.get(name=file_type)
group_id_obj = FileGroup.objects.get(id=group_id)
file = File(path=path,
file_name=os.path.basename(path),
file_type=file_type_obj,
file_group=group_id_obj,
size=1234)
file.save()
file_metadata = {"requestId": request_id, "igoSampleId": sample_id}
file_metadata = FileMetadata(file=file, metadata=file_metadata)
file_metadata.save()
return file
def _create_files(self, file_type, amount):
for i in range(amount):
request_id = "request_%s" % str(i)
sample_id = "sample_%s" % str(i)
if file_type == 'fasta':
file_path_R1 = "/path/to/%s_R1.%s" % (sample_id, file_type)
self._create_single_file(file_path_R1, file_type,
str(self.file_group.id), request_id,
sample_id)
file_path_R2 = "/path/to/%s_R2.%s" % (sample_id, file_type)
self._create_single_file(file_path_R2, file_type,
str(self.file_group.id), request_id,
sample_id)
else:
file_path = "/path/to/%s.%s" % (sample_id, file_type)
self._create_single_file(file_path, file_type,
str(self.file_group.id), request_id,
sample_id)
def _generate_jwt(self, username="******", password="******"):
response = self.client.post('/api-token-auth/', {
'username': username,
'password': password
},
format='json')
return response.data['access']
def test_create_file_unauthorized(self):
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
"invalid_token")
response = self.client.post('/v0/fs/files/', {
"path": "/path/to/file.fasta",
"size": 1234,
"file_group": str(self.file_group.id),
"file_type": str(self.file_type_fasta.name)
},
format='json')
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
def test_create_file_too_long_sample_name(self):
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.post('/v0/fs/files/', {
"path": "/path/to/file.fasta",
"size": 1234,
"file_group": str(self.file_group.id),
"file_type": self.file_type_fasta.name,
"metadata": {
"requestId": "Request_001",
"sampleId": "igoSampleId_001_length_too_long_123"
}
},
format='json')
self.assertContains(response,
'too long',
status_code=status.HTTP_400_BAD_REQUEST)
def test_create_file_successful(self):
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.post('/v0/fs/files/', {
"path": "/path/to/file.fasta",
"size": 1234,
"file_group": str(self.file_group.id),
"file_type": self.file_type_fasta.name,
"metadata": {
"requestId": "Request_001",
"sampleId": "igoSampleId_001"
}
},
format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(response.data['file_name'], "file.fasta")
try:
sample = Sample.objects.get(sample_id="igoSampleId_001")
except Sample.DoesNotExist:
sample = None
self.assertIsNotNone(sample)
def test_create_file_unknown_file_type(self):
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.post('/v0/fs/files/', {
"path": "/path/to/file.fasta",
"size": 1234,
"file_group": str(self.file_group.id),
"file_type": 'unknown',
"metadata": {
"requestId": "Request_001",
"igoSampleId": "igoSampleId_001"
}
},
format='json')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(str(response.data['file_type'][0]),
'Unknown file_type: unknown')
# def test_create_file_invalid_metadata(self):
"""
Return this test when we implement metadata validation
"""
# self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' % self._generate_jwt())
# response = self.client.post('/v0/fs/files/',
# {
# "path": "/path/to/file.fasta",
# "size": 1234,
# "file_group": str(self.file_group.id),
# "file_type": self.file_type_fasta.name,
# "metadata": {}
# },
# format='json')
# self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_create_file_invalid_file_group(self):
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.post('/v0/fs/files/', {
"path": "/path/to/file.fasta",
"size": 1234,
"file_group": str(uuid.uuid4()),
"file_type": self.file_type_fasta.name,
"metadata": {}
},
format='json')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_list_files_by_file_name(self):
self._create_files('fasta', 30)
self._create_files('bam', 30)
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
"invalid_token")
response = self.client.get('/v0/fs/files/', format='json')
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.get('/v0/fs/files/', format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['count'], 90)
self.assertEqual(len(response.data['results']),
settings.PAGINATION_DEFAULT_PAGE_SIZE)
response = self.client.get('/v0/fs/files/?file_type=fasta',
format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['count'], 60)
response = self.client.get(
'/v0/fs/files/?file_type=fasta&filename_regex=sample_1_',
format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['count'], 2)
response = self.client.get(
'/v0/fs/files/?metadata=requestId:request_1', format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['count'], 3)
def test_update_file_metadata(self):
_file = self._create_single_file('/path/to/sample_file.bam', 'bam',
str(self.file_group.id), 'request_id',
'sample_id')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.put('/v0/fs/files/%s/' % str(_file.id), {
"path": _file.file_name,
"size": _file.size,
"file_group": str(_file.file_group.id),
"file_type": _file.file_type.name,
"metadata": {
"requestId":
"Request_001",
"igoSampleId":
_file.filemetadata_set.first().metadata['igoSampleId']
}
},
format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['metadata']['requestId'], "Request_001")
file_metadata_count = FileMetadata.objects.filter(
file=str(_file.id)).count()
self.assertEqual(file_metadata_count, 2)
def test_patch_file_metadata(self):
_file = self._create_single_file('/path/to/sample_file.bam', 'bam',
str(self.file_group.id), 'request_id',
'sample_id')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.patch(
'/v0/fs/files/%s/' % str(_file.id),
{"metadata": {
"requestId": "Request_001",
}},
format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['metadata']['requestId'], "Request_001")
self.assertEqual(response.data['metadata']['igoSampleId'], "sample_id")
file_metadata_count = FileMetadata.objects.filter(
file=str(_file.id)).count()
self.assertEqual(file_metadata_count, 2)
response = self.client.patch(
'/v0/fs/files/%s/' % str(_file.id),
{"metadata": {
"requestId": "Request_002",
}},
format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['metadata']['requestId'], "Request_002")
self.assertEqual(response.data['metadata']['igoSampleId'], "sample_id")
file_metadata_count = FileMetadata.objects.filter(
file=str(_file.id)).count()
self.assertEqual(file_metadata_count, 3)
def test_batch_patch_file_metadata(self):
first_file = self._create_single_file('/path/to/first_file.bam', 'bam',
str(self.file_group.id),
'first_request_id',
'first_sample_id')
second_file = self._create_single_file('/path/to/second_file.bam',
'bam', str(self.file_group.id),
'second_request_id',
'second_sample_id')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
patch_json = {
"patch_files": [{
"id": first_file.id,
"patch": {
"metadata": {
"requestId": "Request_001"
}
}
}, {
"id": second_file.id,
"patch": {
"metadata": {
"requestId": "Request_002"
}
}
}]
}
response = self.client.post('/v0/fs/batch-patch-files',
patch_json,
format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
first_file_metadata = FileMetadata.objects.order_by(
'file', '-version').distinct('file').filter(
file=str(first_file.id)).first()
second_file_metadata = FileMetadata.objects.order_by(
'file', '-version').distinct('file').filter(
file=str(second_file.id)).first()
self.assertEqual(first_file_metadata.metadata['requestId'],
"Request_001")
self.assertEqual(second_file_metadata.metadata['requestId'],
"Request_002")
def test_fail_batch_patch_file_metadata(self):
first_file = self._create_single_file('/path/to/first_file.bam', 'bam',
str(self.file_group.id),
'first_request_id',
'first_sample_id')
second_file = self._create_single_file('/path/to/second_file.bam',
'bam', str(self.file_group.id),
'second_request_id',
'second_sample_id')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
patch_json = {
"patch_files": [{
"id": None,
"patch": {
"metadata": {
"requestId": "Request_001"
}
}
}, {
"id": None,
"patch": {
"metadata": {
"requestId": "Request_002"
}
}
}]
}
response = self.client.post('/v0/fs/batch-patch-files',
patch_json,
format='json')
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
def test_partial_fail_batch_patch_file_metadata(self):
first_file = self._create_single_file('/path/to/first_file.bam', 'bam',
str(self.file_group.id),
'first_request_id',
'first_sample_id')
second_file = self._create_single_file('/path/to/second_file.bam',
'bam', str(self.file_group.id),
'second_request_id',
'second_sample_id')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
patch_json = {
"patch_files": [{
"id": first_file.id,
"patch": {
"metadata": {
"requestId": "Request_001"
}
}
}, {
"id": None,
"patch": {
"metadata": {
"requestId": "Request_002"
}
}
}]
}
response = self.client.post('/v0/fs/batch-patch-files',
patch_json,
format='json')
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
first_file_metadata = FileMetadata.objects.order_by(
'file', '-version').distinct('file').filter(
file=str(first_file.id)).first()
second_file_metadata = FileMetadata.objects.order_by(
'file', '-version').distinct('file').filter(
file=str(second_file.id)).first()
self.assertEqual(first_file_metadata.metadata['requestId'],
"first_request_id")
self.assertEqual(second_file_metadata.metadata['requestId'],
"second_request_id")
def test_update_file_metadata_users_update(self):
_file = self._create_single_file('/path/to/sample_file.bam', 'bam',
str(self.file_group.id), 'request_id',
'sample_id')
user1 = User.objects.create_user(username="******",
password="******",
email='*****@*****.**')
user2 = User.objects.create_user(username="******",
password="******",
email='*****@*****.**')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt(username=user1.username))
response = self.client.put('/v0/fs/files/%s/' % _file.id, {
"path": _file.file_name,
"size": _file.size,
"file_group": str(_file.file_group.id),
"file_type": _file.file_type.name,
"metadata": {
"requestId":
"Request_001",
"igoSampleId":
_file.filemetadata_set.first().metadata['igoSampleId']
}
},
format='json')
self.assertEqual(response.data['user'], user1.username)
self.assertEqual(response.data['metadata']['requestId'], 'Request_001')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt(username=user2.username))
response = self.client.put('/v0/fs/files/%s/' % str(_file.id), {
"path": _file.file_name,
"size": _file.size,
"file_group": str(_file.file_group.id),
"file_type": _file.file_type.name,
"metadata": {
"requestId":
"Request_002",
"igoSampleId":
_file.filemetadata_set.first().metadata['igoSampleId']
}
},
format='json')
self.assertEqual(response.data['user'], user2.username)
self.assertEqual(response.data['metadata']['requestId'], 'Request_002')
# Check listing files as well
response = self.client.get(
'/v0/fs/files/?metadata=requestId:Request_001', format='json')
self.assertEqual(len(response.json()['results']), 0)
response = self.client.get(
'/v0/fs/files/?metadata=requestId:Request_002', format='json')
self.assertEqual(len(response.json()['results']), 1)
# def test_update_file_metadata_invalid(self):
"""
Return this test when we implement metadata validation
"""
# _file = self._create_single_file('/path/to/sample_file.bam',
# 'bam',
# str(self.file_group.id),
# 'request_id',
# 'sample_id')
# self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' % self._generate_jwt())
# response = self.client.put('/v0/fs/files/%s/' % str(_file.id),
# {
# "path": _file.file_name,
# "size": _file.size,
# "file_group": str(_file.file_group.id),
# "file_type": _file.file_type.name,
# "metadata": {
# "igoSampleId": _file.filemetadata_set.first().metadata['igoSampleId']
# }
# },
# format='json')
# self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_delete_file(self):
_file = self._create_single_file('/path/to/sample_file.bam', 'bam',
str(self.file_group.id), 'request_id',
'sample_id')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.delete('/v0/fs/files/%s/' % str(_file.id))
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
file_metadata_count = FileMetadata.objects.filter(
file=str(_file.id)).count()
self.assertEqual(file_metadata_count, 0)
def test_list_all_metadata(self):
_file = self._create_single_file('/path/to/sample_file.bam', 'bam',
str(self.file_group.id), 'request_id',
'sample_id')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.put('/v0/fs/files/%s/' % str(_file.id), {
"path": _file.file_name,
"size": _file.size,
"file_group": str(_file.file_group.id),
"file_type": _file.file_type.name,
"metadata": {
"requestId":
"Request_001",
"igoSampleId":
_file.filemetadata_set.first().metadata['igoSampleId']
}
},
format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
response = self.client.get('/v0/fs/metadata/?file_id=%s' %
str(_file.id),
format='json')
self.assertEqual(len(response.data['results']), 2)
def test_list_files(self):
self._create_single_file('/path/to/file1_R1.fastq', 'fastq',
str(self.file_group.id), '1', '1s')
self._create_single_file('/path/to/file1_R2.fastq', 'fastq',
str(self.file_group.id), '1', '1s')
self._create_single_file('/path/to/file2_R1.fastq', 'fastq',
str(self.file_group.id), '2', '1s')
self._create_single_file('/path/to/file2_R2.fastq', 'fastq',
str(self.file_group.id), '2', '1s')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.get(
'/v0/fs/files/?path=/path/to/file1_R1.fastq', format='json')
self.assertEqual(len(response.json()['results']), 1)
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.get(
'/v0/fs/files/?path=/path/to/file1_R1.fastq&values_metadata=requestId',
format='json')
self.assertEqual(response.json()['results'][0], '1')
def test_metadata_clean_function(self):
test1 = "abc\tdef2"
test2 = """
abc\tdef!
"""
self.assertEqual(MetadataValidator.clean_value(test1), "abc def2")
self.assertEqual(MetadataValidator.clean_value(test2), "abc def")
def test_query_files(self):
self._create_single_file('/path/to/file1.fastq', 'fastq',
str(self.file_group.id), '1', '3')
self._create_single_file('/path/to/file2.fastq', 'fastq',
str(self.file_group.id), '1', '2')
self._create_single_file('/path/to/file3.fastq', 'fastq',
str(self.file_group.id), '2', '1')
self._create_single_file('/path/to/file4.fastq', 'fastq',
str(self.file_group.id), '3', '4')
self._create_single_file('/path/to/file5.fastq', 'fastq',
str(self.file_group.id), '4', '3')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.get(
'/v0/fs/files/?metadata=requestId:1&values_metadata=requestId,igoSampleId',
format='json')
self.assertEqual(len(response.json()['results']), 2)
response = self.client.get(
'/v0/fs/files/?values_metadata=requestId,igoSampleId',
format='json')
self.assertEqual(len(response.json()['results']), 5)
def test_sample_list(self):
sample = Sample.objects.create(sample_id='08944_B')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.get('/v0/fs/sample/', format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(len(response.json()['results']), 1)
def test_sample_update(self):
sample = Sample.objects.create(sample_id='08944_B')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.patch('/v0/fs/sample/%s/' % str(sample.id), {
"redact": True,
},
format='json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.json().get('redact'), True)
def test_sample_create(self):
sample = Sample.objects.create(sample_id='08944_B')
self.client.credentials(HTTP_AUTHORIZATION='Bearer %s' %
self._generate_jwt())
response = self.client.post('/v0/fs/sample/', {
"sample_id": "TEST_001",
"redact": False
},
format='json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(response.json().get('redact'), False)