def create_processor_job(self, pipeline="AFFY_TO_PCL", ram_amount=2048):
job = ProcessorJob(
pipeline_applied=pipeline,
nomad_job_id="PROCESSOR/dispatch-1528945054-e8eaf540",
ram_amount=ram_amount,
num_retries=0,
volume_index="1",
success=None)
job.save()
og_file = OriginalFile()
og_file.source_filename = "doesn't matter"
og_file.filename = "this either"
og_file.absolute_file_path = "nor this"
og_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = job
assoc1.save()
og_file = OriginalFile()
og_file.source_filename = "doesn't matter"
og_file.filename = "this either"
og_file.absolute_file_path = "nor this"
og_file.save()
assoc = ProcessorJobOriginalFileAssociation()
assoc.original_file = og_file
assoc.processor_job = job
assoc.save()
return job
def prepare_original_files(length: str) -> List[OriginalFile]:
og_file = OriginalFile()
og_file.source_filename = "aegilops_tauschii_" + length + ".fa.gz"
og_file.filename = "aegilops_tauschii_" + length + ".fa.gz"
og_file.absolute_file_path = (
"/home/user/data_store/raw/TEST/TRANSCRIPTOME_INDEX/"
"AEGILOPS_TAUSCHII/aegilops_tauschii_short.fa.gz")
og_file.source_url = (
"ftp://ftp.ensemblgenomes.org/pub/release-39/plants/fasta/"
"aegilops_tauschii/dna/Aegilops_tauschii.ASM34733v1.dna.toplevel.fa.gz"
)
og_file.is_downloaded = True
# We need to add the URL here so that _extract_assembly_information works properly
og_file.source_url = "ftp://ftp.ensemblgenomes.org/pub/release-39/plants/fasta/aegilops_tauschii/dna/Aegilops_tauschii.ASM34733v1.dna.toplevel.fa.gz"
og_file.save()
og_file2 = OriginalFile()
og_file2.source_filename = "aegilops_tauschii_" + length + ".gtf.gz"
og_file2.filename = "aegilops_tauschii_" + length + ".gtf.gz"
og_file2.absolute_file_path = (
"/home/user/data_store/raw/TEST/TRANSCRIPTOME_INDEX/"
"AEGILOPS_TAUSCHII/aegilops_tauschii_short.gtf.gz")
og_file2.source_url = (
"ftp://ftp.ensemblgenomes.org/pub/release-39/plants/gtf/"
"aegilops_tauschii/Aegilops_tauschii.ASM34733v1.39.gtf.gz")
og_file2.is_downloaded = True
# We need to add the URL here so that _extract_assembly_information works properly
og_file2.source_url = "ftp://ftp.ensemblgenomes.org/pub/release-39/plants/gtf/aegilops_tauschii/Aegilops_tauschii.ASM34733v1.39.gtf.gz"
og_file2.save()
return [og_file, og_file2]
def create_downloader_job():
job = DownloaderJob(
downloader_task="SRA",
batch_job_id="DEFAULT",
num_retries=0,
accession_code="NUNYA",
success=None,
)
job.save()
og_file = OriginalFile()
og_file.source_filename = "doesn't matter"
og_file.filename = "this either"
og_file.absolute_file_path = "nor this"
og_file.save()
assoc1 = DownloaderJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.downloader_job = job
assoc1.save()
og_file = OriginalFile()
og_file.source_filename = "doesn't matter"
og_file.filename = "this either"
og_file.absolute_file_path = "nor this"
og_file.save()
assoc = DownloaderJobOriginalFileAssociation()
assoc.original_file = og_file
assoc.downloader_job = job
assoc.save()
return job
def create_downloader_job(self):
job = DownloaderJob(
downloader_task="SRA",
nomad_job_id="DOWNLOADER/dispatch-1528945054-e8eaf540",
num_retries=0,
accession_code="NUNYA",
success=None)
job.save()
og_file = OriginalFile()
og_file.source_filename = "doesn't matter"
og_file.filename = "this either"
og_file.absolute_file_path = "nor this"
og_file.save()
assoc1 = DownloaderJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.downloader_job = job
assoc1.save()
og_file = OriginalFile()
og_file.source_filename = "doesn't matter"
og_file.filename = "this either"
og_file.absolute_file_path = "nor this"
og_file.save()
assoc = DownloaderJobOriginalFileAssociation()
assoc.original_file = og_file
assoc.downloader_job = job
assoc.save()
return job
def create_processor_job(pipeline="AFFY_TO_PCL",
ram_amount=2048,
start_time=None):
og_file_1 = OriginalFile()
og_file_1.source_filename = "doesn't matter"
og_file_1.filename = "this either"
og_file_1.absolute_file_path = "nor this"
og_file_1.save()
og_file_2 = OriginalFile()
og_file_2.source_filename = "doesn't matter"
og_file_2.filename = "this either"
og_file_2.absolute_file_path = "nor this"
og_file_2.save()
downloader_job = None
if pipeline == "AFFY_TO_PCL":
downloader_job = DownloaderJob(
downloader_task="SRA",
batch_job_id="DEFAULT",
num_retries=0,
accession_code="NUNYA",
success=None,
)
downloader_job.save()
assoc = DownloaderJobOriginalFileAssociation()
assoc.original_file = og_file_2
assoc.downloader_job = downloader_job
assoc.save()
assoc1 = DownloaderJobOriginalFileAssociation()
assoc1.original_file = og_file_1
assoc1.downloader_job = downloader_job
assoc1.save()
processor_job = ProcessorJob(
downloader_job=downloader_job,
pipeline_applied=pipeline,
batch_job_id="PROCESSOR/dispatch-1528945054-e8eaf540",
ram_amount=ram_amount,
num_retries=0,
success=None,
start_time=start_time,
)
processor_job.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file_1
assoc1.processor_job = processor_job
assoc1.save()
assoc = ProcessorJobOriginalFileAssociation()
assoc.original_file = og_file_2
assoc.processor_job = processor_job
assoc.save()
return processor_job
def test_salmon_quant_one_sample_double_reads(self):
"""Test `salmon quant` on a sample that has double reads."""
# Set up organism index database objects.
prepare_organism_indices()
# Create an Experiment that includes two samples.
# (The first sample has test data available, but the second does not.)
experiment_accession = 'test_experiment'
experiment = Experiment.objects.create(accession_code=experiment_accession)
c_elegans = Organism.get_object_for_name("CAENORHABDITIS_ELEGANS")
# test_sample record
sample_accession = 'test_sample'
test_sample = Sample.objects.create(accession_code=sample_accession,
organism=c_elegans,
source_database='SRA',
technology='RNA-SEQ')
ExperimentSampleAssociation.objects.create(experiment=experiment, sample=test_sample)
# fake_sample record (created to prevent tximport step in this experiment)
fake_sample = Sample.objects.create(accession_code='fake_sample',
source_database='SRA',
technology='RNA-SEQ')
ExperimentSampleAssociation.objects.create(experiment=experiment, sample=fake_sample)
experiment_dir = '/home/user/data_store/salmon_tests/test_experiment'
og_read_1 = OriginalFile()
og_read_1.absolute_file_path = os.path.join(experiment_dir, 'raw/reads_1.fastq')
og_read_1.filename = "reads_1.fastq"
og_read_1.save()
OriginalFileSampleAssociation.objects.create(original_file=og_read_1, sample=test_sample).save()
og_read_2 = OriginalFile()
og_read_2.absolute_file_path = os.path.join(experiment_dir, "raw/reads_2.fastq")
og_read_2.filename = "reads_1.fastq"
og_read_2.save()
OriginalFileSampleAssociation.objects.create(original_file=og_read_2, sample=test_sample).save()
sample_dir = os.path.join(experiment_dir, 'test_sample')
job_context = salmon._prepare_files({"job_dir_prefix": "TEST",
"job_id": "TEST",
"job": ProcessorJob(),
'pipeline': Pipeline(name="Salmon"),
'computed_files': [],
"original_files": [og_read_1, og_read_2]})
# Run salmon.
self.check_salmon_quant(job_context, sample_dir)
# Confirm that this experiment is not ready for tximport yet,
# because `salmon quant` is not run on 'fake_sample'.
experiments_ready = salmon.get_tximport_inputs(job_context)['tximport_inputs']
self.assertEqual(len(experiments_ready), 0)
def prepare_job():
pj = ProcessorJob()
pj.pipeline_applied = "SALMON"
pj.save()
c_elegans = Organism.get_object_for_name("CAENORHABDITIS_ELEGANS")
samp = Sample()
samp.accession_code = "SALMON" # So the test files go to the right place
samp.organism = c_elegans
samp.source_database = 'SRA'
samp.technology = 'RNA-SEQ'
samp.save()
prepare_organism_indices()
og_file = OriginalFile()
og_file.source_filename = "ERR1562482_1.fastq.gz"
og_file.filename = "ERR1562482_1.fastq.gz"
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/SALMON/ERR1562482_1.fastq.gz"
og_file.is_downloaded = True
og_file.save()
og_file2 = OriginalFile()
og_file2.source_filename = "ERR1562482_2.fastq.gz"
og_file2.filename = "ERR1562482_2.fastq.gz"
og_file2.absolute_file_path = "/home/user/data_store/raw/TEST/SALMON/ERR1562482_2.fastq.gz"
og_file2.is_downloaded = True
og_file2.save()
og_file_samp_assoc = OriginalFileSampleAssociation()
og_file_samp_assoc.original_file = og_file
og_file_samp_assoc.sample = samp
og_file_samp_assoc.save()
og_file_samp_assoc2 = OriginalFileSampleAssociation()
og_file_samp_assoc2.original_file = og_file2
og_file_samp_assoc2.sample = samp
og_file_samp_assoc2.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file2
assoc1.processor_job = pj
assoc1.save()
return pj, [og_file, og_file2]
def prepare_job(length):
pj = ProcessorJob()
pj.pipeline_applied = "TRANSCRIPTOME_INDEX_" + length.upper()
pj.save()
homo_sapiens = Organism.get_object_for_name("HOMO_SAPIENS",
taxonomy_id=1001)
samp = Sample()
samp.organism = homo_sapiens
samp.accession_code = "derp" + length
samp.save()
og_file = OriginalFile()
og_file.source_filename = "aegilops_tauschii_" + length + ".fa.gz"
og_file.filename = "aegilops_tauschii_" + length + ".fa.gz"
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/TRANSCRIPTOME_INDEX/AEGILOPS_TAUSCHII/aegilops_tauschii_short.fa.gz"
og_file.is_downloaded = True
og_file.save()
og_file2 = OriginalFile()
og_file2.source_filename = "aegilops_tauschii_" + length + ".gtf.gz"
og_file2.filename = "aegilops_tauschii_" + length + ".gtf.gz"
og_file2.absolute_file_path = "/home/user/data_store/raw/TEST/TRANSCRIPTOME_INDEX/AEGILOPS_TAUSCHII/aegilops_tauschii_short.gtf.gz"
og_file2.is_downloaded = True
og_file2.save()
og_file_samp_assoc = OriginalFileSampleAssociation()
og_file_samp_assoc.original_file = og_file
og_file_samp_assoc.sample = samp
og_file_samp_assoc.save()
og_file_samp_assoc2 = OriginalFileSampleAssociation()
og_file_samp_assoc2.original_file = og_file2
og_file_samp_assoc2.sample = samp
og_file_samp_assoc2.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
assoc2 = ProcessorJobOriginalFileAssociation()
assoc2.original_file = og_file2
assoc2.processor_job = pj
assoc2.save()
return pj
def prepare_job():
pj = ProcessorJob()
pj.pipeline_applied = "AFFY_TO_PCL"
pj.save()
original_file = OriginalFile()
original_file.source_filename = "ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/GEOD/E-GEOD-59071/E-GEOD-59071.raw.3.zip"
original_file.filename = "GSM1426071_CD_colon_active_1.CEL"
original_file.absolute_file_path = "/home/user/data_store/raw/TEST/CEL/GSM1426071_CD_colon_active_1.CEL"
original_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = original_file
assoc1.processor_job = pj
assoc1.save()
c_elegans = Organism.get_object_for_name("CAENORHABDITIS_ELEGANS")
sample = Sample()
sample.title = "Heyo"
sample.organism = c_elegans
sample.is_processed = False
sample.save()
ogsa = OriginalFileSampleAssociation()
ogsa.sample = sample
ogsa.original_file = original_file
ogsa.save()
return pj
def prepare_dotsra_job(filename="ERR1562482.sra"):
pj = ProcessorJob()
pj.pipeline_applied = "SALMON"
pj.id = random.randint(111, 999999)
pj.save()
c_elegans = Organism.get_object_for_name("CAENORHABDITIS_ELEGANS")
samp = Sample()
samp.accession_code = "SALMON" # So the test files go to the right place
samp.organism = c_elegans
samp.save()
prepare_organism_indices()
og_file = OriginalFile()
og_file.source_filename = filename
og_file.filename = filename
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/SALMON/" + filename
og_file.save()
og_file_samp_assoc = OriginalFileSampleAssociation()
og_file_samp_assoc.original_file = og_file
og_file_samp_assoc.sample = samp
og_file_samp_assoc.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
return pj, [og_file]
def _get_actual_file_if_queueable(
extracted_subfile: Dict,
original_file: OriginalFile,
samples: List[Sample]) -> OriginalFile:
"""Returns the actual file from the archive if it should be queued.
If the file has been processed or has an unstarted DownloaderJob,
None will be returned.
`extracted_subfile` should be a Dict containing metadata about the
file that was extracted from an archive.
`original_file` should be the file associated with the CURRENT
DownloaderJob.
`samples` are the samples that the actual file should be associated
with if it has to be created.
"""
# Check to see if we've made this original file before:
potential_existing_files = OriginalFile.objects.filter(
source_filename=original_file.source_filename,
filename=extracted_subfile['filename'],
is_archive=False
)
if potential_existing_files.count() > 0:
# We've already created this record, let's see if we actually
# needed to download it or if we just got it because we needed
# a file in the same archive.
actual_file = potential_existing_files[0]
if actual_file.needs_processing():
if not actual_file.is_downloaded:
actual_file.is_downloaded = True
actual_file.save()
return actual_file
else:
return None
else:
actual_file = OriginalFile()
actual_file.is_downloaded = True
actual_file.is_archive = False
actual_file.absolute_file_path = extracted_subfile['absolute_path']
actual_file.filename = extracted_subfile['filename']
actual_file.calculate_size()
actual_file.calculate_sha1()
actual_file.has_raw = True
actual_file.source_url = original_file.source_url
actual_file.source_filename = original_file.source_filename
actual_file.save()
for sample in samples:
original_file_sample_association = OriginalFileSampleAssociation()
original_file_sample_association.sample = sample
original_file_sample_association.original_file = actual_file
original_file_sample_association.save()
return actual_file
def prepare_illumina_job(organism):
pj = ProcessorJob()
pj.pipeline_applied = "ILLUMINA_TO_PCL"
pj.save()
og_file = OriginalFile()
og_file.source_filename = "ftp://ftp.ncbi.nlm.nih.gov/geo/series/GSE22nnn/GSE22427/suppl/GSE22427%5Fnon%2Dnormalized%2Etxt.gz"
og_file.filename = "GSE22427_non-normalized.txt"
og_file.absolute_file_path = (
"/home/user/data_store/raw/TEST/ILLUMINA/GSE22427_non-normalized.txt")
og_file.is_downloaded = True
og_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
sample_names = [
"LV-C&si-Control-1",
"LV-C&si-Control-2",
"LV-C&si-Control-3",
"LV-C&si-EZH2-1",
"LV-C&si-EZH2-2",
"LV-C&si-EZH2-3",
"LV-EZH2&si-EZH2-1",
"LV-EZH2&si-EZH2-2",
"LV-EZH2&si-EZH2-3",
"LV-T350A&si-EZH2-1",
"LV-T350A&si-EZH2-2",
"LV-T350A&si-EZH2-3",
]
for name in sample_names:
sample = Sample()
sample.accession_code = name
sample.title = name
sample.organism = organism
sample.save()
sa = SampleAnnotation()
sa.sample = sample
sa.data = {"description": [name]}
sa.is_ccdl = False
sa.save()
sample_assoc = OriginalFileSampleAssociation()
sample_assoc.original_file = og_file
sample_assoc.sample = sample
sample_assoc.save()
sample = Sample.objects.get(title="LV-T350A&si-EZH2-3")
sample.title = "ignoreme_for_description"
sample.accession_code = "ignoreme_for_description"
sample.save()
return pj
def _make_original_file_with_contents(contents: str) -> OriginalFile:
_, path = tempfile.mkstemp(suffix=".txt")
with open(path, "w") as f:
f.write(contents)
og_file = OriginalFile()
og_file.source_filename = path
og_file.filename = os.path.basename(path)
og_file.absolute_file_path = os.path.realpath(path)
og_file.is_downloaded = True
og_file.save()
return og_file
def prepare_illumina_job(job_info: Dict) -> ProcessorJob:
pj = ProcessorJob()
pj.pipeline_applied = "ILLUMINA_TO_PCL"
pj.save()
og_file = OriginalFile()
og_file.source_filename = job_info["source_filename"]
og_file.filename = job_info["filename"]
og_file.absolute_file_path = job_info["absolute_file_path"]
og_file.is_downloaded = True
og_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
for s in job_info["samples"]:
# For convenience, if you give a list of strings we'll just use the
# strings as both titles and accessions.
annotation = None
if type(s) == str:
accession_code = s
title = s
elif type(s) == tuple and list(map(type, s)) == [str, str]:
accession_code, title = s
elif type(s) == tuple and list(map(type, s)) == [str, str, dict]:
accession_code, title, annotation = s
else:
raise ValueError(f"Invalid sample type for sample {s}")
sample = Sample()
sample.accession_code = accession_code
sample.title = title
sample.organism = job_info["organism"]
sample.save()
sa = SampleAnnotation()
sa.sample = sample
sa.data = annotation if annotation is not None else {
"description": [title]
}
sa.is_ccdl = False
sa.save()
sample_assoc = OriginalFileSampleAssociation()
sample_assoc.original_file = og_file
sample_assoc.sample = sample
sample_assoc.save()
return pj
def test_convert_processed_illumina(self):
job = ProcessorJob()
job.pipeline_applied = "NO_OP"
job.save()
# ex:
# Reporter Identifier VALUE Detection Pval
# ILMN_1343291 14.943602 0
# ILMN_1343295 13.528082 0
og_file = OriginalFile()
og_file.source_filename = "https://www.ebi.ac.uk/arrayexpress/experiments/E-GEOD-22433/"
og_file.filename = "GSM557500_sample_table.txt"
og_file.absolute_file_path = (
"/home/user/data_store/raw/TEST/NO_OP/GSM557500_sample_table.txt")
og_file.is_downloaded = True
og_file.save()
homo_sapiens = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
homo_sapiens.save()
sample = Sample()
sample.accession_code = "GSM557500"
sample.title = "GSM557500"
sample.platform_accession_code = "A-MEXP-1171"
sample.manufacturer = "ILLUMINA"
sample.organism = homo_sapiens
sample.save()
assoc = OriginalFileSampleAssociation()
assoc.original_file = og_file
assoc.sample = sample
assoc.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = job
assoc1.save()
# To:
# ENSG00000156508 14.943602
# ENSG00000111640 13.528082
final_context = no_op.no_op_processor(job.pk)
self.assertTrue(final_context["success"])
self.assertTrue(os.path.exists(final_context["output_file_path"]))
self.assertEqual(os.path.getsize(final_context["output_file_path"]),
920374)
self.assertTrue(
no_op.check_output_quality(final_context["output_file_path"]))
def test_convert_illumina_no_header(self):
job = ProcessorJob()
job.pipeline_applied = "NO_OP"
job.save()
# ex:
# ILMN_1885639 10.0000 0.7931
# ILMN_2209417 10.0000 0.2029
# ILMN_1765401 152.0873 0.0000
og_file = OriginalFile()
og_file.source_filename = (
"https://github.com/AlexsLemonade/refinebio/files/2255178/GSM1089291-tbl-1.txt"
)
og_file.filename = "GSM1089291-tbl-1.txt"
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/NO_OP/GSM1089291-tbl-1.txt"
og_file.is_downloaded = True
og_file.save()
homo_sapiens = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
homo_sapiens.save()
sample = Sample()
sample.accession_code = "GSM557500"
sample.title = "GSM557500"
sample.platform_accession_code = "A-MEXP-1171"
sample.manufacturer = "ILLUMINA"
sample.organism = homo_sapiens
sample.save()
assoc = OriginalFileSampleAssociation()
assoc.original_file = og_file
assoc.sample = sample
assoc.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = job
assoc1.save()
# To:
# ENSG00000105675 10
# ENSG00000085721 152.0873
# ENSG00000278494 152.0873
final_context = no_op.no_op_processor(job.pk)
self.assertTrue(final_context["success"])
self.assertTrue(os.path.exists(final_context["output_file_path"]))
self.assertEqual(os.path.getsize(final_context["output_file_path"]),
786207)
def test_good_detection(self):
"""GSE54661 appears to be mislabled (illuminaHumanv4) on GEO. Shows our detector works. """
from data_refinery_workers.processors import illumina
pj = ProcessorJob()
pj.pipeline_applied = "ILLUMINA_TO_PCL"
pj.save()
og_file = OriginalFile()
og_file.source_filename = "ftp://ftp.ncbi.nlm.nih.gov/geo/series/GSE54nnn/GSE54661/suppl/GSE54661%5Fnon%5Fnormalized%2Etxt%2Egz"
og_file.filename = "GSE54661_non_normalized.txt"
og_file.absolute_file_path = (
"/home/user/data_store/raw/TEST/ILLUMINA/GSE54661_non_normalized.txt"
)
og_file.is_downloaded = True
og_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
organism = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
organism.save()
sample = Sample()
sample.accession_code = "ABCD-1234"
sample.title = "hypoxia_Signal"
sample.organism = organism
sample.save()
sample_assoc = OriginalFileSampleAssociation()
sample_assoc.original_file = og_file
sample_assoc.sample = sample
sample_assoc.save()
final_context = illumina.illumina_to_pcl(pj.pk)
self.assertEqual(final_context["platform"], "illuminaHumanv3")
for key in final_context["samples"][0].sampleannotation_set.all(
)[0].data.keys():
self.assertTrue(key in [
"detected_platform", "detection_percentage",
"mapped_percentage"
])
# Cleanup after the job since it won't since we aren't running in cloud.
shutil.rmtree(final_context["work_dir"], ignore_errors=True)
def test_convert_illumina_bad_cols(self):
"""
In future, this test may be deprecated. For now it just alerts that it needs attention.
"""
job = ProcessorJob()
job.pipeline_applied = "NO_OP"
job.save()
# ex:
# ILMN_1885639 10.0000 0.7931 11.0000 0.123
# ILMN_2209417 10.0000 0.2029 11.1234 0.543
# LMN_1765401 152.0873 0.0000 99.999 0.19
og_file = OriginalFile()
og_file.source_filename = (
"https://github.com/AlexsLemonade/refinebio/files/2255178/GSM1089291-tbl-1-modified.txt"
)
og_file.filename = "GSM1089291-tbl-1-modified.txt"
og_file.absolute_file_path = (
"/home/user/data_store/raw/TEST/NO_OP/GSM1089291-tbl-1-modified.txt"
)
og_file.is_downloaded = True
og_file.save()
homo_sapiens = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
homo_sapiens.save()
sample = Sample()
sample.accession_code = "GSM557500"
sample.title = "GSM557500"
sample.platform_accession_code = "A-MEXP-1171"
sample.manufacturer = "ILLUMINA"
sample.organism = homo_sapiens
sample.save()
assoc = OriginalFileSampleAssociation()
assoc.original_file = og_file
assoc.sample = sample
assoc.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = job
assoc1.save()
final_context = no_op.no_op_processor(job.pk)
self.assertFalse(final_context["success"])
self.assertTrue("Tell Rich!" in final_context["job"].failure_reason)
def prepare_agilent_twocolor_job():
pj = ProcessorJob()
pj.pipeline_applied = "AGILENT_TWOCOLOR_TO_PCL"
pj.save()
og_file = OriginalFile()
og_file.source_filename = "https://www.ncbi.nlm.nih.gov/geo/download/?acc=GSE22900&format=file"
og_file.filename = "GSM466597_95899_agilent.txt"
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/AGILENT_TWOCOLOR/GSM466597_95899_agilent.txt"
og_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
return pj
def prepare_non_ba_job():
pj = ProcessorJob()
pj.pipeline_applied = "AFFY_TO_PCL"
pj.save()
og_file = OriginalFile()
og_file.source_filename = "ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM45nnn/GSM45588/suppl/GSM45588.CEL.gz"
og_file.filename = "GSM45588.CEL"
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/CEL/GSM45588.CEL"
og_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
return pj
def prepare_ba_job():
pj = ProcessorJob()
pj.pipeline_applied = "AFFY_TO_PCL"
pj.save()
og_file = OriginalFile()
og_file.source_filename = "ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/GEOD/E-GEOD-59071/E-GEOD-59071.raw.3.zip"
og_file.filename = "GSM1426071_CD_colon_active_1.CEL"
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/CEL/GSM1426071_CD_colon_active_1.CEL"
og_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
return pj
def prepare_job(job_info: dict) -> ProcessorJob:
job = ProcessorJob()
job.pipeline_applied = "NO_OP"
job.save()
og_file = OriginalFile()
og_file.source_filename = job_info["source_filename"]
og_file.filename = job_info["filename"]
og_file.absolute_file_path = job_info["absolute_file_path"]
og_file.is_downloaded = True
og_file.save()
sample = Sample()
sample.accession_code = job_info["accession_code"]
sample.title = job_info["accession_code"]
sample.platform_accession_code = job_info["platform_accession_code"]
manufacturer = job_info.get("manufacturer", None)
if manufacturer is not None:
sample.manufacturer = manufacturer
# The illumina samples need the human organism
if manufacturer == "ILLUMINA":
homo_sapiens = Organism(name="HOMO_SAPIENS",
taxonomy_id=9606,
is_scientific_name=True)
homo_sapiens.save()
sample.organism = homo_sapiens
sample.save()
assoc = OriginalFileSampleAssociation()
assoc.original_file = og_file
assoc.sample = sample
assoc.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = job
assoc1.save()
return job
def prepare_huex_v1_job():
pj = ProcessorJob()
pj.pipeline_applied = "AFFY_TO_PCL"
pj.save()
og_file = OriginalFile()
og_file.source_filename = "ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM1364nnn/GSM1364667/suppl/GSM1364667_U_110208_7-02-10_S2.CEL.gz"
og_file.filename = "GSM1364667_U_110208_7-02-10_S2.CEL"
og_file.absolute_file_path = (
"/home/user/data_store/raw/TEST/CEL/GSM1364667_U_110208_7-02-10_S2.CEL"
)
og_file.is_downloaded = True
og_file.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = pj
assoc1.save()
return pj
def test_salmon_quant_two_samples_single_read(self):
"""Test `salmon quant` outputs on two samples that have single
read and that belong to same experiment.
"""
prepare_organism_indices()
# Create one experiment and two related samples, based on:
# https://www.ncbi.nlm.nih.gov/sra/?term=SRP040623
# (For testing purpose, only two of the four samples' data are included.)
experiment_accession = 'PRJNA242809'
experiment = Experiment.objects.create(accession_code=experiment_accession)
c_elegans = Organism.get_object_for_name("CAENORHABDITIS_ELEGANS")
## Sample 1
sample1_accession = 'SRR1206053'
sample1 = Sample.objects.create(accession_code=sample1_accession,
organism=c_elegans)
ExperimentSampleAssociation.objects.create(experiment=experiment, sample=sample1)
experiment_dir = "/home/user/data_store/salmon_tests/PRJNA242809"
og_file_1 = OriginalFile()
og_file_1.absolute_file_path = os.path.join(experiment_dir, "raw/SRR1206053.fastq.gz")
og_file_1.filename = "SRR1206053.fastq.gz"
og_file_1.save()
OriginalFileSampleAssociation.objects.create(original_file=og_file_1, sample=sample1).save()
## Sample 2
sample2_accession = 'SRR1206054'
sample2 = Sample.objects.create(accession_code=sample2_accession,
organism=c_elegans)
ExperimentSampleAssociation.objects.create(experiment=experiment, sample=sample2)
og_file_2 = OriginalFile()
og_file_2.absolute_file_path = os.path.join(experiment_dir, "raw/SRR1206054.fastq.gz")
og_file_2.filename = "SRR1206054.fastq.gz"
og_file_2.save()
OriginalFileSampleAssociation.objects.create(original_file=og_file_2, sample=sample2).save()
# Test `salmon quant` on sample1 (SRR1206053)
sample1_dir = os.path.join(experiment_dir, sample1_accession)
job1_context = salmon._prepare_files({"job_dir_prefix": "TEST",
"job_id": "TEST",
'pipeline': Pipeline(name="Salmon"),
'computed_files': [],
"original_files": [og_file_1]})
# Check quant.sf in `salmon quant` output dir of sample1
self.check_salmon_quant(job1_context, sample1_dir)
# Confirm that this experiment is not ready for tximport yet.
experiments_ready = salmon._get_tximport_inputs(job1_context)
self.assertEqual(len(experiments_ready), 0)
# This job should not have produced any tximport output
# because the other sample isn't ready yet.
self.assertFalse(os.path.exists(os.path.join(job1_context["work_dir"], 'txi_out.RDS')))
# Now run `salmon quant` on sample2 (SRR1206054) too
sample2_dir = os.path.join(experiment_dir, sample2_accession)
job2_context = salmon._prepare_files({"job_dir_prefix": "TEST2",
"job_id": "TEST2",
'pipeline': Pipeline(name="Salmon"),
'computed_files': [],
"original_files": [og_file_2]})
# Clean up tximport output:
rds_filename = os.path.join(job2_context["work_dir"], 'txi_out.RDS')
if (os.path.isfile(rds_filename)):
os.remove(rds_filename)
# Check quant.sf in `salmon quant` output dir of sample2
self.check_salmon_quant(job2_context, sample2_dir)
# rds_filename should have been generated by tximport at this point.
# Note: `tximport` step is launched by subprocess module in Python.
# If input "quant.sf" files are too large, we may have to wait for
# a few seconds before testing the existence of rds_filename.
self.assertTrue(os.path.exists(rds_filename))
for computed_file in job2_context['computed_files']:
if computed_file.filename[-4:] == '.RDS':
rds_file_path = computed_file.absolute_file_path
cmd_tokens = [
"/usr/bin/Rscript", "--vanilla",
"/home/user/data_refinery_workers/processors/test_tximport.R",
"--txi_out", rds_file_path,
"--gene2txmap", job2_context["genes_to_transcripts_path"]
]
tximport_test_result = subprocess.run(cmd_tokens, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if tximport_test_result.returncode != 0:
# If the exit code is not 0 then tximport failed so fail the tests.
self.assertTrue(False)
# Check the individual files
self.assertTrue(len(job2_context['individual_files']), 2)
for file in job2_context['individual_files']:
self.assertTrue(os.path.isfile(file.absolute_file_path))
def run_tximport_at_progress_point(complete_accessions: List[str], incomplete_accessions: List[str]) -> Dict:
"""Create an experiment and associated objects and run tximport on it.
Creates a sample for each accession contained in either input
list. The samples in complete_accessions will be simlulated as
already having salmon quant run on them. The samples in
incomplete_accessions won't.
"""
# Create the experiment
experiment_accession = 'SRP095529'
data_dir = '/home/user/data_store/salmon_tests/'
experiment_dir = data_dir + experiment_accession
experiment = Experiment.objects.create(accession_code=experiment_accession)
zebrafish = Organism.get_object_for_name("DANIO_RERIO")
# This is a lie, but this image doesn't have the dependencies for TRANSCRIPTOME_INDEX
computational_result_short = ComputationalResult(processor=utils.find_processor('SALMON_QUANT'))
computational_result_short.save()
organism_index = OrganismIndex()
organism_index.index_type = "TRANSCRIPTOME_SHORT"
organism_index.organism = zebrafish
organism_index.result = computational_result_short
organism_index.absolute_directory_path = "/home/user/data_store/salmon_tests/ZEBRAFISH_INDEX/SHORT"
organism_index.save()
comp_file = ComputedFile()
# This path will not be used because we already have the files extracted.
comp_file.absolute_file_path = "/home/user/data_store/salmon_tests/ZEBRAFISH_INDEX/SHORT/zebrafish_short.tar.gz"
comp_file.result = computational_result_short
comp_file.size_in_bytes=1337
comp_file.sha1="ABC"
comp_file.s3_key = "key"
comp_file.s3_bucket = "bucket"
comp_file.save()
for accession_code in incomplete_accessions:
last_sample = Sample.objects.create(
accession_code=accession_code,
organism=zebrafish,
source_database='SRA',
technology='RNA-SEQ'
)
ExperimentSampleAssociation.objects.create(experiment=experiment, sample=last_sample)
# Create tximport result and files
quant_processor = utils.find_processor("SALMON_QUANT")
tximport_processor = utils.find_processor("TXIMPORT")
# Create the already processed samples along with their
# ComputationalResults and ComputedFiles. They don't need
# original files for this test because we aren't going to run
# salmon quant on them.
for accession_code in complete_accessions:
sample = Sample.objects.create(
accession_code=accession_code,
organism=zebrafish,
source_database='SRA',
technology='RNA-SEQ'
)
ExperimentSampleAssociation.objects.create(experiment=experiment, sample=sample)
if accession_code == "SRR5125622":
current_sample = sample
# Create and associate quant result and files.
quant_result = ComputationalResult()
quant_result.is_ccdl = True
quant_result.processor = quant_processor
quant_result.save()
# In prod the filename pattern will involve the timestamp
# but here we're using the accession code so we can find
# the archive file for the current sample.
archive_filename = "result-" + accession_code + ".tar.gz"
archive_file = ComputedFile()
archive_file.filename = archive_filename
archive_file.absolute_file_path = os.path.join(experiment_dir, archive_filename)
archive_file.is_public = False
archive_file.is_smashable = False
archive_file.is_qc = False
archive_file.result = quant_result
archive_file.size_in_bytes = 12345
archive_file.save()
quant_file = ComputedFile()
quant_file.filename = "quant.sf"
quant_file.absolute_file_path = experiment_dir + "/quant_files/" + accession_code + "_output/quant.sf"
quant_file.is_public = False
quant_file.is_smashable = False
quant_file.is_qc = False
quant_file.result = quant_result
quant_file.size_in_bytes = 12345
quant_file.s3_bucket = "bucket"
quant_file.s3_key = "key"
quant_file.save()
SampleResultAssociation.objects.get_or_create(
sample=sample,
result=quant_result
)
# Processor jobs need at least one original file associated with
# them so they know what they're processing.
current_og = OriginalFile()
current_og.absolute_file_path = os.path.join(experiment_dir, 'SRR5125622.fastq.gz')
current_og.filename = "SRR5125622.fastq.gz"
current_og.save()
OriginalFileSampleAssociation.objects.create(original_file=current_og, sample=current_sample).save()
pj = ProcessorJob()
pj.pipeline_applied = "TXIMPORT"
pj.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = current_og
assoc1.processor_job = pj
assoc1.save()
# Prep our job context
job_context = tximport._prepare_files({"job_dir_prefix": "TEST3",
"job_id": "TEST3",
"job": pj,
"index_directory": organism_index.absolute_directory_path,
"pipeline": Pipeline(name="Salmon"),
"computed_files": [],
"original_files": [current_og]})
# We don't have the raw file to run _determine_index_length so
# just pick one, it doesn't matter that much because we aren't
# checking the output data.
job_context["index_length"] = "short"
job_context = salmon._find_or_download_index(job_context)
job_context = salmon.get_tximport_inputs(job_context)
job_context = salmon.tximport(job_context)
return job_context
def download_geo(job_id: int) -> None:
"""The main function for the GEO Downloader.
Downloads a single tar file containing the files representing
samples relating to a single experiment stored in
GEO.
"""
job = utils.start_job(job_id)
accession_code = job.accession_code
original_file = job.original_files.first()
if not original_file:
job.failure_reason = "No files associated with the job."
logger.error("No files associated with the job.", downloader_job=job_id)
utils.end_downloader_job(job, success=False)
return
url = original_file.source_url
related_samples = original_file.samples.exclude(technology="RNA-SEQ")
# First, download the sample archive URL.
# Then, unpack all the ones downloaded.
# Then create processor jobs!
# The files for all of the samples are contained within the same zip file. Therefore only
# download the one.
os.makedirs(LOCAL_ROOT_DIR + "/" + accession_code, exist_ok=True)
dl_file_path = LOCAL_ROOT_DIR + "/" + accession_code + "/" + url.split("/")[-1]
logger.debug("Starting to download: " + url, job_id=job_id, accession_code=accession_code)
_download_file(url, dl_file_path, job)
original_file.absolute_file_path = dl_file_path
original_file.is_downloaded = True
original_file.save()
unpacked_sample_files = []
try:
# enumerate all files inside the archive
archived_files = list(ArchivedFile(dl_file_path).get_files())
except FileExtractionError as e:
job.failure_reason = e
logger.exception(
"Error occurred while extracting file.", path=dl_file_path, exception=str(e)
)
utils.end_downloader_job(job, success=False)
return
for og_file in archived_files:
sample = og_file.get_sample()
# We don't want RNA-Seq data from GEO:
# https://github.com/AlexsLemonade/refinebio/issues/966
if sample and sample.technology == "RNA-SEQ":
logger.warn("RNA-Seq sample found in GEO downloader job.", sample=sample)
continue
if not sample and (
not og_file.is_processable() or og_file.experiment_accession_code() != accession_code
):
# skip the files that we know are not processable and can't be associated with a sample
# also skip the files were we couldn't find a sample and they don't mention the current experiment
continue
potential_existing_file = OriginalFile.objects.filter(
source_filename=original_file.source_filename,
filename=og_file.filename,
is_archive=False,
).first()
if potential_existing_file:
# We've already created this record, let's see if we actually
# needed to download it or if we just got it because we needed
# a file in the same archive.
if potential_existing_file.needs_processing():
if not potential_existing_file.is_downloaded:
potential_existing_file.is_downloaded = True
potential_existing_file.save()
unpacked_sample_files.append(potential_existing_file)
continue
# Then this is a new file and we should create an original file for it
actual_file = OriginalFile()
actual_file.is_downloaded = True
actual_file.is_archive = False
actual_file.absolute_file_path = og_file.file_path
actual_file.filename = og_file.filename
actual_file.calculate_size()
actual_file.calculate_sha1()
actual_file.has_raw = True
actual_file.source_url = original_file.source_url
actual_file.source_filename = original_file.source_filename
actual_file.save()
# try to see if the file should be associated with a sample
if sample:
original_file_sample_association = OriginalFileSampleAssociation()
original_file_sample_association.sample = sample
original_file_sample_association.original_file = actual_file
original_file_sample_association.save()
else:
# if not, we can associate this file with all samples in the experiment
for sample in related_samples:
original_file_sample_association = OriginalFileSampleAssociation()
original_file_sample_association.sample = sample
original_file_sample_association.original_file = actual_file
original_file_sample_association.save()
unpacked_sample_files.append(actual_file)
if len(unpacked_sample_files) > 0:
success = True
logger.debug(
"File downloaded and extracted successfully.",
url=url,
dl_file_path=dl_file_path,
downloader_job=job_id,
)
else:
success = False
logger.info(
"Unable to extract any files.",
url=url,
dl_file_path=dl_file_path,
downloader_job=job_id,
)
job.failure_reason = "Failed to extract any downloaded files."
if success:
create_processor_jobs_for_original_files(unpacked_sample_files, job)
if original_file.is_archive:
original_file.delete_local_file()
utils.end_downloader_job(job, success)
return success
def download_geo(job_id: int) -> None:
"""The main function for the GEO Downloader.
Downloads a single tar file containing the files representing
samples relating to a single experiement stored in
GEO.
"""
job = utils.start_job(job_id)
file_assocs = DownloaderJobOriginalFileAssociation.objects.filter(
downloader_job=job)
original_file = file_assocs[0].original_file
url = original_file.source_url
accession_code = job.accession_code
sample_assocs = OriginalFileSampleAssociation.objects.filter(
original_file=original_file)
related_samples = Sample.objects.filter(
id__in=sample_assocs.values('sample_id'))
# First, download the sample archive URL.
# Then, unpack all the ones downloaded.
# Then create processor jobs!
# The files for all of the samples are
# contained within the same zip file. Therefore only
# download the one.
os.makedirs(LOCAL_ROOT_DIR + '/' + accession_code, exist_ok=True)
dl_file_path = LOCAL_ROOT_DIR + '/' + accession_code + '/' + url.split(
'/')[-1]
logger.debug("Starting to download: " + url,
job_id=job_id,
accession_code=accession_code)
_download_file(url, dl_file_path, job)
original_file.absolute_file_path = dl_file_path
original_file.is_downloaded = True
original_file.save()
has_raw = True
unpacked_sample_files = []
# These files are tarred, and also subsequently gzipped
if '.tar' in dl_file_path:
try:
extracted_files = _extract_tar(dl_file_path, accession_code)
except Exception as e:
job.failure_reason = e
logger.exception("Error occured while extracting tar file.",
path=dl_file_path,
exception=str(e))
utils.end_downloader_job(job, success=False)
return
for og_file in extracted_files:
filename = og_file['filename']
if '_' in filename:
sample_id = filename.split('_')[0]
else:
sample_id = filename.split('.')[0]
try:
sample = Sample.objects.get(accession_code=sample_id)
except Exception as e:
# We don't have this sample, but it's not a total failure. This happens.
continue
try:
# Files from the GEO supplemental file are gzipped inside of the tarball. Great!
archive_file = OriginalFile.objects.get(
source_filename__contains=sample_id)
archive_file.is_downloaded = True
archive_file.is_archive = True
archive_file.absolute_file_path = og_file['absolute_path']
archive_file.calculate_size()
archive_file.calculate_sha1()
archive_file.save()
if '.gz' in og_file['filename']:
extracted_subfile = _extract_gz(og_file['absolute_path'],
accession_code)
else:
extracted_subfile = [og_file]
actual_file = OriginalFile()
actual_file.is_downloaded = True
actual_file.is_archive = False
actual_file.absolute_file_path = extracted_subfile[0][
'absolute_path']
actual_file.filename = extracted_subfile[0]['filename']
actual_file.calculate_size()
actual_file.calculate_sha1()
actual_file.has_raw = True
actual_file.source_url = original_file.source_url
actual_file.source_filename = original_file.source_filename
actual_file.save()
original_file_sample_association = OriginalFileSampleAssociation(
)
original_file_sample_association.sample = sample
original_file_sample_association.original_file = actual_file
original_file_sample_association.save()
archive_file.delete_local_file()
archive_file.is_downloaded = False
archive_file.save()
unpacked_sample_files.append(actual_file)
except Exception as e:
# TODO - is this worth failing a job for?
logger.debug(
"Found a file we didn't have an OriginalFile for! Why did this happen?: "
+ og_file['filename'],
exc_info=1,
file=og_file['filename'],
sample_id=sample_id,
accession_code=accession_code)
# If we don't know why we have it, get rid of it.
os.remove(og_file["absolute_path"])
# This is a .tgz file.
elif '.tgz' in dl_file_path:
# If this is the MINiML file, it has been preprocessed
if '_family.xml.tgz' in dl_file_path:
has_raw = False
try:
extracted_files = _extract_tgz(dl_file_path, accession_code)
except Exception as e:
job.failure_reason = e
logger.exception("Error occured while extracting tgz file.",
path=dl_file_path,
exception=str(e))
utils.end_downloader_job(job, success=False)
return
for og_file in extracted_files:
if '.txt' in og_file['filename']:
try:
gsm_id = og_file['filename'].split('-')[0]
sample = Sample.objects.get(accession_code=gsm_id)
except Exception as e:
os.remove(og_file["absolute_path"])
continue
actual_file = OriginalFile()
actual_file.is_downloaded = True
actual_file.is_archive = False
actual_file.absolute_file_path = og_file['absolute_path']
actual_file.filename = og_file['filename']
actual_file.calculate_size()
actual_file.calculate_sha1()
actual_file.has_raw = has_raw
actual_file.source_url = original_file.source_url
actual_file.source_filename = original_file.source_filename
actual_file.save()
original_file_sample_association = OriginalFileSampleAssociation(
)
original_file_sample_association.sample = sample
original_file_sample_association.original_file = actual_file
original_file_sample_association.save()
unpacked_sample_files.append(actual_file)
# These files are only gzipped.
# These are generally the _actually_ raw (rather than the non-raw data in a RAW file) data
elif '.gz' in dl_file_path:
try:
extracted_files = _extract_gz(dl_file_path, accession_code)
except Exception as e:
job.failure_reason = e
logger.exception("Error occured while extracting gz file.",
path=dl_file_path,
exception=str(e))
utils.end_downloader_job(job, success=False)
return
for og_file in extracted_files:
filename = og_file['filename']
sample_id = filename.split('.')[0]
try:
# The archive we downloaded
archive_file = OriginalFile.objects.get(
source_filename__contains=filename)
archive_file.is_downloaded = True
archive_file.is_archive = True
archive_file.absolute_file_path = dl_file_path
archive_file.calculate_size()
archive_file.calculate_sha1()
archive_file.save()
actual_file = OriginalFile()
actual_file.is_downloaded = True
actual_file.is_archive = False
actual_file.absolute_file_path = og_file['absolute_path']
actual_file.filename = og_file['filename']
actual_file.calculate_size()
actual_file.calculate_sha1()
actual_file.has_raw = True
actual_file.source_url = original_file.source_url
actual_file.source_filename = original_file.source_filename
actual_file.save()
for sample in related_samples:
new_association = OriginalFileSampleAssociation()
new_association.original_file = actual_file
new_association.sample = sample
new_association.save()
archive_file.delete_local_file()
archive_file.is_downloaded = False
archive_file.save()
unpacked_sample_files.append(actual_file)
except Exception as e:
logger.debug(
"Found a file we didn't have an OriginalFile for! Why did this happen?: "
+ og_file['filename'],
exc_info=1,
file=og_file['filename'],
sample_id=sample_id,
accession_code=accession_code)
os.remove(og_file["absolute_path"])
# This is probably just a .txt file
else:
filename = dl_file_path.split('/')[-1]
sample_id = filename.split('_')[0]
actual_file = OriginalFile()
actual_file.is_downloaded = True
actual_file.is_archive = False
actual_file.absolute_file_path = dl_file_path
actual_file.filename = filename
actual_file.calculate_size()
actual_file.calculate_sha1()
actual_file.has_raw = True
actual_file.source_url = original_file.source_url
actual_file.source_filename = original_file.source_filename
actual_file.save()
for sample in related_samples:
new_association = OriginalFileSampleAssociation()
new_association.original_file = actual_file
new_association.sample = sample
new_association.save()
unpacked_sample_files.append(actual_file)
if len(unpacked_sample_files) > 0:
success = True
logger.debug("File downloaded and extracted successfully.",
url=url,
dl_file_path=dl_file_path,
downloader_job=job_id)
else:
success = False
logger.info("Unable to extract any files.",
url=url,
dl_file_path=dl_file_path,
downloader_job=job_id)
job.failure_reason = "Failed to extract any downloaded files."
if success:
utils.create_processor_jobs_for_original_files(unpacked_sample_files,
job)
if original_file.is_archive:
original_file.delete_local_file()
utils.end_downloader_job(job, success)
return success
def test_convert_simple_pcl(self):
""" """
job = ProcessorJob()
job.pipeline_applied = "NO_OP"
job.save()
# ID_REF, VALUE
og_file = OriginalFile()
og_file.source_filename = "https://www.ebi.ac.uk/arrayexpress/experiments/E-GEOD-51013/"
og_file.filename = "GSM1234847_sample_table.txt"
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/NO_OP/GSM1234847_sample_table.txt"
og_file.is_downloaded = True
og_file.save()
sample = Sample()
sample.accession_code = "GSM1234847"
sample.title = "GSM1234847"
sample.platform_accession_code = 'A-AFFY-38'
sample.save()
assoc = OriginalFileSampleAssociation()
assoc.original_file = og_file
assoc.sample = sample
assoc.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = job
assoc1.save()
final_context = no_op.no_op_processor(job.pk)
# No header - ex
# AFFX-BioB-3_at 0.74218756
og_file = OriginalFile()
og_file.source_filename = "ftp://ftp.ncbi.nlm.nih.gov/geo/series/GSE10nnn/GSE10188/miniml/GSE10188_family.xml.tgz"
og_file.filename = "GSM269747-tbl-1.txt"
og_file.absolute_file_path = "/home/user/data_store/raw/TEST/NO_OP/GSM269747-tbl-1.txt"
og_file.is_downloaded = True
og_file.save()
sample = Sample()
sample.accession_code = "GSM269747"
sample.title = "GSM269747"
sample.platform_accession_code = 'GPL1319'
sample.save()
assoc = OriginalFileSampleAssociation()
assoc.original_file = og_file
assoc.sample = sample
assoc.save()
job = ProcessorJob()
job.pipeline_applied = "NO_OP"
job.save()
assoc1 = ProcessorJobOriginalFileAssociation()
assoc1.original_file = og_file
assoc1.processor_job = job
assoc1.save()
final_context = no_op.no_op_processor(job.pk)
self.assertTrue(final_context['success'])
self.assertTrue(os.path.exists(final_context['output_file_path']))
self.assertEqual(os.path.getsize(final_context['output_file_path']),
346535)