def _replace_dotsra_with_fastq_files( sample: Sample, downloader_job: DownloaderJob, original_file: OriginalFile) -> List[OriginalFile]: """Replaces a .SRA file with two .fastq files. This function should only be called on a sample which has unmated reads, so it makes the assumption that the sample passed into it has at least two read files in ENA. """ read_one_url = _build_ena_file_url(sample.accession_code, "_1") read_two_url = _build_ena_file_url(sample.accession_code, "_2") # Technically this is a different file, but deleting this one and # its associations just to recreate another with the same # associations seems rather pointless. original_file.source_url = read_one_url original_file.source_filename = read_one_url.split("/")[-1] original_file.save() read_two_original_file = OriginalFile.objects.get_or_create( source_url=read_two_url, source_filename=read_two_url.split("/")[-1], has_raw=True)[0] OriginalFileSampleAssociation.objects.get_or_create( original_file=read_two_original_file, sample=sample) DownloaderJobOriginalFileAssociation.objects.get_or_create( original_file=read_two_original_file, downloader_job=downloader_job) return [original_file, read_two_original_file]
def _has_unmated_reads(accession_code: str, downloader_job: DownloaderJob) -> bool: """Checks if the SRA accession has unmated reads. Returns True if it does and False if it doesn't, and also whether or not it successfully connected to the ENA server""" full_ftp_link = _build_ena_file_url(accession_code) # Strip off the protocol code because we know it's FTP and the FTP # library doesn't want the protocol. no_protocol_link = full_ftp_link.split("://")[1] # We need to extract the server so we can login to it. split_link = no_protocol_link.split("/") ftp_server = split_link[0] # We need to get the FTP directory the file is in so we can check # how many other files are in it. Therefore we're looking to get # the path between the server and the filename itself. sample_directory = "/".join(split_link[1:-1]) # Try to connect to FTP ftp = FTP(ftp_server) ftp.login() ftp.cwd(sample_directory) try: # If there's three files then there's unmated reads, because # there's the one read file, the other read file, and the # unmated reads. if len(ftp.nlst()) == 3: return True else: return False except ftplib.all_errors: # If we can't find the sample on ENA's FTP server, then we # shouldn't try to download it from there. return False finally: if ftp: ftp.close() # Shouldn't reach here, but just in case default to NCBI. return False
def _generate_experiment_and_samples( self, run_accession: str, study_accession: str = None) -> (Experiment, List[Sample]): """Generates Experiments and Samples for the provided run_accession.""" metadata = SraSurveyor.gather_all_metadata(run_accession) if metadata == {}: if study_accession: logger.error( "Could not discover any metadata for run.", accession=run_accession, study_accession=study_accession, ) else: logger.error("Could not discover any metadata for run.", accession=run_accession) return (None, None) # This will cascade properly if DOWNLOAD_SOURCE == "ENA": if metadata["library_layout"] == "PAIRED": files_urls = [ _build_ena_file_url(run_accession, "_1"), _build_ena_file_url(run_accession, "_2"), ] else: files_urls = [_build_ena_file_url(run_accession)] else: files_urls = [SraSurveyor._build_ncbi_file_url(run_accession)] # Figure out the Organism for this sample organism_name = metadata.pop("organism_name", None) if not organism_name: logger.error("Could not discover organism type for run.", accession=run_accession) return (None, None) # This will cascade properly organism_name = organism_name.upper() organism = Organism.get_object_for_name(organism_name) ## # Experiment ## experiment_accession_code = metadata.get("study_accession") try: experiment_object = Experiment.objects.get( accession_code=experiment_accession_code) logger.debug( "Experiment already exists, skipping object creation.", experiment_accession_code=experiment_accession_code, survey_job=self.survey_job.id, ) except Experiment.DoesNotExist: experiment_object = Experiment() experiment_object.accession_code = experiment_accession_code SraSurveyor._apply_metadata_to_experiment(experiment_object, metadata) experiment_object.save() ## # Experiment Metadata ## json_xa = ExperimentAnnotation() json_xa.experiment = experiment_object json_xa.data = metadata json_xa.is_ccdl = False json_xa.save() ## # Samples ## sample_accession_code = metadata.pop("run_accession") # Create the sample object try: sample_object = Sample.objects.get( accession_code=sample_accession_code) # If current experiment includes new protocol information, # merge it into the sample's existing protocol_info. protocol_info, is_updated = self.update_sample_protocol_info( sample_object.protocol_info, experiment_object.protocol_description, experiment_object.source_url, ) if is_updated: sample_object.protocol_info = protocol_info sample_object.save() logger.debug( "Sample %s already exists, skipping object creation.", sample_accession_code, experiment_accession_code=experiment_object.accession_code, survey_job=self.survey_job.id, ) except Sample.DoesNotExist: sample_object = Sample() sample_object.source_database = "SRA" sample_object.accession_code = sample_accession_code sample_object.organism = organism sample_object.platform_name = metadata.get( "platform_instrument_model", "UNKNOWN") # The platform_name is human readable and contains spaces, # accession codes shouldn't have spaces though: sample_object.platform_accession_code = sample_object.platform_name.replace( " ", "") sample_object.technology = "RNA-SEQ" if ("ILLUMINA" in sample_object.platform_name.upper() or "NEXTSEQ" in sample_object.platform_name.upper()): sample_object.manufacturer = "ILLUMINA" elif "ION TORRENT" in sample_object.platform_name.upper(): sample_object.manufacturer = "ION_TORRENT" else: sample_object.manufacturer = "UNKNOWN" SraSurveyor._apply_harmonized_metadata_to_sample( sample_object, metadata) protocol_info, is_updated = self.update_sample_protocol_info( existing_protocols=[], experiment_protocol=experiment_object.protocol_description, experiment_url=experiment_object.source_url, ) # Do not check is_updated the first time because we must # save a list so we can append to it later. sample_object.protocol_info = protocol_info sample_object.save() for file_url in files_urls: original_file = OriginalFile.objects.get_or_create( source_url=file_url, source_filename=file_url.split("/")[-1], has_raw=True)[0] OriginalFileSampleAssociation.objects.get_or_create( original_file=original_file, sample=sample_object) # Create associations if they don't already exist ExperimentSampleAssociation.objects.get_or_create( experiment=experiment_object, sample=sample_object) ExperimentOrganismAssociation.objects.get_or_create( experiment=experiment_object, organism=organism) return experiment_object, [sample_object]