def create_experiment_and_samples_from_api(
self, experiment_accession_code) -> (Experiment, List[Sample]):
""" The main surveyor - find the Experiment and Samples from NCBI GEO.
Uses the GEOParse library, for which docs can be found here: https://geoparse.readthedocs.io/en/latest/usage.html#working-with-geo-objects
"""
# Cleaning up is tracked here: https://github.com/guma44/GEOparse/issues/41
gse = GEOparse.get_GEO(experiment_accession_code,
destdir=self.get_temp_path(),
how="brief",
silent=True)
preprocessed_samples = harmony.preprocess_geo(gse.gsms.items())
harmonized_samples = harmony.harmonize(preprocessed_samples)
# Create the experiment object
try:
experiment_object = Experiment.objects.get(
accession_code=experiment_accession_code)
logger.debug(
"Experiment %s already exists, skipping object creation.",
experiment_accession_code,
survey_job=self.survey_job.id)
except Experiment.DoesNotExist:
experiment_object = Experiment()
experiment_object.accession_code = experiment_accession_code
experiment_object.source_url = (
"https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=" +
experiment_accession_code)
experiment_object.source_database = "GEO"
experiment_object.title = gse.metadata.get('title', [''])[0]
experiment_object.description = gse.metadata.get('summary',
[''])[0]
# Source doesn't provide time information, assume midnight.
submission_date = gse.metadata["submission_date"][
0] + " 00:00:00 UTC"
experiment_object.source_first_published = dateutil.parser.parse(
submission_date)
last_updated_date = gse.metadata["last_update_date"][
0] + " 00:00:00 UTC"
experiment_object.source_last_updated = dateutil.parser.parse(
last_updated_date)
unique_institutions = list(set(gse.metadata["contact_institute"]))
experiment_object.submitter_institution = ", ".join(
unique_institutions)
experiment_object.pubmed_id = gse.metadata.get("pubmed_id",
[""])[0]
# Scrape publication title and authorship from Pubmed
if experiment_object.pubmed_id:
pubmed_metadata = utils.get_title_and_authors_for_pubmed_id(
experiment_object.pubmed_id)
experiment_object.publication_title = pubmed_metadata[0]
experiment_object.publication_authors = pubmed_metadata[1]
experiment_object.save()
experiment_annotation = ExperimentAnnotation()
experiment_annotation.data = gse.metadata
experiment_annotation.experiment = experiment_object
experiment_annotation.is_ccdl = False
experiment_annotation.save()
# Okay, here's the situation!
# Sometimes, samples have a direct single representation for themselves.
# Othertimes, there is a single file with references to every sample in it.
created_samples = []
for sample_accession_code, sample in gse.gsms.items():
try:
sample_object = Sample.objects.get(
accession_code=sample_accession_code)
logger.debug(
"Sample %s from experiment %s already exists, skipping object creation.",
sample_accession_code,
experiment_object.accession_code,
survey_job=self.survey_job.id)
# Associate it with the experiment, but since it
# already exists it already has original files
# associated with it and it's already been downloaded,
# so don't add it to created_samples.
ExperimentSampleAssociation.objects.get_or_create(
experiment=experiment_object, sample=sample_object)
ExperimentOrganismAssociation.objects.get_or_create(
experiment=experiment_object,
organism=sample_object.organism)
except Sample.DoesNotExist:
organism = Organism.get_object_for_name(
sample.metadata['organism_ch1'][0].upper())
sample_object = Sample()
sample_object.source_database = "GEO"
sample_object.accession_code = sample_accession_code
sample_object.organism = organism
# If data processing step, it isn't raw.
sample_object.has_raw = not sample.metadata.get(
'data_processing', None)
ExperimentOrganismAssociation.objects.get_or_create(
experiment=experiment_object, organism=organism)
sample_object.title = sample.metadata['title'][0]
self.set_platform_properties(sample_object, sample.metadata,
gse)
# Directly assign the harmonized properties
harmonized_sample = harmonized_samples[sample_object.title]
for key, value in harmonized_sample.items():
setattr(sample_object, key, value)
# Sample-level protocol_info
sample_object.protocol_info = self.get_sample_protocol_info(
sample.metadata, sample_accession_code)
sample_object.save()
logger.debug("Created Sample: " + str(sample_object))
sample_annotation = SampleAnnotation()
sample_annotation.sample = sample_object
sample_annotation.data = sample.metadata
sample_annotation.is_ccdl = False
sample_annotation.save()
sample_supplements = sample.metadata.get(
'supplementary_file', [])
for supplementary_file_url in sample_supplements:
# Why do they give us this?
if supplementary_file_url == "NONE":
break
# We never want these!
if "idat.gz" in supplementary_file_url.lower():
continue
if "chp.gz" in supplementary_file_url.lower():
continue
if "ndf.gz" in supplementary_file_url.lower():
continue
if "pos.gz" in supplementary_file_url.lower():
continue
if "pair.gz" in supplementary_file_url.lower():
continue
if "gff.gz" in supplementary_file_url.lower():
continue
# Sometimes, we are lied to about the data processing step.
lower_file_url = supplementary_file_url.lower()
if '.cel' in lower_file_url \
or ('_non_normalized.txt' in lower_file_url) \
or ('_non-normalized.txt' in lower_file_url) \
or ('-non-normalized.txt' in lower_file_url) \
or ('-non_normalized.txt' in lower_file_url):
sample_object.has_raw = True
sample_object.save()
# filename and source_filename are the same for these
filename = supplementary_file_url.split('/')[-1]
original_file = OriginalFile.objects.get_or_create(
source_url=supplementary_file_url,
filename=filename,
source_filename=filename,
has_raw=sample_object.has_raw,
is_archive=True)[0]
logger.debug("Created OriginalFile: " + str(original_file))
original_file_sample_association = OriginalFileSampleAssociation.objects.get_or_create(
original_file=original_file, sample=sample_object)
if original_file.is_affy_data():
# Only Affymetrix Microarrays produce .CEL files
sample_object.technology = 'MICROARRAY'
sample_object.manufacturer = 'AFFYMETRTIX'
sample_object.save()
# It's okay to survey RNA-Seq samples from GEO, but we
# don't actually want to download/process any RNA-Seq
# data unless it comes from SRA.
if sample_object.technology != 'RNA-SEQ':
created_samples.append(sample_object)
# Now that we've determined the technology at the
# sample level, we can set it at the experiment level,
# just gotta make sure to only do it once. There can
# be more than one technology, this should be changed
# as part of:
# https://github.com/AlexsLemonade/refinebio/issues/1099
if not experiment_object.technology:
experiment_object.technology = sample_object.technology
experiment_object.save()
ExperimentSampleAssociation.objects.get_or_create(
experiment=experiment_object, sample=sample_object)
# These supplementary files _may-or-may-not_ contain the type of raw data we can process.
for experiment_supplement_url in gse.metadata.get(
'supplementary_file', []):
# filename and source_filename are the same for these
filename = experiment_supplement_url.split('/')[-1]
original_file = OriginalFile.objects.get_or_create(
source_url=experiment_supplement_url,
filename=filename,
source_filename=filename,
has_raw=sample_object.has_raw,
is_archive=True)[0]
logger.debug("Created OriginalFile: " + str(original_file))
lower_supplement_url = experiment_supplement_url.lower()
if ('_non_normalized.txt' in lower_supplement_url) \
or ('_non-normalized.txt' in lower_supplement_url) \
or ('-non-normalized.txt' in lower_supplement_url) \
or ('-non_normalized.txt' in lower_supplement_url):
for sample_object in created_samples:
sample_object.has_raw = True
sample_object.save()
OriginalFileSampleAssociation.objects.get_or_create(
sample=sample_object, original_file=original_file)
# Delete this Original file if it isn't being used.
if OriginalFileSampleAssociation.objects.filter(
original_file=original_file).count() == 0:
original_file.delete()
# These are the Miniml/Soft/Matrix URLs that are always(?) provided.
# GEO describes different types of data formatting as "families"
family_url = self.get_miniml_url(experiment_accession_code)
miniml_original_file = OriginalFile.objects.get_or_create(
source_url=family_url,
source_filename=family_url.split('/')[-1],
has_raw=sample_object.has_raw,
is_archive=True)[0]
for sample_object in created_samples:
# We don't need a .txt if we have a .CEL
if sample_object.has_raw:
continue
OriginalFileSampleAssociation.objects.get_or_create(
sample=sample_object, original_file=miniml_original_file)
# Delete this Original file if it isn't being used.
if OriginalFileSampleAssociation.objects.filter(
original_file=miniml_original_file).count() == 0:
miniml_original_file.delete()
# Trash the temp path
try:
shutil.rmtree(self.get_temp_path())
except Exception:
# There was a problem during surveying so this didn't get created.
# It's not a big deal.
pass
return experiment_object, created_samples
def create_experiment_and_samples_from_api(
self, experiment_accession_code) -> (Experiment, List[Sample]):
""" The main surveyor - find the Experiment and Samples from NCBI GEO.
Uses the GEOParse library, for which docs can be found here: https://geoparse.readthedocs.io/en/latest/usage.html#working-with-geo-objects
"""
# Cleaning up is tracked here: https://github.com/guma44/GEOparse/issues/41
gse = GEOparse.get_GEO(experiment_accession_code,
destdir=self.get_temp_path(),
how="brief",
silent=True)
preprocessed_samples = harmony.preprocess_geo(gse.gsms.items())
harmonized_samples = harmony.harmonize(preprocessed_samples)
# Create the experiment object
try:
experiment_object = Experiment.objects.get(
accession_code=experiment_accession_code)
logger.debug(
"Experiment %s already exists, skipping object creation.",
experiment_accession_code,
survey_job=self.survey_job.id,
)
except Experiment.DoesNotExist:
experiment_object = Experiment()
experiment_object.accession_code = experiment_accession_code
GeoSurveyor._apply_metadata_to_experiment(experiment_object, gse)
experiment_object.save()
experiment_annotation = ExperimentAnnotation()
experiment_annotation.data = gse.metadata
experiment_annotation.experiment = experiment_object
experiment_annotation.is_ccdl = False
experiment_annotation.save()
# Okay, here's the situation!
# Sometimes, samples have a direct single representation for themselves.
# Othertimes, there is a single file with references to every sample in it.
created_samples = []
for sample_accession_code, sample in gse.gsms.items():
try:
sample_object = Sample.objects.get(
accession_code=sample_accession_code)
logger.debug(
"Sample %s from experiment %s already exists, skipping object creation.",
sample_accession_code,
experiment_object.accession_code,
survey_job=self.survey_job.id,
)
# Associate it with the experiment, but since it
# already exists it already has original files
# associated with it and it's already been downloaded,
# so don't add it to created_samples.
ExperimentSampleAssociation.objects.get_or_create(
experiment=experiment_object, sample=sample_object)
ExperimentOrganismAssociation.objects.get_or_create(
experiment=experiment_object,
organism=sample_object.organism)
except Sample.DoesNotExist:
organism = Organism.get_object_for_name(
sample.metadata["organism_ch1"][0].upper())
sample_object = Sample()
sample_object.source_database = "GEO"
sample_object.accession_code = sample_accession_code
sample_object.organism = organism
# If data processing step, it isn't raw.
sample_object.has_raw = not sample.metadata.get(
"data_processing", None)
ExperimentOrganismAssociation.objects.get_or_create(
experiment=experiment_object, organism=organism)
sample_object.title = sample.metadata["title"][0]
self.set_platform_properties(sample_object, sample.metadata,
gse)
GeoSurveyor._apply_harmonized_metadata_to_sample(
sample_object, harmonized_samples[sample_object.title])
# Sample-level protocol_info
sample_object.protocol_info = self.get_sample_protocol_info(
sample.metadata, sample_accession_code)
sample_object.save()
logger.debug("Created Sample: " + str(sample_object))
sample_annotation = SampleAnnotation()
sample_annotation.sample = sample_object
sample_annotation.data = sample.metadata
sample_annotation.is_ccdl = False
sample_annotation.save()
sample_supplements = sample.metadata.get(
"supplementary_file", [])
for supplementary_file_url in sample_supplements:
# Why do they give us this?
if supplementary_file_url == "NONE":
break
# We never want these!
if "idat.gz" in supplementary_file_url.lower():
continue
if "chp.gz" in supplementary_file_url.lower():
continue
if "ndf.gz" in supplementary_file_url.lower():
continue
if "pos.gz" in supplementary_file_url.lower():
continue
if "pair.gz" in supplementary_file_url.lower():
continue
if "gff.gz" in supplementary_file_url.lower():
continue
# Sometimes, we are lied to about the data processing step.
lower_file_url = supplementary_file_url.lower()
if (".cel" in lower_file_url
or ("_non_normalized.txt" in lower_file_url)
or ("_non-normalized.txt" in lower_file_url)
or ("-non-normalized.txt" in lower_file_url)
or ("-non_normalized.txt" in lower_file_url)):
sample_object.has_raw = True
sample_object.save()
# filename and source_filename are the same for these
filename = FileUtils.get_filename(supplementary_file_url)
original_file = OriginalFile.objects.get_or_create(
source_url=supplementary_file_url,
filename=filename,
source_filename=filename,
has_raw=sample_object.has_raw,
is_archive=FileUtils.is_archive(filename),
)[0]
logger.debug("Created OriginalFile: " + str(original_file))
original_file_sample_association = OriginalFileSampleAssociation.objects.get_or_create(
original_file=original_file, sample=sample_object)
if original_file.is_affy_data():
# Only Affymetrix Microarrays produce .CEL files
sample_object.technology = "MICROARRAY"
sample_object.manufacturer = "AFFYMETRIX"
sample_object.save()
# It's okay to survey RNA-Seq samples from GEO, but we
# don't actually want to download/process any RNA-Seq
# data unless it comes from SRA.
if sample_object.technology != "RNA-SEQ":
created_samples.append(sample_object)
# Now that we've determined the technology at the
# sample level, we can set it at the experiment level,
# just gotta make sure to only do it once. There can
# be more than one technology, this should be changed
# as part of:
# https://github.com/AlexsLemonade/refinebio/issues/1099
if not experiment_object.technology:
experiment_object.technology = sample_object.technology
experiment_object.save()
ExperimentSampleAssociation.objects.get_or_create(
experiment=experiment_object, sample=sample_object)
# These supplementary files _may-or-may-not_ contain the type of raw data we can process.
for experiment_supplement_url in gse.metadata.get(
"supplementary_file", []):
# filename and source_filename are the same for these
filename = experiment_supplement_url.split("/")[-1]
original_file = OriginalFile.objects.get_or_create(
source_url=experiment_supplement_url,
filename=filename,
source_filename=filename,
has_raw=sample_object.has_raw,
is_archive=True,
)[0]
logger.debug("Created OriginalFile: " + str(original_file))
lower_supplement_url = experiment_supplement_url.lower()
if (("_non_normalized.txt" in lower_supplement_url)
or ("_non-normalized.txt" in lower_supplement_url)
or ("-non-normalized.txt" in lower_supplement_url)
or ("-non_normalized.txt" in lower_supplement_url)):
for sample_object in created_samples:
sample_object.has_raw = True
sample_object.save()
OriginalFileSampleAssociation.objects.get_or_create(
sample=sample_object, original_file=original_file)
# Delete this Original file if it isn't being used.
if (OriginalFileSampleAssociation.objects.filter(
original_file=original_file).count() == 0):
original_file.delete()
# These are the Miniml/Soft/Matrix URLs that are always(?) provided.
# GEO describes different types of data formatting as "families"
family_url = self.get_miniml_url(experiment_accession_code)
miniml_original_file = OriginalFile.objects.get_or_create(
source_url=family_url,
source_filename=family_url.split("/")[-1],
has_raw=sample_object.has_raw,
is_archive=True,
)[0]
for sample_object in created_samples:
# We don't need a .txt if we have a .CEL
if sample_object.has_raw:
continue
OriginalFileSampleAssociation.objects.get_or_create(
sample=sample_object, original_file=miniml_original_file)
# Delete this Original file if it isn't being used.
if (OriginalFileSampleAssociation.objects.filter(
original_file=miniml_original_file).count() == 0):
miniml_original_file.delete()
# Trash the temp path
try:
shutil.rmtree(self.get_temp_path())
except Exception:
# There was a problem during surveying so this didn't get created.
# It's not a big deal.
pass
return experiment_object, created_samples
def handle(self, *args, **options):
"""Refreshes the metadata for all samples, or samples from a specific database
"""
possible_source_databases = ["ARRAY_EXPRESS", "GEO", "SRA"]
if options.get("source_database", None) is None:
samples = Sample.objects.all()
elif options["source_database"] in possible_source_databases:
source_database = options["source_database"]
samples = Sample.objects.filter(source_database=source_database)
else:
logger.error('Invalid source database "{}"'.format(
options["source_database"]) +
"\nPossible source databases: {}".format(", ".join(
possible_source_databases)))
sys.exit(1)
paginator = PerformantPaginator(samples, PAGE_SIZE)
page = paginator.page()
while True:
for sample in samples:
logger.debug("Refreshing metadata for a sample.",
sample=sample.accession_code)
if sample.source_database == "SRA":
metadata = SraSurveyor.gather_all_metadata(
sample.accession_code)
SraSurveyor._apply_harmonized_metadata_to_sample(
sample, metadata)
elif sample.source_database == "GEO":
gse = GEOparse.get_GEO(
sample.experiments.first().accession_code,
destdir="/tmp/management",
how="brief",
silent=True,
)
preprocessed_samples = harmony.preprocess_geo(
gse.gsms.items())
harmonized_samples = harmony.harmonize(
preprocessed_samples)
GeoSurveyor._apply_harmonized_metadata_to_sample(
sample, harmonized_samples[sample.title])
elif sample.source_database == "ARRAY_EXPRESS":
SDRF_URL_TEMPLATE = (
"https://www.ebi.ac.uk/arrayexpress/files/{code}/{code}.sdrf.txt"
)
sdrf_url = SDRF_URL_TEMPLATE.format(
code=sample.experiments.first().accession_code)
sdrf_samples = harmony.parse_sdrf(sdrf_url)
harmonized_samples = harmony.harmonize(sdrf_samples)
ArrayExpressSurveyor._apply_harmonized_metadata_to_sample(
sample, harmonized_samples[sample.title])
sample.save()
if not page.has_next():
break
else:
page = paginator.page(page.next_page_number())
# 2000 samples queued up every five minutes should be fast
# enough and also not thrash the DB.
time.sleep(60 * 5)