def test_get_normalized_platform(self):
""" Test a particular normaization we need to perform """
self.assertEqual(utils.get_normalized_platform("hugene10stv1"),
"hugene10st")
self.assertEqual(utils.get_normalized_platform("hugene10stv2"),
"hugene10st")
self.assertEqual(utils.get_normalized_platform("stv1hugene10"),
"stv1hugene10")
def set_platform_properties(self, sample_object: Sample,
sample_metadata: Dict,
gse: GEOparse.GSM) -> Sample:
"""Sets platform-related properties on `sample_object`.
Uses metadata from `gse` to populate platform_name,
platform_accession_code, and technology on `sample_object`.
"""
# Determine platform information
external_accession = get_normalized_platform(
gse.metadata.get("platform_id", [UNKNOWN])[0])
if external_accession == UNKNOWN:
sample_object.platform_accession_code = UNKNOWN
sample_object.platform_name = UNKNOWN
sample_object.manufacturer = UNKNOWN
# If this sample is Affy, we potentially can extract the
# platform information from the .CEL file. If it's not we
# can't do anything. Therefore assume the technology is
# microarray when we have no platform information.
sample_object.technology = "MICROARRAY"
return sample_object
platform_accession_code = UNKNOWN
gpl = GEOparse.get_GEO(external_accession,
destdir=self.get_temp_path(),
how="brief",
silent=True)
platform_title = gpl.metadata.get("title", [UNKNOWN])[0]
# Check if this is a supported microarray platform.
for platform in get_supported_microarray_platforms():
if platform["external_accession"] == external_accession:
platform_accession_code = platform["platform_accession"]
if platform_accession_code != UNKNOWN:
# It's a supported microarray platform.
# We are using the brain array package as the platform accession code,
# so, for instance, GPL3213 becomes 'chicken'.
sample_object.platform_accession_code = platform_accession_code
sample_object.technology = "MICROARRAY"
try:
# Related: https://github.com/AlexsLemonade/refinebio/issues/354
# If it's Affy we can get a readable name:
sample_object.platform_name = get_readable_affymetrix_names(
)[platform_accession_code]
sample_object.manufacturer = "AFFYMETRIX"
# Sometimes Affymetrix samples have weird channel
# protocol metadata, so if we find that it's
# Affymetrix return it now. Example: GSE113945
return sample_object
except KeyError:
# Otherwise we'll use what we've got.
sample_object.platform_name = platform_title
# Determine manufacturer
platform = sample_object.pretty_platform.upper()
if "AGILENT" in platform:
sample_object.manufacturer = "AGILENT"
elif "ILLUMINA" in platform or "NEXTSEQ" in platform:
sample_object.manufacturer = "ILLUMINA"
elif "AFFYMETRIX" in platform:
sample_object.manufacturer = "AFFYMETRIX"
else:
sample_object.manufacturer = UNKNOWN
return sample_object
# Check to see if this is a supported RNASeq technology:
# GEO RNASeq platform titles often have organisms appended to
# an otherwise recognizable platform. The list of supported
# RNASeq platforms isn't long, so see if any of them are
# contained within what GEO gave us.
# Example: GSE69572 has a platform title of:
# 'Illumina Genome Analyzer IIx (Glycine max)'
# Which should really just be 'Illumina Genome Analyzer IIx'
# because RNASeq platforms are organism agnostic. However,
# the platforms 'Illumina Genome Analyzer' and 'Illumina
# Genome Analyzer II' would also be matched, so make sure that
# the longest platform names are tested first:
sorted_platform_list = get_supported_rnaseq_platforms().copy()
sorted_platform_list.sort(key=len, reverse=True)
for platform in sorted_platform_list:
if platform.upper() in platform_title.upper():
sample_object.technology = "RNA-SEQ"
sample_object.platform_name = platform
# We just use RNASeq platform titles as accessions
sample_object.platform_accession_code = platform
if "ILLUMINA" in sample_object.platform_name.upper():
sample_object.manufacturer = "ILLUMINA"
elif "NEXTSEQ" in sample_object.platform_name.upper():
sample_object.manufacturer = "NEXTSEQ"
elif "ION TORRENT" in sample_object.platform_name.upper():
sample_object.manufacturer = "ION_TORRENT"
else:
sample_object.manufacturer = UNKNOWN
return sample_object
# If we've made it this far, we don't know what this platform
# is, therefore we can't know what its technology is. What we
# do know is what GEO said was it's platform's accession and
# title are, and that it's unsupported.
sample_object.platform_name = platform_title
sample_object.platform_accession_code = external_accession
sample_object.technology = UNKNOWN
sample_object.manufacturer = UNKNOWN
return sample_object
def create_experiment_from_api(
self, experiment_accession_code: str) -> (Experiment, Dict):
"""Given an experiment accession code, create an Experiment object.
Also returns a dictionary of additional information about the
platform discovered for the experiment.
Will raise an UnsupportedPlatformException if this experiment was
conducted using a platform which we don't support.
See an example at: https://www.ebi.ac.uk/arrayexpress/json/v3/experiments/E-MTAB-3050/sample
"""
request_url = EXPERIMENTS_URL + experiment_accession_code
experiment_request = utils.requests_retry_session().get(request_url,
timeout=60)
try:
parsed_json = experiment_request.json(
)["experiments"]["experiment"][0]
except KeyError:
logger.error("Remote experiment has no Experiment data!",
experiment_accession_code=experiment_accession_code,
survey_job=self.survey_job.id)
raise
experiment = {}
experiment["name"] = parsed_json["name"]
experiment["experiment_accession_code"] = experiment_accession_code
# This experiment has no platform at all, and is therefore useless.
if 'arraydesign' not in parsed_json or len(
parsed_json["arraydesign"]) == 0:
logger.warn("Remote experiment has no arraydesign listed.",
experiment_accession_code=experiment_accession_code,
survey_job=self.survey_job.id)
raise UnsupportedPlatformException
# If there is more than one arraydesign listed in the experiment
# then there is no other way to determine which array was used
# for which sample other than looking at the header of the CEL
# file. That obviously cannot happen until the CEL file has been
# downloaded so we can just mark it as UNKNOWN and let the
# downloader inspect the downloaded file to determine the
# array then.
elif len(parsed_json["arraydesign"]
) != 1 or "accession" not in parsed_json["arraydesign"][0]:
experiment["platform_accession_code"] = UNKNOWN
experiment["platform_accession_name"] = UNKNOWN
experiment["manufacturer"] = UNKNOWN
else:
external_accession = parsed_json["arraydesign"][0]["accession"]
for platform in get_supported_microarray_platforms():
if platform["external_accession"] == external_accession:
experiment[
"platform_accession_code"] = get_normalized_platform(
platform["platform_accession"])
# Illumina appears in the accession codes for
# platforms manufactured by Illumina
if "ILLUMINA" in experiment[
"platform_accession_code"].upper():
experiment["manufacturer"] = "ILLUMINA"
experiment["platform_accession_name"] = platform[
"platform_accession"]
else:
# It's not Illumina, the only other supported Microarray platform is
# Affy. As our list of supported platforms grows this logic will
# need to get more sophisticated.
experiment["manufacturer"] = "AFFYMETRIX"
platform_mapping = get_readable_affymetrix_names()
experiment[
"platform_accession_name"] = platform_mapping[
platform["platform_accession"]]
if "platform_accession_code" not in experiment:
# We don't know what platform this accession corresponds to.
experiment["platform_accession_code"] = external_accession
experiment["platform_accession_name"] = UNKNOWN
experiment["manufacturer"] = UNKNOWN
experiment["release_date"] = parsed_json["releasedate"]
if "lastupdatedate" in parsed_json:
experiment["last_update_date"] = parsed_json["lastupdatedate"]
else:
experiment["last_update_date"] = parsed_json["releasedate"]
# Create the experiment object
try:
experiment_object = Experiment.objects.get(
accession_code=experiment_accession_code)
logger.debug(
"Experiment already exists, skipping object creation.",
experiment_accession_code=experiment_accession_code,
survey_job=self.survey_job.id)
except Experiment.DoesNotExist:
# We aren't sure these fields will be populated, or how many there will be.
# Try to join them all together, or set a sensible default.
experiment_descripton = ""
if "description" in parsed_json and len(
parsed_json["description"]) > 0:
for description_item in parsed_json["description"]:
if "text" in description_item:
experiment_descripton = experiment_descripton + description_item[
"text"] + "\n"
if experiment_descripton == "":
experiment_descripton = "Description not available.\n"
experiment_object = Experiment()
experiment_object.accession_code = experiment_accession_code
experiment_object.source_url = request_url
experiment_object.source_database = "ARRAY_EXPRESS"
experiment_object.title = parsed_json["name"]
# This will need to be updated if we ever use Array
# Express to get other kinds of data.
experiment_object.technology = "MICROARRAY"
experiment_object.description = experiment_descripton
experiment_object.source_first_published = parse_datetime(
experiment["release_date"])
experiment_object.source_last_modified = parse_datetime(
experiment["last_update_date"])
experiment_object.save()
json_xa = ExperimentAnnotation()
json_xa.experiment = experiment_object
json_xa.data = parsed_json
json_xa.is_ccdl = False
json_xa.save()
## Fetch and parse the IDF/SDRF file for any other fields
IDF_URL_TEMPLATE = "https://www.ebi.ac.uk/arrayexpress/files/{code}/{code}.idf.txt"
idf_url = IDF_URL_TEMPLATE.format(code=experiment_accession_code)
idf_text = utils.requests_retry_session().get(idf_url,
timeout=60).text
lines = idf_text.split('\n')
idf_dict = {}
for line in lines:
keyval = line.strip().split('\t')
if len(keyval) == 2:
idf_dict[keyval[0]] = keyval[1]
elif len(keyval) > 2:
idf_dict[keyval[0]] = keyval[1:]
idf_xa = ExperimentAnnotation()
idf_xa.data = idf_dict
idf_xa.experiment = experiment_object
idf_xa.is_ccdl = False
idf_xa.save()
if 'Investigation Title' in idf_dict:
experiment_object.title = idf_dict['Investigation Title']
if 'Person Affiliation' in idf_dict:
# This is very rare, ex: E-MEXP-32
if isinstance(idf_dict['Person Affiliation'], list):
unique_people = list(set(idf_dict['Person Affiliation']))
experiment_object.submitter_institution = ", ".join(
unique_people)[:255]
else:
experiment_object.submitter_institution = idf_dict[
'Person Affiliation']
# Get protocol_description from "<experiment_url>/protocols"
# instead of from idf_dict, because the former provides more
# details.
protocol_url = request_url + '/protocols'
protocol_request = utils.requests_retry_session().get(protocol_url,
timeout=60)
try:
experiment_object.protocol_description = protocol_request.json(
)['protocols']
except KeyError:
logger.warning(
"Remote experiment has no protocol data!",
experiment_accession_code=experiment_accession_code,
survey_job=self.survey_job.id)
if 'Publication Title' in idf_dict:
# This will happen for some superseries.
# Ex: E-GEOD-29536
# Assume most recent is "best:, store the rest in experiment annotation.
if isinstance(idf_dict['Publication Title'], list):
experiment_object.publication_title = "; ".join(
idf_dict['Publication Title'])
else:
experiment_object.publication_title = idf_dict[
'Publication Title']
experiment_object.has_publication = True
if 'Publication DOI' in idf_dict:
if isinstance(idf_dict['Publication DOI'], list):
experiment_object.publication_doi = ", ".join(
idf_dict['Publication DOI'])
else:
experiment_object.publication_doi = idf_dict[
'Publication DOI']
experiment_object.has_publication = True
if 'PubMed ID' in idf_dict:
if isinstance(idf_dict['PubMed ID'], list):
experiment_object.pubmed_id = ", ".join(
idf_dict['PubMed ID'])
else:
experiment_object.pubmed_id = idf_dict['PubMed ID']
experiment_object.has_publication = True
# Scrape publication title and authorship from Pubmed
if experiment_object.pubmed_id:
pubmed_metadata = utils.get_title_and_authors_for_pubmed_id(
experiment_object.pubmed_id)
experiment_object.publication_title = pubmed_metadata[0]
experiment_object.publication_authors = pubmed_metadata[1]
experiment_object.save()
platform_dict = {}
for k in ('platform_accession_code', 'platform_accession_name',
'manufacturer'):
platform_dict[k] = experiment[k]
return experiment_object, platform_dict
