def test_ordering_mismatch(self):
"""Makes sure that the order samples' keys are in does not affect the title chosen.
Related: https://github.com/AlexsLemonade/refinebio/pull/304
"""
experiment_accession_code = "E-TABM-38"
samples_endpoint = SAMPLES_URL.format(experiment_accession_code)
r = utils.requests_retry_session().get(samples_endpoint, timeout=60)
json_samples = r.json()["experiment"]["sample"]
flattened_json_samples = [
utils.flatten(json_sample) for json_sample in json_samples
]
SDRF_URL_TEMPLATE = "https://www.ebi.ac.uk/arrayexpress/files/{code}/{code}.sdrf.txt"
sdrf_url = SDRF_URL_TEMPLATE.format(code=experiment_accession_code)
parsed_samples = parse_sdrf(sdrf_url)
title_field = determine_title_field(parsed_samples,
flattened_json_samples)
sdrf_samples = harmonize_all_samples(parsed_samples, title_field)
json_titles = [
extract_title(json_sample, title_field)
for json_sample in flattened_json_samples
]
# The titles won't match up if the order of the sample dicts
# isn't corrected for, resulting in a KeyError being raised.
# So if this doesn't raise a KeyError, then we're good.
for title in json_titles:
sdrf_samples[title]
def create_samples_from_api(self, experiment: Experiment,
platform_dict: Dict) -> List[Sample]:
"""Generates a Sample item for each sample in an AE experiment.
There are many possible data situations for a sample:
- If the sample only has raw data available:
- If it is on a platform that we support:
Download this raw data and process it
- If it is not on a platform we support:
Don't download anything, don't process anything
- If the sample has both raw and derived data:
- If the raw data is on a platform we support:
Download the raw data and process it, abandon the derived data
- If the raw data is not on a platform we support
Download the derived data and no-op it, abandon the raw data
- If the sample only has derived data:
Download the derived data and no-op it.
See an example at: https://www.ebi.ac.uk/arrayexpress/json/v3/experiments/E-MTAB-3050/samples
"""
created_samples = []
samples_endpoint = SAMPLES_URL.format(experiment.accession_code)
r = utils.requests_retry_session().get(samples_endpoint, timeout=60)
samples = r.json()["experiment"]["sample"]
# The SDRF is the complete metadata record on a sample/property basis.
# We run this through our harmonizer and then attach the properties
# to our created samples.
SDRF_URL_TEMPLATE = "https://www.ebi.ac.uk/arrayexpress/files/{code}/{code}.sdrf.txt"
sdrf_url = SDRF_URL_TEMPLATE.format(code=experiment.accession_code)
sdrf_samples = harmony.parse_sdrf(sdrf_url)
harmonized_samples = harmony.harmonize(sdrf_samples)
# An experiment can have many samples
for sample_data in samples:
# For some reason, this sample has no files associated with it.
if "file" not in sample_data or len(sample_data['file']) == 0:
continue
# Each sample is given an experimenatlly-unique title.
flat_sample = utils.flatten(sample_data)
title = harmony.extract_title(flat_sample)
# A sample may actually have many sub files.
# If there is raw data, take that.
# If not, take the derived.
has_raw = False
for sub_file in sample_data['file']:
# For ex: E-GEOD-15645
if isinstance(sub_file['comment'], list):
sub_file_mod = sub_file
sub_file_mod['comment'] = sub_file['comment'][0]
else:
sub_file_mod = sub_file
# Some have the 'data' field, but not the actual data
# Ex: E-GEOD-9656
if sub_file_mod['type'] == "data" and sub_file_mod[
'comment'].get('value', None) != None:
has_raw = True
if 'raw' in sub_file_mod['comment'].get('value', ''):
has_raw = True
skip_sample = False
for sub_file in sample_data['file']:
# Don't get the raw data if it's only a 1-color sample.
if 'Cy3' in str(sample_data) and 'Cy5' not in str(sample_data):
has_raw = False
# Skip derived data if we have it raw.
if has_raw and "derived data" in sub_file['type']:
continue
download_url = None
filename = sub_file["name"]
# sub_file["comment"] is only a list if there's
# more than one comment...
comments = sub_file["comment"]
if isinstance(comments, list):
# Could be: "Derived ArrayExpress Data Matrix FTP
# file" or: "ArrayExpress FTP file". If there is
# no comment with a name including "FTP file" then
# we don't know where to download it so we need to
# mark this job as an error. Therefore don't catch
# the potential exception where download_url
# doesn't get defined.
for comment in comments:
if "FTP file" in comment["name"]:
download_url = comment["value"]
break
else:
download_url = comments["value"]
if not download_url:
logger.error(
"Sample %s did not specify a download url, skipping.",
sample_accession_code,
experiment_accession_code=experiment.accession_code,
survey_job=self.survey_job.id,
sub_file=sub_file)
skip_sample = True
continue
if not filename:
logger.error(
"Sample %s did not specify a filename, skipping.",
sample_accession_code,
experiment_accession_code=experiment.accession_code,
survey_job=self.survey_job.id,
sub_file=sub_file)
skip_sample = True
continue
if skip_sample:
continue
# The accession code is not a simple matter to determine.
sample_source_name = sample_data["source"].get("name", "")
sample_assay_name = sample_data["assay"].get("name", "")
sample_accession_code = self.determine_sample_accession(
experiment.accession_code, sample_source_name,
sample_assay_name, filename)
# Figure out the Organism for this sample
organism_name = UNKNOWN
for characteristic in sample_data["characteristic"]:
if characteristic["category"].upper() == "ORGANISM":
organism_name = characteristic["value"].upper()
if organism_name == UNKNOWN:
logger.error(
"Sample %s did not specify the organism name.",
sample_accession_code,
experiment_accession_code=experiment.accession_code,
survey_job=self.survey_job.id)
organism = None
continue
else:
organism = Organism.get_object_for_name(organism_name)
# Create the sample object
try:
# Associate it with the experiment, but since it
# already exists it already has original files
# associated with it and it's already been downloaded,
# so don't add it to created_samples.
sample_object = Sample.objects.get(
accession_code=sample_accession_code)
# If input experiment includes new protocol information,
# update sample's protocol_info.
existing_protocols = sample_object.protocol_info
protocol_info, is_updated = self.update_sample_protocol_info(
existing_protocols, experiment.protocol_description,
experiment.source_url + '/protocols')
if is_updated:
sample_object.protocol_info = protocol_info
sample_obejct.save()
logger.debug(
"Sample %s already exists, skipping object creation.",
sample_accession_code,
experiment_accession_code=experiment.accession_code,
survey_job=self.survey_job.id)
except Sample.DoesNotExist:
sample_object = Sample()
# The basics
sample_object.source_database = "ARRAY_EXPRESS"
sample_object.title = title
sample_object.accession_code = sample_accession_code
sample_object.source_archive_url = samples_endpoint
sample_object.organism = organism
sample_object.platform_name = platform_dict[
"platform_accession_name"]
sample_object.platform_accession_code = platform_dict[
"platform_accession_code"]
sample_object.manufacturer = platform_dict["manufacturer"]
sample_object.technology = "MICROARRAY"
protocol_info, is_updated = self.update_sample_protocol_info(
existing_protocols=[],
experiment_protocol=experiment.protocol_description,
protocol_url=experiment.source_url + '/protocols')
# Do not check is_updated the first time because we must
# save a list so we can append to it later.
sample_object.protocol_info = protocol_info
sample_object.save()
# Directly assign the harmonized properties
harmonized_sample = harmonized_samples[title]
for key, value in harmonized_sample.items():
setattr(sample_object, key, value)
sample_object.save()
sample_annotation = SampleAnnotation()
sample_annotation.data = sample_data
sample_annotation.sample = sample_object
sample_annotation.is_ccdl = False
sample_annotation.save()
original_file = OriginalFile()
original_file.filename = filename
original_file.source_filename = filename
original_file.source_url = download_url
original_file.is_downloaded = False
original_file.is_archive = True
original_file.has_raw = has_raw
original_file.save()
original_file_sample_association = OriginalFileSampleAssociation(
)
original_file_sample_association.original_file = original_file
original_file_sample_association.sample = sample_object
original_file_sample_association.save()
created_samples.append(sample_object)
logger.debug(
"Created " + str(sample_object),
experiment_accession_code=experiment.accession_code,
survey_job=self.survey_job.id,
sample=sample_object.id)
# Create associations if they don't already exist
ExperimentSampleAssociation.objects.get_or_create(
experiment=experiment, sample=sample_object)
ExperimentOrganismAssociation.objects.get_or_create(
experiment=experiment, organism=organism)
return created_samples