def test_executor_block_overwrite_import(self):
assay_A = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
assay_B = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
executor = ImportExecutor(self.load, self.user)
executor.start()
executor.parse_context({
"loa_pks": [assay_A.pk, assay_B.pk],
"matched_assays": True,
"use_assay_times": False,
})
series = [*self._generate_series(assays=[assay_A, assay_B])]
executor.import_series_data(series)
added, updated = executor.finish_import()
# reset load after finish
self.load.status = LoadRequest.Status.PROCESSING
self.load.store()
# now going again to get overwrites
executor.import_series_data(series)
with pytest.raises(exceptions.UnplannedOverwriteError):
executor.finish_import()
assert self.load.status == LoadRequest.Status.FAILED
assert added == 10
assert updated == 0
def test_executor_allow_overwrite_import(self):
assay_A = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
assay_B = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
self.load.options |= LoadRequest.Options.allow_overwrite
executor = ImportExecutor(self.load, self.user)
executor.start()
executor.parse_context({
"loa_pks": [assay_A.pk, assay_B.pk],
"matched_assays": True,
"use_assay_times": False,
})
series = [*self._generate_series(assays=[assay_A, assay_B])]
executor.import_series_data(series)
executor.finish_import()
# reset load after finish
self.load.status = LoadRequest.Status.PROCESSING
self.load.store()
# now going again to get overwrites
executor.import_series_data(series)
added, updated = executor.finish_import()
# asserts
assert added == 10
assert updated == 10
assert self.load.status == LoadRequest.Status.COMPLETED
def test_resolve_with_partial_overdetermined_time(self):
# setup
time = models.MetadataType.system("Time")
assay_A = main_factory.AssayFactory(
line=self.BW1, metadata={time.pk: 24}, protocol=self.protocol,
)
# missing time on this one!
assay_B = main_factory.AssayFactory(line=self.BW1, protocol=self.protocol,)
# including time in points
mpr = list(
self._generate_measurement_product(
names=[assay_A.name, assay_B.name], points=[[[12], [10]]],
)
)
parsed = ParseResult(
series_data=mpr, record_src="row", any_time=True, has_all_times=True,
)
resolver, type_resolver = self._make_resolvers(parsed)
# exec
context = resolver.resolve(type_resolver)
# asserts
assert context == {
"conflicted_from_import": 0,
"conflicted_from_study": 0,
"file_has_times": True,
"file_has_units": True,
"importId": self.load.request,
"loa_pks": [assay_A.pk, assay_B.pk],
"matched_assays": True,
"totalPages": 1,
"total_vals": 2,
"use_assay_times": False,
}
def test_resolve_without_times_on_assays(self):
# setup
assay_A = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
assay_B = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
mpr = list(
self._generate_measurement_product(
names=[assay_A.name, assay_B.name],
points=[[[None], [10]]],
))
parsed = ParseResult(
series_data=mpr,
record_src="row",
any_time=False,
has_all_times=False,
)
resolver, type_resolver = self._make_resolvers(parsed)
# exec
context = resolver.resolve(type_resolver)
# asserts
assert context == {
"conflicted_from_import": 0,
"conflicted_from_study": 0,
"file_has_times": False,
"file_has_units": True,
"importId": self.load.request,
"loa_pks": {assay_A.pk, assay_B.pk},
"matched_assays": True,
"totalPages": 1,
"total_vals": 2,
"use_assay_times": False,
}
def test_resolve_with_multiple_existing_values(self):
# setup
assay_A = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
assay_B = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
hours = models.MeasurementUnit.objects.get(unit_name="hours")
na = models.MeasurementUnit.objects.get(unit_name="n/a")
mtype = main_factory.MeasurementTypeFactory()
measurement_A = main_factory.MeasurementFactory(assay=assay_A,
measurement_type=mtype,
x_units=hours,
y_units=na)
measurement_B = main_factory.MeasurementFactory(assay=assay_B,
measurement_type=mtype,
x_units=hours,
y_units=na)
# making multiple existing points for same type at same time
main_factory.ValueFactory(measurement=measurement_A, x=[24])
main_factory.ValueFactory(measurement=measurement_A, x=[24])
main_factory.ValueFactory(measurement=measurement_A, x=[24])
main_factory.ValueFactory(measurement=measurement_B, x=[24])
main_factory.ValueFactory(measurement=measurement_B, x=[24])
main_factory.ValueFactory(measurement=measurement_B, x=[24])
mpr = list(
self._generate_measurement_product(
names=[assay_A.name, assay_B.name],
types=[mtype.type_name],
points=[[[24], [1]]],
))
parsed = ParseResult(
series_data=mpr,
record_src="row",
any_time=True,
has_all_times=True,
)
resolver, type_resolver = self._make_resolvers(parsed)
# exec
context = resolver.resolve(type_resolver)
# asserts
assert context == {
"conflicted_from_import": 2,
"conflicted_from_study": 6,
"file_has_times": True,
"file_has_units": True,
"importId": self.load.request,
"loa_pks": {assay_A.pk, assay_B.pk},
"matched_assays": True,
"totalPages": 1,
"total_vals": 2,
"use_assay_times": False,
}
def test_simple_import(self):
"""Test an import where everything is already resolved."""
line = factory.LineFactory(study=self.target_study)
protocol = factory.ProtocolFactory()
assay = factory.AssayFactory(line=line, protocol=protocol)
mtype = factory.MetaboliteFactory()
unit = factory.UnitFactory()
# no measurements exist before import
self.assertEqual(
models.Measurement.objects.filter(
study_id=self.target_study.pk).count(), 0)
self._set_permission(permission_type=models.StudyPermission.WRITE)
run = TableImport(self.target_study, self.user)
added, updated = run.import_series_data([{
"line_id":
line.id,
"assay_id":
assay.id,
"measurement_id":
mtype.id,
"comp_id":
models.Measurement.Compartment.UNKNOWN,
"units_id":
unit.id,
"metadata": {},
"data": [[0, 0]],
}])
# after the self-reported add/update are correct
self.assertEqual(added, 1)
self.assertEqual(updated, 0)
# and the counts are correct
self.assertEqual(
models.Measurement.objects.filter(
study_id=self.target_study.pk).count(), 1)
def setUpTestData(cls):
super().setUpTestData()
cls.user = UserFactory()
# initialize study for worklist
cls.study = factory.StudyFactory()
cls.study.userpermission_set.update_or_create(
permission_type=models.StudyPermission.READ, user=cls.user)
# initialize protocol for worklist
cls.protocol = factory.ProtocolFactory(name="potato")
# initialize twenty lines in the study, 1-base index
cls.lines = [
factory.LineFactory(name=f"L{i}", study=cls.study)
for i in range(1, 21)
]
# initialize 3x assays on lines 6, 8, 10, 12, 14 (1-base)
# assays at times 12, 24, 36
time = models.MetadataType.system("Time")
cls.assays = [
factory.AssayFactory(
line=cls.lines[line],
metadata={time.pk: 12 * i},
name=f"{cls.lines[line].name}-A{i}",
protocol=cls.protocol,
) for line in range(5, 14, 2) for i in range(1, 4)
]
def _build_measurements(cls, measurement_id, category, line, metabolite, values):
protocol = factory.ProtocolFactory(categorization=category)
assay = factory.AssayFactory(line=line, protocol=protocol)
measurement = factory.MeasurementFactory(
id=measurement_id, assay=assay, measurement_type=metabolite
)
for x, y in values:
factory.ValueFactory(measurement=measurement, x=[x], y=[y])
def test_resolve_with_missing_assays(self):
# setup
assay_A = main_factory.AssayFactory(line=self.BW1, protocol=self.protocol)
assay_B = main_factory.AssayFactory(line=self.BW1, protocol=self.protocol)
points = [*self._make_scalars([None], [10])]
mpr = list(
self._generate_measurement_product(
names=[assay_A.name, assay_B.name, "foobar"], points=points,
)
)
parsed = ParseResult(
series_data=mpr, record_src="row", any_time=True, has_all_times=True,
)
resolver, type_resolver = self._make_resolvers(parsed)
# exec + assert
with pytest.raises(exceptions.UnmatchedAssayError):
resolver.resolve(type_resolver)
def _setup_study_with_data_for_export(self):
# create study and line with 30 assays
# each assay with one measurement having one value
self.study = factory.StudyFactory()
self.line = factory.LineFactory(study=self.study)
for _i in range(30):
assay = factory.AssayFactory(line=self.line)
measurement = factory.MeasurementFactory(assay=assay)
factory.ValueFactory(measurement=measurement)
def test_executor_with_existing_assays(self):
main_factory.AssayFactory(line=self.BW1, protocol=self.protocol)
main_factory.AssayFactory(line=self.BW1, protocol=self.protocol)
executor = ImportExecutor(self.load, self.user)
executor.start()
executor.parse_context({
"loa_pks": [self.BW1.pk, self.arcA.pk],
"matched_assays": False,
"use_assay_times": False,
})
# default series makes 5x points per line
# with 2x lines
executor.import_series_data(self._generate_series())
added, updated = executor.finish_import()
# asserts
assert added == 10
assert updated == 0
assert self.load.status == LoadRequest.Status.COMPLETED
def test_resolve_with_duplicate_assays(self):
# setup
assay_A = main_factory.AssayFactory(line=self.BW1, protocol=self.protocol)
assay_B = main_factory.AssayFactory(line=self.BW1, protocol=self.protocol)
main_factory.AssayFactory(
line=self.BW1, name=assay_A.name, protocol=self.protocol
)
mpr = list(
self._generate_measurement_product(
names=[assay_A.name, assay_B.name], points=[[[None], [10]]],
)
)
parsed = ParseResult(
series_data=mpr, record_src="row", any_time=True, has_all_times=True,
)
resolver, type_resolver = self._make_resolvers(parsed)
# exec + assert
with pytest.raises(exceptions.DuplicateAssayError):
resolver.resolve(type_resolver)
def setUpTestData(cls):
super().setUpTestData()
User = get_user_model()
cls.admin = User.objects.get(username="******")
# create study and line with 30 assays
# each assay with one measurement having one value
cls.study = factory.StudyFactory()
cls.line = factory.LineFactory(study=cls.study)
for _i in range(30):
assay = factory.AssayFactory(line=cls.line)
measurement = factory.MeasurementFactory(assay=assay)
factory.ValueFactory(measurement=measurement)
def test_executor_with_partial_time_lookup_errors(self):
time = models.MetadataType.system("Time")
# has time
assay_A = main_factory.AssayFactory(
line=self.BW1,
metadata={time.pk: "12"},
protocol=self.protocol,
)
# missing time
assay_B = main_factory.AssayFactory(line=self.BW1,
protocol=self.protocol)
with reporting.tracker(self.load.request):
executor = ImportExecutor(self.load, self.user)
executor.start()
executor.parse_context({
"loa_pks": [assay_A.pk, assay_B.pk],
"matched_assays": True,
"use_assay_times": True,
})
with pytest.raises(exceptions.MissingAssayTimeError):
executor.import_series_data([])
assert self.load.status == LoadRequest.Status.FAILED
def test_executor_with_existing_assays_and_times(self):
time = models.MetadataType.system("Time")
x = 12
y = 42
assay_A = main_factory.AssayFactory(
line=self.BW1,
metadata={time.pk: x},
protocol=self.protocol,
)
assay_B = main_factory.AssayFactory(
line=self.BW1,
metadata={time.pk: x},
protocol=self.protocol,
)
executor = ImportExecutor(self.load, self.user)
executor.start()
executor.parse_context({
"loa_pks": [assay_A.pk, assay_B.pk],
"matched_assays": True,
"use_assay_times": True,
})
executor.import_series_data(
self._generate_series(
assays=[assay_A, assay_B],
series=[
[*self._make_scalars([None], [y])],
[*self._make_scalars([None], [y])],
],
))
added, updated = executor.finish_import()
# asserts
assert added == 2
assert updated == 0
assert self.load.status == LoadRequest.Status.COMPLETED
qs = models.MeasurementValue.objects.filter(measurement__assay=assay_A)
assert [*qs.values_list("x", "y")] == [([x], [y])]
qs = models.MeasurementValue.objects.filter(measurement__assay=assay_B)
assert [*qs.values_list("x", "y")] == [([x], [y])]
def _setup_parse_success(self):
mime = "text/csv"
assay = factory.AssayFactory(study=self.study, protocol=self.protocol)
factory.ProteinFactory(accession_code="P12345")
ParserFactory(
layout=self.layout,
mime_type=mime,
parser_class="edd.load.parsers.skyline.SkylineCsvParser",
)
content = f"""
Replicate Name,Protein Name,Total Area
{assay.name},sp|P12345,42
"""
file = SimpleUploadedFile("example", content.encode("utf-8"), content_type=mime)
self.load.update({"file": file})
return self.load.request
def _setup_parse_excel(self):
mime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
assay = factory.AssayFactory(study=self.study, protocol=self.protocol)
factory.ProteinFactory(accession_code="P12345")
ParserFactory(
layout=self.layout,
mime_type=mime,
parser_class="edd.load.parsers.skyline.SkylineExcelParser",
)
wb = Workbook()
wb.active.title = "Simple Upload"
wb.active.append(["Replicate Name", "Protein Name", "Total Area"])
wb.active.append([assay.name, "sp|P12345", 42])
with NamedTemporaryFile() as temp:
wb.save(temp)
temp.seek(0)
file = SimpleUploadedFile("example", temp.read(), content_type=mime)
self.load.update({"file": file})
return self.load.request
