def test_new_parameterset(self):
psm = ParameterSetManager(parentObject=self.datafile,
schema="http://localhost/psmtest/df2/")
self.assertTrue(
psm.get_schema().namespace == "http://localhost/psmtest/df2/")
psm.set_param("newparam1", "test3", "New Parameter 1")
self.assertTrue(psm.get_param("newparam1").string_value == "test3")
self.assertTrue(
psm.get_param("newparam1").name.full_name == "New Parameter 1")
psm.new_param("newparam1", "test4")
self.assertTrue(len(psm.get_params("newparam1", True)) == 2)
psm.set_param_list("newparam2", ("a", "b", "c", "d"))
self.assertTrue(len(psm.get_params("newparam2")) == 4)
psm.set_params_from_dict({"newparam2": "test5", "newparam3": 3})
self.assertTrue(psm.get_param("newparam2", True) == "test5")
# the newparam3 gets created and '3' is set to a string_value
# since once cannot assume that an initial numeric value
# will imply continuing numeric type for this new param
self.assertTrue(psm.get_param("newparam3").string_value == '3')
psm.delete_params("newparam1")
self.assertTrue(len(psm.get_params("newparam1", True)) == 0)
def _test_metadata(self,schema,name,dataset,fields):
""" Check that metadata is correct"""
try:
sch = models.Schema.objects.get(namespace__exact=schema,name=name)
except Schema.DoesNotExist:
self.assertTrue(False)
self.assertEqual(schema,sch.namespace)
self.assertEqual(name,sch.name)
try:
datasetparameterset = models.DatasetParameterSet.objects.get(schema=sch, dataset=dataset)
except DatasetParameterSet.DoesNotExist:
self.assertTrue(False)
psm = ParameterSetManager(parameterset=datasetparameterset)
for key, field_type, value in fields:
logger.debug("key=%s,field_type=%s,value=%s" % (key,field_type, value))
try:
# First check stringed value
param = psm.get_param(key,value=True)
self.assertEquals(str(param),str(value))
# Then correct type
param = psm.get_param(key,value=False)
self.assertEquals(param.name.data_type,field_type)
except DatasetParameter.DoesNotExist:
logger.error("cannot find %s" % key)
self.assertTrue(False)
def test_existing_parameterset(self):
psm = ParameterSetManager(parameterset=self.datafileparameterset)
self.assertTrue(psm.get_schema().namespace\
== "http://localhost/psmtest/df/")
self.assertTrue(psm.get_param("parameter1").string_value == "test1")
self.assertTrue(psm.get_param("parameter2", True) == 2)
def test_existing_parameterset(self):
psm = ParameterSetManager(parameterset=self.datafileparameterset)
self.assertTrue(psm.get_schema().namespace ==
"http://localhost/psmtest/df/")
self.assertTrue(psm.get_param("parameter1").string_value == "test1")
self.assertTrue(psm.get_param("parameter2", True) == 2)
def testPersisterStoresEntryMetadata(self):
# Create user to associate with dataset
user = User(username="******")
user.save()
feed, entry = self._getTestEntry()
p = AtomPersister()
dataset = p.process(feed, entry)
parameterset = dataset.getParameterSets().get(schema=AtomImportSchemas.get_schema(Schema.DATASET))
expect(parameterset) != None
psm = ParameterSetManager(parameterset)
expect(psm.get_param("EntryID").get()).to_equal(entry.id)
expect(psm.get_param("Updated").name.isDateTime()).to_be_truthy()
# Compare against non-timezoned update time
expect(psm.get_param("Updated", True)).to_equal(iso8601.parse_date(entry.updated))
def test_link_parameter_type(self):
"""
Test that Parameter.link_gfk (GenericForeignKey) is correctly
assigned after using Parameter.set_value(some_url) for a LINK Parameter.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
# Check link to experiment
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.assertTrue(psm.get_param("exp_link").string_value == exp_url)
self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
# Check link to dataset
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.assertTrue(
psm.get_param("dataset_link").string_value == dataset_url)
self.assertTrue(
psm.get_param("dataset_link").link_id == self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
def test_link_parameter_type(self):
"""
Test that Parameter.link_gfk (GenericForeignKey) is correctly
assigned after using Parameter.set_value(some_url) for a LINK Parameter.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
# Check link to experiment
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.assertTrue(psm.get_param("exp_link").string_value ==
exp_url)
self.assertTrue(psm.get_param("exp_link").link_id ==
self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
# Check link to dataset
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.assertTrue(psm.get_param("dataset_link").string_value ==
dataset_url)
self.assertTrue(psm.get_param("dataset_link").link_id ==
self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
def get_or_create_unique_parameterset(self, schema, parametername, value):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=schema,
experiment__id=self.experiment_id)
experiment = Experiment.objects.get(id=self.experiment_id)
psm = None
if not len(parameterset):
psm = ParameterSetManager(schema=schema,
parentObject=experiment)
return psm
else:
for ps in parameterset:
psm = ParameterSetManager(parameterset=ps)
try:
ps_value = psm.get_param("party_id",
True)
if value == ps_value:
return psm
except ExperimentParameter.DoesNotExist:
pass # keep going
psm = ParameterSetManager(schema=schema,
parentObject=experiment)
return psm
def testPersisterAgainstPicasa(self):
'''
Test against Picasa - useful for development testing
'''
# Build a persister with "make_local_copy" mocked out
# (file transfer isn't part of this test)
persister = flexmock(AtomPersister())
persister.should_receive('make_local_copy').times(50)
doc = feedparser.parse('picasa_example.atom')
# Process the first twenty entries
for entry in list(reversed(doc.entries))[0:20]:
persister.process(doc.feed, entry)
# We processed 20 images (Picasa only has one image per dataset)
eq_(Dataset.objects.count(), 20)
# This part has 2 users
eq_(User.objects.count(), 2)
# This part covers 6 albums
eq_(Experiment.objects.count(), 6)
for experiment in Experiment.objects.all():
pset = experiment.getParameterSets().get(schema=AtomImportSchemas. \
get_schema(Schema.EXPERIMENT))
pset_mgr = ParameterSetManager(pset)
assert experiment.title != pset_mgr.get_param('ExperimentID')
# Change the experiment titles, to check this won't be a problem
experiment.title = "Title removed for testing"
experiment.save()
# Process the rest of the entries
for entry in list(reversed(doc.entries))[20:]:
persister.process(doc.feed, entry)
# We processed 50 images (Picasa only has one image per dataset)
eq_(Dataset.objects.count(), 50)
# This part has 2 users
eq_(User.objects.count(), 4)
# This part covers 6 albums
eq_(Experiment.objects.count(), 9)
def testPersisterStoresEntryMetadata(self):
# Create user to associate with dataset
user = User(username="******")
user.save()
feed, entry = self._getTestEntry()
p = AtomPersister()
dataset = p.process(feed, entry)
parameterset = dataset.getParameterSets() \
.get(schema=AtomImportSchemas. \
get_schema(Schema.DATASET))
expect(parameterset) != None
psm = ParameterSetManager(parameterset)
expect(psm.get_param('EntryID').get()).to_equal(entry.id)
expect(psm.get_param('Updated').name.isDateTime()).to_be_truthy()
# Compare against non-timezoned update time
expect(psm.get_param('Updated', True)) \
.to_equal(iso8601.parse_date(entry.updated))
def _get_dict_from_ps(self, ps):
'''
Build dictionary by getting the parameter values from the keys, then
zipping it all together.
'''
psm = ParameterSetManager(ps)
return dict([('id', ps.id)]+ # Use set ID
zip(self.parameter_names,
(psm.get_param(k, True) for k in self.parameter_names)))
def _get_dict_from_ps(self, ps):
'''
Build dictionary by getting the parameter values from the keys, then
zipping it all together.
'''
psm = ParameterSetManager(ps)
return dict([('id', ps.id)] + # Use set ID
zip(self.parameter_names, (psm.get_param(k, True)
for k in self.parameter_names)))
def test_tz_aware_date_handling(self):
"""
Ensure that dates are handling in a timezone-aware way.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", '1970-01-01T08:00:00+08:00')
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def test_tz_naive_date_handling(self):
'''
Ensure that dates are handling in a timezone-aware way.
'''
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", datetime(1970,01,01,10,0,0))
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970,01,01,0,0,0,tzinfo=pytz.utc))
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ["type", "identifier", "title", "notes"]
try:
return dict(
[("id", ps.id)]
+ zip(parameter_names, (psm.get_param(k, True) for k in parameter_names)) # Use set ID
)
except ExperimentParameter.DoesNotExist:
return dict() # drop Related_Info record with missing fields
def test_tz_naive_date_handling(self):
"""
Ensure that dates are handling in a timezone-aware way.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", str(datetime(1970, 01, 01, 10, 0, 0)))
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ['type','identifier','title','notes']
try:
return dict([('id', ps.id)] + # Use set ID
zip(parameter_names,
(psm.get_param(k, True) \
for k in parameter_names)))
except ExperimentParameter.DoesNotExist:
return dict() # drop Related_Info record with missing fields
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ['type','identifier','title','notes']
try:
return dict([('id', ps.id)] + # Use set ID
zip(parameter_names,
(psm.get_param(k, True) \
for k in parameter_names)))
except ExperimentParameter.DoesNotExist:
return dict() # drop Related_Info record with missing fields
def testJEOLComplex(self):
JEOLSEMFilter()(None, instance=self.datafiles[1])
# Check a parameter set was created
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(dataset.getParameterSets()[0])
expect(psm.get_param('metadata-filename', True))\
.to_equal(self.datafiles[1].filename)
expect(psm.get_param('instrument', True)).to_equal('7001F_TTL')
expect(psm.get_param('accel_volt', True)).to_equal(15.0)
expect(psm.get_param('micron_bar', True)).to_equal(213)
expect(psm.get_param('micron_marker', True)).to_equal(100)
# Check we won't create a duplicate dataset
JEOLSEMFilter()(None, instance=self.datafiles[1])
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
def testJEOLComplex(self):
JEOLSEMFilter()(None, instance=self.datafiles[1])
# Check a parameter set was created
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(dataset.getParameterSets()[0])
expect(psm.get_param('metadata-filename', True))\
.to_equal(self.datafiles[1].filename)
expect(psm.get_param('instrument', True)).to_equal('7001F_TTL')
expect(psm.get_param('accel_volt', True)).to_equal(15.0)
expect(psm.get_param('micron_bar', True)).to_equal(213)
expect(psm.get_param('micron_marker', True)).to_equal(100)
# Check we won't create a duplicate dataset
JEOLSEMFilter()(None, instance=self.datafiles[1])
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
def test_tz_aware_date_handling(self):
'''
Ensure that dates are handling in a timezone-aware way.
'''
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3",
iso8601.parse_date('1970-01-01T08:00:00+08:00'))
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970,01,01,0,0,0,tzinfo=pytz.utc))
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ['type', 'identifier', 'title', 'notes']
try:
parameters = {
key: psm.get_param(key, True)
for key in parameter_names}
parameters['id'] = ps.id
return parameters
except ExperimentParameter.DoesNotExist:
return dict() # drop Related_Info record with missing fields
def _get_param(self, key, namespace, experiment):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=namespace, experiment__id=experiment.id)
if len(parameterset) > 0:
psm = ParameterSetManager(parameterset=parameterset[0])
try:
return psm.get_param(key, True)
except MultipleObjectsReturned:
return psm.get_params(key, True)
except ObjectDoesNotExist:
return None
def _get_param(self, key, namespace, experiment):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=namespace,
experiment__id=experiment.id)
if len(parameterset) > 0:
psm = ParameterSetManager(parameterset=parameterset[0])
try:
return psm.get_param(key, True)
except MultipleObjectsReturned:
return psm.get_params(key, True)
except ObjectDoesNotExist:
return None
def testFilter(self):
DataGrabberFilter()(None, instance=self.datafiles[0])
# Check a DATASET parameter set was created
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(dataset.getParameterSets()[0])
expect(psm.get_param('admin-filename', True))\
.to_equal(self.datafiles[0].filename)
expect(psm.get_param('user_name', True)).to_equal('s.crawley')
expect(psm.get_param('facility_name', True)).to_equal('QBP NIKON TIE')
expect(psm.get_param('account_name', True)).to_equal('200 - CMM STAFF')
expect(psm.get_param('session_uuid', True)).to_equal(
'cbc86da5-e143-4a11-9951-abaffef70efa')
expect(psm.get_param('record_uuid', True)).to_equal(
'ddf0df63-d985-4ffa-9102-e34a78b3fd1f')
# Repeat for the datafile parameters
expect(self.datafiles[0].getParameterSets().count()).to_equal(0)
expect(self.datafiles[1].getParameterSets().count()).to_equal(1)
psm = ParameterSetManager(self.datafiles[1].getParameterSets()[0])
expect(psm.get_param('instrument_pathname', True)).to_equal(
'S:\\Garry\\testfile.txt')
# Check we won't create a duplicate dataset
DataGrabberFilter()(None, instance=self.datafiles[0])
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
def test_new_parameterset(self):
psm = ParameterSetManager(parentObject=self.datafile,
schema="http://localhost/psmtest/df2/")
self.assertTrue(psm.get_schema().namespace ==
"http://localhost/psmtest/df2/")
psm.set_param("newparam1", "test3", "New Parameter 1")
self.assertTrue(psm.get_param("newparam1").string_value ==
"test3")
self.assertTrue(psm.get_param("newparam1").name.full_name ==
"New Parameter 1")
psm.new_param("newparam1", "test4")
self.assertTrue(len(psm.get_params("newparam1", True)) == 2)
psm.set_param_list("newparam2", ("a", "b", "c", "d"))
self.assertTrue(len(psm.get_params("newparam2")) == 4)
psm.set_params_from_dict(
{"newparam2": "test5", "newparam3": 3})
self.assertTrue(psm.get_param("newparam2", True) == "test5")
# the newparam3 gets created and '3' is set to a string_value
# since once cannot assume that an initial numeric value
# will imply continuing numeric type for this new param
self.assertTrue(psm.get_param("newparam3").string_value == '3')
psm.delete_params("newparam1")
self.assertTrue(len(psm.get_params("newparam1", True)) == 0)
def testFlexstationSimple(self):
"""
Simple test running the filter and making sure the Softmax version number was saved
"""
filter = FlexstationFilter("Flexstation Test Schema", "http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
# Check a parameter set was created for the datafile
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
# Check that at least the verion number was extracted
psm = ParameterSetManager(datafile.getParameterSets()[0])
expect(psm.get_param('softmax_version', True)).to_equal('5.42.1.0')
# Check we won't create a duplicate datafile
filter = FlexstationFilter("Flexstation Test Schema", "http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
def source_path(datafile, exclude=[], strip='S:\\',
windowsPath=True, rootdir=''):
for ps in datafile.getParameterSets():
if ps.schema.namespace != DataGrabberFilter.SCHEMA2:
continue
psm = ParameterSetManager(ps)
try:
pathname = psm.get_param('instrument_pathname', value=True)
if pathname.startswith(strip):
pathname = pathname[len(strip):]
if windowsPath:
pathname = pathname.replace('\\', '/')
for pattern in exclude:
regex = compile(pattern)
if regex.match(pathname):
return None
return pathname
except DatafileParameter.DoesNotExist:
pass
return None
def get_existing_ldap_party_info(self):
pais = PartyActivityInformationService()
pai = pais.get_pai()
eps = ExperimentParameter.objects.filter(name__name='party_id',
parameterset__schema__namespace='http://localhost/pilot/party/1.0/',
parameterset__experiment__id=self.experiment_id)
party_info = []
for ep in eps:
psm = ParameterSetManager(parameterset=ep.parameterset)
display_name = pai.get_display_name_for_party(ep.string_value)
info = {}
info['party_id'] = ep.string_value
info['party_fullname'] = display_name
info['relation'] = psm.get_param('relationToCollection', True)
party_info.append(info)
return party_info
def testFlexstationSimple(self):
"""
Simple test running the filter and making sure the Softmax version number was saved
"""
filter = FlexstationFilter("Flexstation Test Schema",
"http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
# Check a parameter set was created for the datafile
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
# Check that at least the verion number was extracted
psm = ParameterSetManager(datafile.getParameterSets()[0])
expect(psm.get_param('softmax_version', True)).to_equal('5.42.1.0')
# Check we won't create a duplicate datafile
filter = FlexstationFilter("Flexstation Test Schema",
"http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
def test_link_parameter_type_extra(self):
# make a second ParameterSet for testing some variations
# in URL values
self.datafileparameterset2 = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset2.save()
psm = ParameterSetManager(parameterset=self.datafileparameterset2)
self.dataset_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_dataset_link)
# /dataset/1 - no trailing slash
dataset_url = self.dataset.get_absolute_url()
self.dataset_link_param2.set_value(dataset_url)
self.dataset_link_param2.save()
# Check link_id/link_ct/link_gfk to dataset
self.assertTrue(psm.get_param("dataset_link").link_id ==
self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
# Test links of the form /api/v1/experiment/<experiment_id>/
self.exp_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_exp_link)
exp_url = '/api/v1/experiment/%s/' % self.exp.id
self.exp_link_param2.set_value(exp_url)
self.exp_link_param2.save()
# Check link_id/link_ct/link_gfk to experiment
self.assertTrue(psm.get_param("exp_link").link_id ==
self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
def test_link_parameter_type_extra(self):
# make a second ParameterSet for testing some variations
# in URL values
self.datafileparameterset2 = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset2.save()
psm = ParameterSetManager(parameterset=self.datafileparameterset2)
self.dataset_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_dataset_link)
# /dataset/1 - no trailing slash
dataset_url = self.dataset.get_absolute_url()
self.dataset_link_param2.set_value(dataset_url)
self.dataset_link_param2.save()
# Check link_id/link_ct/link_gfk to dataset
self.assertTrue(
psm.get_param("dataset_link").link_id == self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
# Test links of the form /api/v1/experiment/<experiment_id>/
self.exp_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_exp_link)
exp_url = '/api/v1/experiment/%s/' % self.exp.id
self.exp_link_param2.set_value(exp_url)
self.exp_link_param2.save()
# Check link_id/link_ct/link_gfk to experiment
self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
def get_subject(ps, type_):
psm = ParameterSetManager(ps)
return { 'text': psm.get_param('code', True),
'type': type_ }
def testFlexstationTwoExperimentsInFile(self):
"""
Simple test running the filter and making sure the Softmax version number was saved
"""
filter = FlexstationFilter("Flexstation Test Schema", "http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[5])
# Check a parameter set was created for the datafile
datafile = Dataset_File.objects.get(id=self.datafiles[5].id)
expect(datafile.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(datafile.getParameterSets()[0])
expect(psm.get_param('softmax_version', True)).to_equal('5.4.52.1.0')
expect(psm.get_param('experiment_name', True)).to_equal('Exp01')
expect(psm.get_param('analysis_notes', True)).to_equal(
u'Revision_101: PROTOCOL REVISION HISTORY:\rv1.0.0: original protocol created (MDC)\rv1.0.1: 06/30/05 - Updated, spell checked, & formatted to new style guide. (DW)\r\rREADER SUITABILITY:\r\nEMax, VMax, ThermoMax, VersaMax, SpectraMax, SpectraMax Plus, SpectraMax Plus 384, SpectraMax 190, SpectraMax 340PC, SpectraMax 340PC 384, SpectraMax M2, SpectraMax M5. Intro: MIPS resupply 3759, 3720 latin square\rcells seeded 48hrs prior using FTA. Cells were 95-100% confluent on the day of the experiment.\rColumns 1,3,5,7,9,11 are non-transfected HEK cells, columns 2,4,6,8,10,12 are hTRPV4 HEK.\rCells were induced the evening prior to assay with 0.1\xb5g/ml tet \rLoaded with FURA-2 at 805 Inhibitor at 900\rLatin square\r 1 2 3 4 5 6 \ra x x x x x x\rb c z y x w v \rc v c z y x w\rd w v c z y x\re x w v c z y\rf y x w v c z\rg z y x w v c\rh x x x x x x\rc control V vehicle 0.1% DMSO, w 3759 10\xb5M x 3759 1\xb5M y 3720 10\xb5M z 3720 1\xb5M\rInjection 1 at 15"\rSLIGRL 30\xb5M\rInjection 2 at 80" \rGSK 30nM ')
expect(psm.get_param('instrument_info', True)).to_equal('Flexstation III ROM v3.0.22 16Feb11')
#expect(psm.get_param('plate_read_time', True)).to_equal('')
#expect(psm.get_param('read_type', True)).to_equal('')
#expect(psm.get_param('data_mode', True)).to_equal('')
#expect(psm.get_param('data_type', True)).to_equal('')
expect(psm.get_param('strips', True)).to_equal('1-12')
expect(psm.get_param('trans', True)).to_equal(u'Trans1: H=80\xb5, R=4, V=20.0\xb5, \x4015. Trans2: H=100\xb5, R=4, V=25.0\xb5, \x4080')
expect(psm.get_param('kinetic_points', True)).to_equal(39.0)
expect(psm.get_param('kinetic_flex_read_time', True)).to_equal(150.0)
expect(psm.get_param('kinetic_flex_interval', True)).to_equal(3.9)
expect(psm.get_param('number_of_wavelengths', True)).to_equal(2)
expect(psm.get_param('read_wavelength', True)).to_equal('520 520')
expect(psm.get_param('number_of_wells_or_cuvette', True)).to_equal(96.0)
expect(psm.get_param('excitation_wavelengths', True)).to_equal('340 380')
def get_subject(ps, type_):
psm = ParameterSetManager(ps)
return {"text": psm.get_param("code", True), "type": type_}
def test_save_metadata(self):
from os import path
try:
from tardis.apps.microtardis.filters.spctags import SPCTagsFilter
except:
raise SkipTest()
exp = models.Experiment(title='exp: test spc filter',
institution_name='rmit',
approved=True,
created_by=self.user,
public=False)
exp.save()
logger.debug("experiment %s" % exp)
self.assertEqual(exp.get_or_create_directory(),
path.join(settings.FILE_STORE_PATH, str(exp.id)))
self.experiment_path = path.join(settings.FILE_STORE_PATH, str(exp.id))
dataset = models.Dataset(description="dataset description...", experiment=exp)
dataset.save()
## FEIQuanta200 TIF image testing ##
filename = path.join(path.abspath(path.dirname(__file__)), 'testing/Quanta200/test.spc')
logger.debug("filename %s" % filename)
df_file = models.Dataset_File(dataset=dataset, filename='test.spc', url=filename, protocol='staging')
df_file.save()
#self.assertEqual("",models.Schema.objects.all())
sch = models.Schema.objects.get(name="EDAXGenesis_SPC")
self.assertEqual("EDAXGenesis_SPC",sch.name)
datafileparameterset = models.DatafileParameterSet.objects.get(schema=sch, dataset_file=df_file)
psm = ParameterSetManager(parameterset=datafileparameterset)
self.assertEqual(10,len(psm.parameters))
self.assertEqual(str(psm.get_param("Acc. Voltage").numerical_value), "19.981")
self.assertEqual(str(psm.get_param("Peak ID Element 1").string_value), "Atomic=O, Line=K, Energy=0.5150, Height=2209")
self.assertEqual(str(psm.get_param("Peak ID Element 2").string_value), "Atomic=Al, Line=K, Energy=1.4853, Height=6622")
self.assertEqual(str(psm.get_param("Peak ID Element 3").string_value), "Atomic=Si, Line=K, Energy=1.7388, Height=491")
self.assertEqual(str(psm.get_param("Peak ID Element 4").string_value), "Atomic=P, Line=K, Energy=2.0128, Height=160")
self.assertEqual(str(psm.get_param("Peak ID Element 5").string_value), "Atomic=K, Line=K, Energy=3.3156, Height=124")
self.assertEqual(str(psm.get_param("Peak ID Element 6").string_value), "Atomic=Mn, Line=K, Energy=5.8972, Height=1056")
self.assertEqual(str(psm.get_param("Live Time").numerical_value), "120.0")
self.assertEqual(str(psm.get_param("Time Constant").numerical_value), "120.0")
self.assertEqual(str(psm.get_param("Sample Type (Label)").string_value), "sample 2 - 20 kv area")
## nanoSEM TIF image testing ##
filename = path.join(path.abspath(path.dirname(__file__)), 'testing/NovaNanoSEM/test.spc')
logger.debug("filename %s" % filename)
df_file = models.Dataset_File(dataset=dataset, filename='test.spc', url=filename, protocol='staging')
df_file.save()
#self.assertEqual("",models.Schema.objects.all())
sch = models.Schema.objects.get(name="EDAXGenesis_SPC")
self.assertEqual("EDAXGenesis_SPC",sch.name)
datafileparameterset = models.DatafileParameterSet.objects.get(schema=sch, dataset_file=df_file)
psm = ParameterSetManager(parameterset=datafileparameterset)
self.assertEqual(9,len(psm.parameters))
self.assertEqual(str(psm.get_param("Acc. Voltage").numerical_value), "15.0")
self.assertEqual(str(psm.get_param("Peak ID Element 1").string_value), "Atomic=C, Line=K, Energy=0.2660, Height=74078")
self.assertEqual(str(psm.get_param("Peak ID Element 2").string_value), "Atomic=O, Line=K, Energy=0.5150, Height=2158")
self.assertEqual(str(psm.get_param("Peak ID Element 3").string_value), "Atomic=Cu, Line=L, Energy=0.9316, Height=1760")
self.assertEqual(str(psm.get_param("Peak ID Element 4").string_value), "Atomic=S, Line=K, Energy=2.3075, Height=1128")
self.assertEqual(str(psm.get_param("Peak ID Element 5").string_value), "Atomic=Sb, Line=L, Energy=3.6041, Height=390")
self.assertEqual(str(psm.get_param("Live Time").numerical_value), "343.9")
self.assertEqual(str(psm.get_param("Time Constant").numerical_value), "500.0")
self.assertEqual(str(psm.get_param("Sample Type (Label)").string_value), "Surface")
def testFlexstationAllFields(self):
"""
Simple test running the filter and making sure the Softmax version number was saved
"""
filter = FlexstationFilter("Flexstation Test Schema",
"http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
# Check a parameter set was created for the datafile
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(datafile.getParameterSets()[0])
expect(psm.get_param('softmax_version', True)).to_equal('5.42.1.0')
expect(psm.get_param('experiment_name', True)).to_equal('Experiment#1')
expect(psm.get_param('analysis_notes', True)).to_equal(
u'Notes#1: TRPV1 Phosphate mutants\rCells seeded 48hrs prior 40K cel/well \rInduced with tetracycline for 3 hrs washed once with hepes (50microL/well) then loaded with fura2 for 1 hr (50 microL/well). then washed twice with 60microl or HEPES buffer per well finaly loaded with 60microl of hepes.\rCells:\rcolumn 1: Nt, 2: WtV1, 3: C1, 4: C2, 5: C3, 6: C4, 7: C5, 8: N1, 9: N6\rinjection 1: rows A-D buffer only, E-H 100microM SLIGRL\rinjection 2: Row A,E DMSO 1%, B,F 1microM CAPS, C,G 10microM CAPS, D,H 100microM caps'
)
expect(psm.get_param(
'instrument_info',
True)).to_equal('Flexstation III ROM v2.1.35 20May09')
#expect(psm.get_param('plate_read_time', True)).to_equal('')
#expect(psm.get_param('read_type', True)).to_equal('')
#expect(psm.get_param('data_mode', True)).to_equal('')
#expect(psm.get_param('data_type', True)).to_equal('')
expect(psm.get_param('strips', True)).to_equal('1-9')
expect(psm.get_param('trans', True)).to_equal(
u'Trans1: H=80\xb5, R=4, V=20.0\xb5, \x4015. Trans2: H=100\xb5, R=4, V=20.0\xb5, \x40115'
)
expect(psm.get_param('kinetic_points', True)).to_equal(65.0)
expect(psm.get_param('kinetic_flex_read_time', True)).to_equal(250.0)
expect(psm.get_param('kinetic_flex_interval', True)).to_equal(3.9)
expect(psm.get_param('number_of_wavelengths', True)).to_equal(2)
expect(psm.get_param('read_wavelength', True)).to_equal('520 520')
expect(psm.get_param('number_of_wells_or_cuvette',
True)).to_equal(96.0)
expect(psm.get_param('excitation_wavelengths',
True)).to_equal('340 380')
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ['type','identifier','title','notes']
return dict([('id', ps.id)]+ # Use set ID
zip(parameter_names,
(psm.get_param(k, True) for k in parameter_names)))
def testFlexstationTwoExperimentsInFile(self):
"""
Simple test running the filter and making sure the Softmax version number was saved
"""
filter = FlexstationFilter("Flexstation Test Schema",
"http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[5])
# Check a parameter set was created for the datafile
datafile = Dataset_File.objects.get(id=self.datafiles[5].id)
expect(datafile.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(datafile.getParameterSets()[0])
expect(psm.get_param('softmax_version', True)).to_equal('5.4.52.1.0')
expect(psm.get_param('experiment_name', True)).to_equal('Exp01')
expect(psm.get_param('analysis_notes', True)).to_equal(
u'Revision_101: PROTOCOL REVISION HISTORY:\rv1.0.0: original protocol created (MDC)\rv1.0.1: 06/30/05 - Updated, spell checked, & formatted to new style guide. (DW)\r\rREADER SUITABILITY:\r\nEMax, VMax, ThermoMax, VersaMax, SpectraMax, SpectraMax Plus, SpectraMax Plus 384, SpectraMax 190, SpectraMax 340PC, SpectraMax 340PC 384, SpectraMax M2, SpectraMax M5. Intro: MIPS resupply 3759, 3720 latin square\rcells seeded 48hrs prior using FTA. Cells were 95-100% confluent on the day of the experiment.\rColumns 1,3,5,7,9,11 are non-transfected HEK cells, columns 2,4,6,8,10,12 are hTRPV4 HEK.\rCells were induced the evening prior to assay with 0.1\xb5g/ml tet \rLoaded with FURA-2 at 805 Inhibitor at 900\rLatin square\r 1 2 3 4 5 6 \ra x x x x x x\rb c z y x w v \rc v c z y x w\rd w v c z y x\re x w v c z y\rf y x w v c z\rg z y x w v c\rh x x x x x x\rc control V vehicle 0.1% DMSO, w 3759 10\xb5M x 3759 1\xb5M y 3720 10\xb5M z 3720 1\xb5M\rInjection 1 at 15"\rSLIGRL 30\xb5M\rInjection 2 at 80" \rGSK 30nM '
)
expect(psm.get_param(
'instrument_info',
True)).to_equal('Flexstation III ROM v3.0.22 16Feb11')
#expect(psm.get_param('plate_read_time', True)).to_equal('')
#expect(psm.get_param('read_type', True)).to_equal('')
#expect(psm.get_param('data_mode', True)).to_equal('')
#expect(psm.get_param('data_type', True)).to_equal('')
expect(psm.get_param('strips', True)).to_equal('1-12')
expect(psm.get_param('trans', True)).to_equal(
u'Trans1: H=80\xb5, R=4, V=20.0\xb5, \x4015. Trans2: H=100\xb5, R=4, V=25.0\xb5, \x4080'
)
expect(psm.get_param('kinetic_points', True)).to_equal(39.0)
expect(psm.get_param('kinetic_flex_read_time', True)).to_equal(150.0)
expect(psm.get_param('kinetic_flex_interval', True)).to_equal(3.9)
expect(psm.get_param('number_of_wavelengths', True)).to_equal(2)
expect(psm.get_param('read_wavelength', True)).to_equal('520 520')
expect(psm.get_param('number_of_wells_or_cuvette',
True)).to_equal(96.0)
expect(psm.get_param('excitation_wavelengths',
True)).to_equal('340 380')
def test_save_metadata(self):
from os import path
try:
from tardis.apps.microtardis.filters.exiftags import EXIFTagsFilter
except:
raise SkipTest()
exp = models.Experiment(title='exp: test exif filter',
institution_name='rmit',
approved=True,
created_by=self.user,
public=False)
exp.save()
logger.debug("experiment %s" % exp)
self.assertEqual(exp.get_or_create_directory(),
path.join(settings.FILE_STORE_PATH, str(exp.id)))
self.experiment_path = path.join(settings.FILE_STORE_PATH, str(exp.id))
dataset = models.Dataset(description="dataset description...", experiment=exp)
dataset.save()
## FEIQuanta200 TIF image testing ##
filename = path.join(path.abspath(path.dirname(__file__)), 'testing/Quanta200/test.tif')
logger.debug("filename %s" % filename)
df_file = models.Dataset_File(dataset=dataset, filename='test.tif', url=filename, protocol='staging')
df_file.save()
#self.assertEqual("",models.Schema.objects.all())
sch = models.Schema.objects.get(name="Quanta200_EXIF")
self.assertEqual("Quanta200_EXIF",sch.name)
datafileparameterset = models.DatafileParameterSet.objects.get(schema=sch, dataset_file=df_file)
psm = ParameterSetManager(parameterset=datafileparameterset)
self.assertEqual(14,len(psm.parameters))
self.assertEqual(str(psm.get_param("[User] UserText").string_value), "R-2")
self.assertEqual(str(psm.get_param("[User] Date").string_value), "11/09/2010")
self.assertEqual(str(psm.get_param("[User] Time").string_value), "03:28PM")
self.assertEqual(str(psm.get_param("[Beam] HV").numerical_value), "25.0")
self.assertEqual(str(psm.get_param("[Beam] Spot").numerical_value), "5.0")
self.assertEqual(str(psm.get_param("[Scan] HorFieldSize").numerical_value), "2.13333e-05")
self.assertEqual(str(psm.get_param("[Stage] WorkingDistance").numerical_value), "5.2")
self.assertEqual(str(psm.get_param("[Vacuum] UserMode").string_value), "Lowvacuum")
self.assertEqual(str(psm.get_param("[Vacuum] CHPressure").string_value), "0.9976 Torr (1.33 mbar)")
self.assertEqual(str(psm.get_param("[Detectors] Name").string_value), "Lfd")
self.assertEqual(str(psm.get_param("[Lfd] Contrast").numerical_value), "88.2")
self.assertEqual(str(psm.get_param("[Lfd] Brightness").numerical_value), "21.2")
## nanoSEM TIF image testing ##
filename = path.join(path.abspath(path.dirname(__file__)), 'testing/NovaNanoSEM/test.tif')
logger.debug("filename %s" % filename)
df_file = models.Dataset_File(dataset=dataset, filename='test.tif', url=filename, protocol='staging')
df_file.save()
#self.assertEqual("",models.Schema.objects.all())
sch = models.Schema.objects.get(name="NovaNanoSEM_EXIF")
self.assertEqual("NovaNanoSEM_EXIF",sch.name)
datafileparameterset = models.DatafileParameterSet.objects.get(schema=sch, dataset_file=df_file)
psm = ParameterSetManager(parameterset=datafileparameterset)
self.assertEqual(14,len(psm.parameters))
self.assertEqual(str(psm.get_param("[User] UserText").string_value), "")
self.assertEqual(str(psm.get_param("[User] Date").string_value), "03/29/2011")
self.assertEqual(str(psm.get_param("[User] Time").string_value), "10:10:52 AM")
self.assertEqual(str(psm.get_param("[Beam] HV").numerical_value), "15.0")
self.assertEqual(str(psm.get_param("[Beam] Spot").numerical_value), "3.5")
self.assertEqual(str(psm.get_param("[Scan] HorFieldSize").numerical_value), "0.00018592")
self.assertEqual(str(psm.get_param("[Stage] WorkingDistance").numerical_value), "4.9")
self.assertEqual(str(psm.get_param("[Vacuum] UserMode").string_value), "High vacuum")
self.assertEqual(str(psm.get_param("[Vacuum] CHPressure").string_value), "1.589E-05 Torr (2.118E-05 mbar)")
self.assertEqual(str(psm.get_param("[Detectors] Name").string_value), "TLD")
self.assertEqual(str(psm.get_param("[TLD] Contrast").numerical_value), "68.1")
self.assertEqual(str(psm.get_param("[TLD] Brightness").numerical_value), "51.6")
def testFlexstationAllFields(self):
"""
Simple test running the filter and making sure the Softmax version number was saved
"""
filter = FlexstationFilter("Flexstation Test Schema", "http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
# Check a parameter set was created for the datafile
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(datafile.getParameterSets()[0])
expect(psm.get_param('softmax_version', True)).to_equal('5.42.1.0')
expect(psm.get_param('experiment_name', True)).to_equal('Experiment#1')
expect(psm.get_param('analysis_notes', True)).to_equal(u'Notes#1: TRPV1 Phosphate mutants\rCells seeded 48hrs prior 40K cel/well \rInduced with tetracycline for 3 hrs washed once with hepes (50microL/well) then loaded with fura2 for 1 hr (50 microL/well). then washed twice with 60microl or HEPES buffer per well finaly loaded with 60microl of hepes.\rCells:\rcolumn 1: Nt, 2: WtV1, 3: C1, 4: C2, 5: C3, 6: C4, 7: C5, 8: N1, 9: N6\rinjection 1: rows A-D buffer only, E-H 100microM SLIGRL\rinjection 2: Row A,E DMSO 1%, B,F 1microM CAPS, C,G 10microM CAPS, D,H 100microM caps')
expect(psm.get_param('instrument_info', True)).to_equal('Flexstation III ROM v2.1.35 20May09')
#expect(psm.get_param('plate_read_time', True)).to_equal('')
#expect(psm.get_param('read_type', True)).to_equal('')
#expect(psm.get_param('data_mode', True)).to_equal('')
#expect(psm.get_param('data_type', True)).to_equal('')
expect(psm.get_param('strips', True)).to_equal('1-9')
expect(psm.get_param('trans', True)).to_equal(u'Trans1: H=80\xb5, R=4, V=20.0\xb5, \x4015. Trans2: H=100\xb5, R=4, V=20.0\xb5, \x40115')
expect(psm.get_param('kinetic_points', True)).to_equal(65.0)
expect(psm.get_param('kinetic_flex_read_time', True)).to_equal(250.0)
expect(psm.get_param('kinetic_flex_interval', True)).to_equal(3.9)
expect(psm.get_param('number_of_wavelengths', True)).to_equal(2)
expect(psm.get_param('read_wavelength', True)).to_equal('520 520')
expect(psm.get_param('number_of_wells_or_cuvette', True)).to_equal(96.0)
expect(psm.get_param('excitation_wavelengths', True)).to_equal('340 380')
def get_subject(ps, type_):
psm = ParameterSetManager(ps)
return { 'text': psm.get_param('code', True),
'type': type_ }
