def testPersisterAgainstPicasa(self):
'''
Test against Picasa - useful for development testing
'''
# Build a persister with "make_local_copy" mocked out
# (file transfer isn't part of this test)
persister = flexmock(AtomPersister())
persister.should_receive('make_local_copy').times(50)
doc = feedparser.parse('picasa_example.atom')
# Process the first twenty entries
for entry in list(reversed(doc.entries))[0:20]:
persister.process(doc.feed, entry)
# We processed 20 images (Picasa only has one image per dataset)
eq_(Dataset.objects.count(), 20)
# This part has 2 users
eq_(User.objects.count(), 2)
# This part covers 6 albums
eq_(Experiment.objects.count(), 6)
for experiment in Experiment.objects.all():
pset = experiment.getParameterSets().get(schema=AtomImportSchemas. \
get_schema(Schema.EXPERIMENT))
pset_mgr = ParameterSetManager(pset)
assert experiment.title != pset_mgr.get_param('ExperimentID')
# Change the experiment titles, to check this won't be a problem
experiment.title = "Title removed for testing"
experiment.save()
# Process the rest of the entries
for entry in list(reversed(doc.entries))[20:]:
persister.process(doc.feed, entry)
# We processed 50 images (Picasa only has one image per dataset)
eq_(Dataset.objects.count(), 50)
# This part has 2 users
eq_(User.objects.count(), 4)
# This part covers 6 albums
eq_(Experiment.objects.count(), 9)
def _create_experiment_id_parameter_set(self, experiment, experimentId):
'''
Adds ExperimentID field to dataset schema
'''
namespace = AtomImportSchemas.get_schema(Schema.EXPERIMENT).namespace
mgr = ParameterSetManager(parentObject=experiment, schema=namespace)
mgr.new_param(IngestOptions.PARAM_EXPERIMENT_ID, experimentId)
def get_or_create_cc_parameterset(self, create=True):
"""
Gets the creative commons parameterset for the experiment
:param create: If true, creates a new parameterset object to
hold the cc license if one doesn't exist
:type create: boolean
:return: The parameterset manager for the cc parameterset
:rtype: :class:`tardis.tardis_portal.ParameterSetManager.
ParameterSetManager`
"""
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=self.schema,
experiment__id=self.experiment_id)
if not len(parameterset):
if create:
experiment = Experiment.objects.get(id=self.experiment_id)
self.psm = ParameterSetManager(schema=self.schema,
parentObject=experiment)
else:
return None
else:
self.psm = ParameterSetManager(parameterset=parameterset[0])
return self.psm
def testHandlesMultipleEntries(self):
from ..views import SCHEMA_URI
param_list = ({'type': 'website',
'identifier': 'https://www.example.test/%d' % i,
'title': 'Title #%d' % i,
'notes': 'This is note #%d.' % i} for i in range(10))
for params in param_list:
psm = ParameterSetManager(parentObject=self.experiment,
schema=SCHEMA_URI)
for k, v in params.items():
psm.set_param(k, v)
response = self.client.get(
reverse('tardis.apps.related_info.views.'
+ 'list_or_create_related_info',
args=[self.experiment.id]))
expect(response.status_code).to_equal(200)
expect(response['Content-Type'])\
.to_equal('application/json; charset=utf-8')
objs = json.loads(response.content)
expect(len(objs)).to_be(10)
for obj in objs:
expect(obj['type']).to_equal('website')
expect(obj['identifier']).to_match(r'www.example.test/\d+$')
expect(obj['title']).to_match(r'^Title #\d+$')
expect(obj['notes']).to_match(r'note #\d+\.$')
def _create_experiment(user, bad):
experiment = Experiment(title='Norwegian Blue',
description='Parrot + 40kV',
created_by=user)
experiment.public_access = Experiment.PUBLIC_ACCESS_METADATA
experiment.save()
experiment.author_experiment_set.create(
order=0, author="John Cleese", url="http://nla.gov.au/nla.party-1")
experiment.author_experiment_set.create(
order=1, author="Michael Palin", url="http://nla.gov.au/nla.party-2")
psm = ParameterSetManager(parentObject=experiment, schema=SCHEMA_URI)
if bad:
params = {
'type': 'website',
'identifier': 'https://www.badexample.com/'
}
else:
params = {
'type': 'website',
'identifier': 'https://www.example.com/',
'title': 'Google',
'notes': 'This is a note.'
}
for k, v in params.items():
psm.set_param(k, v)
return experiment
def testHandlesSingleEntry(self):
from ..views import SCHEMA_URI
psm = ParameterSetManager(parentObject=self.experiment,
schema=SCHEMA_URI)
params = {
'type': 'website',
'identifier': 'https://www.google.com/',
'title': 'Google',
'notes': 'This is a note.'
}
for k, v in params.items():
psm.set_param(k, v)
response = self.client.get(
reverse('tardis.apps.related_info.views.' +
'list_or_create_related_info',
args=[self.experiment.id]))
expect(response.status_code).to_equal(200)
expect(response['Content-Type'])\
.to_equal('application/json; charset=utf-8')
objs = json.loads(response.content)
expect(len(objs)).to_be(1)
for k in params.keys():
expect(objs[0][k]).to_equal(params[k])
def get_or_create_cc_parameterset(self, create=True):
"""
Gets the creative commons parameterset for the experiment
:param create: If true, creates a new parameterset object to
hold the cc license if one doesn't exist
:type create: boolean
:return: The parameterset manager for the cc parameterset
:rtype: :class:`tardis.tardis_portal.ParameterSetManager.
ParameterSetManager`
"""
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=self.schema,
experiment__id=self.experiment_id)
if not len(parameterset):
if create:
experiment = Experiment.objects.get(id=self.experiment_id)
self.psm = ParameterSetManager(schema=self.schema,
parentObject=experiment)
else:
return None
else:
self.psm = ParameterSetManager(parameterset=parameterset[0])
return self.psm
def test_link_parameter_type(self):
"""
Test that Parameter.link_gfk (GenericForeignKey) is correctly
assigned after using Parameter.set_value(some_url) for a LINK Parameter.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
# Check link to experiment
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.assertTrue(psm.get_param("exp_link").string_value == exp_url)
self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
# Check link to dataset
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.assertTrue(
psm.get_param("dataset_link").string_value == dataset_url)
self.assertTrue(
psm.get_param("dataset_link").link_id == self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
def _get_params(self, key, namespace, experiment):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=namespace, experiment__id=experiment.id)
if parameterset:
psm = ParameterSetManager(parameterset=parameterset[0])
return psm.get_params(key, True)
return []
def get_or_create_unique_parameterset(self, schema, parametername, value):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=schema,
experiment__id=self.experiment_id)
experiment = Experiment.objects.get(id=self.experiment_id)
psm = None
if not len(parameterset):
psm = ParameterSetManager(schema=schema,
parentObject=experiment)
return psm
else:
for ps in parameterset:
psm = ParameterSetManager(parameterset=ps)
try:
ps_value = psm.get_param("party_id",
True)
if value == ps_value:
return psm
except ExperimentParameter.DoesNotExist:
pass # keep going
psm = ParameterSetManager(schema=schema,
parentObject=experiment)
return psm
def _test_metadata(self,schema,name,dataset,fields):
""" Check that metadata is correct"""
try:
sch = models.Schema.objects.get(namespace__exact=schema,name=name)
except Schema.DoesNotExist:
self.assertTrue(False)
self.assertEqual(schema,sch.namespace)
self.assertEqual(name,sch.name)
try:
datasetparameterset = models.DatasetParameterSet.objects.get(schema=sch, dataset=dataset)
except DatasetParameterSet.DoesNotExist:
self.assertTrue(False)
psm = ParameterSetManager(parameterset=datasetparameterset)
for key, field_type, value in fields:
logger.debug("key=%s,field_type=%s,value=%s" % (key,field_type, value))
try:
# First check stringed value
param = psm.get_param(key,value=True)
self.assertEquals(str(param),str(value))
# Then correct type
param = psm.get_param(key,value=False)
self.assertEquals(param.name.data_type,field_type)
except DatasetParameter.DoesNotExist:
logger.error("cannot find %s" % key)
self.assertTrue(False)
def _get_params(self, key, namespace, experiment):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=namespace,
experiment__id=experiment.id)
if parameterset:
psm = ParameterSetManager(parameterset=parameterset[0])
return psm.get_params(key, True)
return []
def _get_dict_from_ps(self, ps):
'''
Build dictionary by getting the parameter values from the keys, then
zipping it all together.
'''
psm = ParameterSetManager(ps)
return dict([('id', ps.id)] + # Use set ID
zip(self.parameter_names, (psm.get_param(k, True)
for k in self.parameter_names)))
def _get_dict_from_ps(self, ps):
'''
Build dictionary by getting the parameter values from the keys, then
zipping it all together.
'''
psm = ParameterSetManager(ps)
return dict([('id', ps.id)]+ # Use set ID
zip(self.parameter_names,
(psm.get_param(k, True) for k in self.parameter_names)))
def save_datafile_add_form(schema, parentObject, request):
psm = ParameterSetManager(schema=schema, parentObject=parentObject)
for key, value in sorted(request.POST.iteritems()):
if value:
stripped_key = key.replace('_s47_', '/')
stripped_key = stripped_key.rpartition('__')[0]
psm.new_param(stripped_key, value)
def test_tz_naive_date_handling(self):
"""
Ensure that dates are handling in a timezone-aware way.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", str(datetime(1970, 01, 01, 10, 0, 0)))
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def test_tz_naive_date_handling(self):
'''
Ensure that dates are handling in a timezone-aware way.
'''
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", datetime(1970,01,01,10,0,0))
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970,01,01,0,0,0,tzinfo=pytz.utc))
def test_existing_parameterset(self):
psm = ParameterSetManager(parameterset=self.datafileparameterset)
self.assertTrue(psm.get_schema().namespace ==
"http://localhost/psmtest/df/")
self.assertTrue(psm.get_param("parameter1").string_value == "test1")
self.assertTrue(psm.get_param("parameter2", True) == 2)
def test_tz_aware_date_handling(self):
"""
Ensure that dates are handling in a timezone-aware way.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", '1970-01-01T08:00:00+08:00')
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def test_existing_parameterset(self):
psm = ParameterSetManager(parameterset=self.datafileparameterset)
self.assertTrue(psm.get_schema().namespace\
== "http://localhost/psmtest/df/")
self.assertTrue(psm.get_param("parameter1").string_value == "test1")
self.assertTrue(psm.get_param("parameter2", True) == 2)
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ['type','identifier','title','notes']
try:
return dict([('id', ps.id)] + # Use set ID
zip(parameter_names,
(psm.get_param(k, True) \
for k in parameter_names)))
except ExperimentParameter.DoesNotExist:
return dict() # drop Related_Info record with missing fields
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ['type','identifier','title','notes']
try:
return dict([('id', ps.id)] + # Use set ID
zip(parameter_names,
(psm.get_param(k, True) \
for k in parameter_names)))
except ExperimentParameter.DoesNotExist:
return dict() # drop Related_Info record with missing fields
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ["type", "identifier", "title", "notes"]
try:
return dict(
[("id", ps.id)]
+ zip(parameter_names, (psm.get_param(k, True) for k in parameter_names)) # Use set ID
)
except ExperimentParameter.DoesNotExist:
return dict() # drop Related_Info record with missing fields
def save_datafile_add_form(schema, parentObject, request):
psm = ParameterSetManager(schema=schema, parentObject=parentObject)
for key, value in sorted(request.POST.iteritems()):
if value:
stripped_key = key.replace("_s47_", "/")
stripped_key = stripped_key.rpartition("__")[0]
psm.new_param(stripped_key, value)
def test_link_parameter_type(self):
"""
Test that Parameter.link_gfk (GenericForeignKey) is correctly
assigned after using Parameter.set_value(some_url) for a LINK Parameter.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
# Check link to experiment
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.assertTrue(psm.get_param("exp_link").string_value ==
exp_url)
self.assertTrue(psm.get_param("exp_link").link_id ==
self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
# Check link to dataset
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.assertTrue(psm.get_param("dataset_link").string_value ==
dataset_url)
self.assertTrue(psm.get_param("dataset_link").link_id ==
self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
def get_related_info(ps):
psm = ParameterSetManager(ps)
parameter_names = ['type', 'identifier', 'title', 'notes']
try:
parameters = {
key: psm.get_param(key, True)
for key in parameter_names}
parameters['id'] = ps.id
return parameters
except ExperimentParameter.DoesNotExist:
return dict() # drop Related_Info record with missing fields
def _get_param(self, key, namespace, experiment):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=namespace, experiment__id=experiment.id)
if len(parameterset) > 0:
psm = ParameterSetManager(parameterset=parameterset[0])
try:
return psm.get_param(key, True)
except MultipleObjectsReturned:
return psm.get_params(key, True)
except ObjectDoesNotExist:
return None
def save_datafile_edit_form(parameterset, request):
psm = ParameterSetManager(parameterset=parameterset)
psm.delete_all_params()
for key, value in sorted(request.POST.iteritems()):
if value:
stripped_key = key.replace('_s47_', '/')
stripped_key = stripped_key.rpartition('__')[0]
psm.new_param(stripped_key, value)
def test_tz_aware_date_handling(self):
'''
Ensure that dates are handling in a timezone-aware way.
'''
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3",
iso8601.parse_date('1970-01-01T08:00:00+08:00'))
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970,01,01,0,0,0,tzinfo=pytz.utc))
def _get_param(self, key, namespace, experiment):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=namespace,
experiment__id=experiment.id)
if len(parameterset) > 0:
psm = ParameterSetManager(parameterset=parameterset[0])
try:
return psm.get_param(key, True)
except MultipleObjectsReturned:
return psm.get_params(key, True)
except ObjectDoesNotExist:
return None
def testPersisterStoresEntryMetadata(self):
# Create user to associate with dataset
user = User(username="******")
user.save()
feed, entry = self._getTestEntry()
p = AtomPersister()
dataset = p.process(feed, entry)
parameterset = dataset.getParameterSets().get(schema=AtomImportSchemas.get_schema(Schema.DATASET))
expect(parameterset) != None
psm = ParameterSetManager(parameterset)
expect(psm.get_param("EntryID").get()).to_equal(entry.id)
expect(psm.get_param("Updated").name.isDateTime()).to_be_truthy()
# Compare against non-timezoned update time
expect(psm.get_param("Updated", True)).to_equal(iso8601.parse_date(entry.updated))
def testFilter(self):
DataGrabberFilter()(None, instance=self.datafiles[0])
# Check a DATASET parameter set was created
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(dataset.getParameterSets()[0])
expect(psm.get_param('admin-filename', True))\
.to_equal(self.datafiles[0].filename)
expect(psm.get_param('user_name', True)).to_equal('s.crawley')
expect(psm.get_param('facility_name', True)).to_equal('QBP NIKON TIE')
expect(psm.get_param('account_name', True)).to_equal('200 - CMM STAFF')
expect(psm.get_param('session_uuid', True)).to_equal(
'cbc86da5-e143-4a11-9951-abaffef70efa')
expect(psm.get_param('record_uuid', True)).to_equal(
'ddf0df63-d985-4ffa-9102-e34a78b3fd1f')
# Repeat for the datafile parameters
expect(self.datafiles[0].getParameterSets().count()).to_equal(0)
expect(self.datafiles[1].getParameterSets().count()).to_equal(1)
psm = ParameterSetManager(self.datafiles[1].getParameterSets()[0])
expect(psm.get_param('instrument_pathname', True)).to_equal(
'S:\\Garry\\testfile.txt')
# Check we won't create a duplicate dataset
DataGrabberFilter()(None, instance=self.datafiles[0])
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
def _get_or_create_publish_parameterset(self, create):
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=self.schema,
experiment__id=self.experiment_id)
if len(parameterset) == 1:
psm = ParameterSetManager(parameterset=parameterset[0])
elif create:
experiment = Experiment.objects.get(id=self.experiment_id)
psm = ParameterSetManager(schema=self.schema,
parentObject=experiment)
psm.new_param(self.access_type_key, UNPUBLISHED)
else:
psm = None
return psm
def testPersisterStoresEntryMetadata(self):
# Create user to associate with dataset
user = User(username="******")
user.save()
feed, entry = self._getTestEntry()
p = AtomPersister()
dataset = p.process(feed, entry)
parameterset = dataset.getParameterSets() \
.get(schema=AtomImportSchemas. \
get_schema(Schema.DATASET))
expect(parameterset) != None
psm = ParameterSetManager(parameterset)
expect(psm.get_param('EntryID').get()).to_equal(entry.id)
expect(psm.get_param('Updated').name.isDateTime()).to_be_truthy()
# Compare against non-timezoned update time
expect(psm.get_param('Updated', True)) \
.to_equal(iso8601.parse_date(entry.updated))
def save_metadata(self, datafile, schema, metadata):
psm = ParameterSetManager(parentObject=datafile.dataset,
schema=schema.namespace)
psm.set_param('metadata-filename', datafile.filename)
for key, value in metadata:
try:
psm.set_param(key, value)
except ValueError, e:
pn = ParameterName.objects.get(name=key, schema=schema)
psm.set_param(key, value.strip(pn.units))
def get_or_create_cc_parameterset(self, create=True):
# get cc license parameterset, if any
parameterset = ExperimentParameterSet.objects.filter(
schema__namespace=self.schema,
experiment__id=self.experiment_id)
if not len(parameterset):
if create:
experiment = Experiment.objects.get(id=self.experiment_id)
self.psm = ParameterSetManager(schema=self.schema,
parentObject=experiment)
else:
return None
else:
self.psm = ParameterSetManager(parameterset=parameterset[0])
return self.psm
def testFlexstationSimple(self):
"""
Simple test running the filter and making sure the Softmax version number was saved
"""
filter = FlexstationFilter("Flexstation Test Schema", "http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
# Check a parameter set was created for the datafile
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
# Check that at least the verion number was extracted
psm = ParameterSetManager(datafile.getParameterSets()[0])
expect(psm.get_param('softmax_version', True)).to_equal('5.42.1.0')
# Check we won't create a duplicate datafile
filter = FlexstationFilter("Flexstation Test Schema", "http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
def save_metadata(self, datafile, schema, metadata):
psm = ParameterSetManager(parentObject=datafile.dataset, schema=schema.namespace)
psm.set_param("metadata-filename", datafile.filename)
for key, value in metadata:
try:
psm.set_param(key, value)
except ValueError, e:
pn = ParameterName.objects.get(name=key, schema=schema)
psm.set_param(key, value.strip(pn.units))
def source_path(datafile, exclude=[], strip='S:\\',
windowsPath=True, rootdir=''):
for ps in datafile.getParameterSets():
if ps.schema.namespace != DataGrabberFilter.SCHEMA2:
continue
psm = ParameterSetManager(ps)
try:
pathname = psm.get_param('instrument_pathname', value=True)
if pathname.startswith(strip):
pathname = pathname[len(strip):]
if windowsPath:
pathname = pathname.replace('\\', '/')
for pattern in exclude:
regex = compile(pattern)
if regex.match(pathname):
return None
return pathname
except DatafileParameter.DoesNotExist:
pass
return None
def testJEOLComplex(self):
JEOLSEMFilter()(None, instance=self.datafiles[1])
# Check a parameter set was created
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(dataset.getParameterSets()[0])
expect(psm.get_param('metadata-filename', True))\
.to_equal(self.datafiles[1].filename)
expect(psm.get_param('instrument', True)).to_equal('7001F_TTL')
expect(psm.get_param('accel_volt', True)).to_equal(15.0)
expect(psm.get_param('micron_bar', True)).to_equal(213)
expect(psm.get_param('micron_marker', True)).to_equal(100)
# Check we won't create a duplicate dataset
JEOLSEMFilter()(None, instance=self.datafiles[1])
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
def testHandlesFound(self):
from ..views import SCHEMA_URI
psm = ParameterSetManager(parentObject=self.experiment,
schema=SCHEMA_URI)
params = {'type': 'website',
'identifier': 'https://www.google.com/',
'title': 'Google',
'notes': 'This is a note.'}
for k, v in params.items():
psm.set_param(k, v)
response = self.client.get(
reverse('tardis.apps.related_info.views.' +
'get_or_update_or_delete_related_info',
args=[self.experiment.id, psm.parameterset.id]))
expect(response.status_code).to_equal(200)
obj = json.loads(response.content)
for k, v in params.items():
expect(obj[k]).to_equal(v)
def get_existing_ldap_party_info(self):
pais = PartyActivityInformationService()
pai = pais.get_pai()
eps = ExperimentParameter.objects.filter(name__name='party_id',
parameterset__schema__namespace='http://localhost/pilot/party/1.0/',
parameterset__experiment__id=self.experiment_id)
party_info = []
for ep in eps:
psm = ParameterSetManager(parameterset=ep.parameterset)
display_name = pai.get_display_name_for_party(ep.string_value)
info = {}
info['party_id'] = ep.string_value
info['party_fullname'] = display_name
info['relation'] = psm.get_param('relationToCollection', True)
party_info.append(info)
return party_info
def testFlexstationSimple(self):
"""
Simple test running the filter and making sure the Softmax version number was saved
"""
filter = FlexstationFilter("Flexstation Test Schema",
"http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
# Check a parameter set was created for the datafile
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
# Check that at least the verion number was extracted
psm = ParameterSetManager(datafile.getParameterSets()[0])
expect(psm.get_param('softmax_version', True)).to_equal('5.42.1.0')
# Check we won't create a duplicate datafile
filter = FlexstationFilter("Flexstation Test Schema",
"http://rmit.edu.au/flexstation_test")
filter.__call__(None, instance=self.datafiles[0])
datafile = Dataset_File.objects.get(id=self.datafiles[0].id)
expect(datafile.getParameterSets().count()).to_equal(1)
def _create_entry_parameter_set(self, dataset, entryId, updated):
'''
Creates or updates schema for dataset with populated 'EntryID' and 'Updated' fields
'''
schema = AtomImportSchemas.get_schema(Schema.DATASET)
# I'm not sure why mgr.set_param always creates additional parametersets. Anyway
# we can't use it. --SB.
try:
p = DatasetParameter.objects.get(parameterset__dataset=dataset, parameterset__schema=schema,
name__name=IngestOptions.PARAM_ENTRY_ID)
except DatasetParameter.DoesNotExist:
mgr = ParameterSetManager(parentObject=dataset, schema=schema.namespace)
mgr.new_param(IngestOptions.PARAM_ENTRY_ID, entryId)
try:
p = DatasetParameter.objects.get(parameterset__dataset=dataset, parameterset__schema=schema,
name__name=IngestOptions.PARAM_UPDATED)
i=iso8601.parse_date(updated)
l=get_local_time_naive(i)
p.datetime_value = l
p.save()
except DatasetParameter.DoesNotExist:
mgr = ParameterSetManager(parentObject=dataset, schema=schema.namespace)
t = get_local_time_naive(iso8601.parse_date(updated))
logging.getLogger(__name__).debug("Setting update parameter with datetime %s" % t)
mgr.new_param(IngestOptions.PARAM_UPDATED, t)
def _create_related_info(request, experiment_id):
if not authz.has_write_permissions(request, experiment_id):
return return_response_error(request)
form = RelatedInfoForm(json.loads(request.body))
if not form.is_valid():
return HttpResponse('', status=400)
ps = ExperimentParameterSet(experiment_id=experiment_id,
schema=_get_schema())
ps.save()
ParameterSetManager(ps).set_params_from_dict(form.cleaned_data)
return HttpResponse(json.dumps(_get_dict_from_ps(ps)),
content_type='application/json; charset=utf-8',
status=201)
def _create(self, request, experiment_id):
from tardis.tardis_portal.auth.decorators import has_experiment_write
if not has_experiment_write(request, experiment_id):
return return_response_error(request)
form = self.form_cls(json.loads(request.body))
if not form.is_valid():
return HttpResponse('', status=400)
ps = ExperimentParameterSet(experiment_id=experiment_id,
schema=self.schema)
ps.save()
ParameterSetManager(ps).set_params_from_dict(form.cleaned_data)
return HttpResponse(json.dumps(self._get_dict_from_ps(ps)),
content_type='application/json; charset=utf-8',
status=201)
def testHandlesSingleEntry(self):
from ..views import SCHEMA_URI
psm = ParameterSetManager(parentObject=self.experiment,
schema=SCHEMA_URI)
params = {'type': 'website',
'identifier': 'https://www.google.com/',
'title': 'Google',
'notes': 'This is a note.'}
for k, v in params.items():
psm.set_param(k, v)
response = self.client.get(
reverse('tardis.apps.related_info.views.' +
'list_or_create_related_info',
args=[self.experiment.id]))
expect(response.status_code).to_equal(200)
expect(response['Content-Type'])\
.to_equal('application/json; charset=utf-8')
objs = json.loads(response.content)
expect(len(objs)).to_be(1)
for k, v in params.items():
expect(objs[0][k]).to_equal(v)
def _create_experiment(user, bad):
experiment = Experiment(title='Norwegian Blue',
description='Parrot + 40kV',
created_by=user)
experiment.public_access = Experiment.PUBLIC_ACCESS_METADATA
experiment.save()
experiment.experimentauthor_set.create(order=0,
author="John Cleese",
url="http://nla.gov.au/nla.party-1")
experiment.experimentauthor_set.create(order=1,
author="Michael Palin",
url="http://nla.gov.au/nla.party-2")
psm = ParameterSetManager(parentObject=experiment, schema=SCHEMA_URI)
if bad:
params = {'type': 'website',
'identifier': 'https://www.badexample.com/'}
else:
params = {'type': 'website',
'identifier': 'https://www.example.com/',
'title': 'Google',
'notes': 'This is a note.'}
for k, v in params.items():
psm.set_param(k, v)
return experiment
def save_datafile_edit_form(parameterset, request):
psm = ParameterSetManager(parameterset=parameterset)
psm.delete_all_params()
for key, value in sorted(request.POST.iteritems()):
if value:
stripped_key = key.replace('_s47_', '/')
stripped_key = stripped_key.rpartition('__')[0]
psm.new_param(stripped_key, value)
def _update_related_info(request, experiment_id, related_info_id):
if not authz.has_write_permissions(request, experiment_id):
return return_response_error(request)
form = RelatedInfoForm(json.loads(request.body))
if not form.is_valid():
return HttpResponse('', status=400)
try:
ps = ExperimentParameterSet.objects.get(experiment_id=experiment_id,
id=related_info_id)
except ExperimentParameterSet.DoesNotExist:
return HttpResponse('', status=404)
ParameterSetManager(ps).set_params_from_dict(form.cleaned_data)
return HttpResponse(json.dumps(_get_dict_from_ps(ps)),
content_type='application/json; charset=utf-8',
status=201)
def test_link_parameter_type_extra(self):
# make a second ParameterSet for testing some variations
# in URL values
self.datafileparameterset2 = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset2.save()
psm = ParameterSetManager(parameterset=self.datafileparameterset2)
self.dataset_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_dataset_link)
# /dataset/1 - no trailing slash
dataset_url = self.dataset.get_absolute_url()
self.dataset_link_param2.set_value(dataset_url)
self.dataset_link_param2.save()
# Check link_id/link_ct/link_gfk to dataset
self.assertTrue(
psm.get_param("dataset_link").link_id == self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
# Test links of the form /api/v1/experiment/<experiment_id>/
self.exp_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_exp_link)
exp_url = '/api/v1/experiment/%s/' % self.exp.id
self.exp_link_param2.set_value(exp_url)
self.exp_link_param2.save()
# Check link_id/link_ct/link_gfk to experiment
self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
def test_unresolvable_link_parameter(self):
"""
Test that LINK Parameters that can't be resolved to a model (including
non-URL values) still work.
"""
self.datafileparameterset3 = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset3.save()
psm = ParameterSetManager(parameterset=self.datafileparameterset3)
# Create a Parameter of type LINK to an unresolvable (non-URL)
# free-text value
self.freetext_link_param = DatafileParameter(
parameterset=self.datafileparameterset3,
name=self.parametername_unresolvable_link)
self.assertRaises(
SuspiciousOperation,
lambda: self.freetext_link_param.set_value("FREETEXT_ID_123"))
def testJEOLComplex(self):
JEOLSEMFilter()(None, instance=self.datafiles[1])
# Check a parameter set was created
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
# Check all the expected parameters are there
psm = ParameterSetManager(dataset.getParameterSets()[0])
expect(psm.get_param('metadata-filename', True))\
.to_equal(self.datafiles[1].filename)
expect(psm.get_param('instrument', True)).to_equal('7001F_TTL')
expect(psm.get_param('accel_volt', True)).to_equal(15.0)
expect(psm.get_param('micron_bar', True)).to_equal(213)
expect(psm.get_param('micron_marker', True)).to_equal(100)
# Check we won't create a duplicate dataset
JEOLSEMFilter()(None, instance=self.datafiles[1])
dataset = Dataset.objects.get(id=self.dataset.id)
expect(dataset.getParameterSets().count()).to_equal(1)
def create_parameterset_edit_form(parameterset, request=None):
from tardis.tardis_portal.models import ParameterName
# if POST data to save
if request:
from django.utils.datastructures import SortedDict
fields = SortedDict()
for key, value in sorted(request.POST.iteritems()):
x = 1
stripped_key = key.replace('_s47_', '/')
stripped_key = stripped_key.rpartition('__')[0]
parameter_name = ParameterName.objects.get(
schema=parameterset.schema, name=stripped_key)
units = ""
if parameter_name.units:
units = " (" + parameter_name.units + ")"
# if not valid, spit back as exact
if parameter_name.isNumeric():
fields[key] = \
forms.DecimalField(label=parameter_name.full_name + units,
required=False,
initial=value)
elif parameter_name.isLongString():
fields[key] = \
forms.CharField(widget=forms.Textarea, label=parameter_name.full_name + units,
max_length=255, required=False,
initial=value)
else:
fields[key] = \
forms.CharField(label=parameter_name.full_name + units,
max_length=255, required=False,
initial=value)
return type('DynamicForm', (forms.BaseForm, ), {'base_fields': fields})
else:
from django.utils.datastructures import SortedDict
fields = SortedDict()
psm = ParameterSetManager(parameterset=parameterset)
for dfp in psm.parameters:
x = 1
form_id = dfp.name.name + "__" + str(x)
while form_id in fields:
x = x + 1
form_id = dfp.name.name + "__" + str(x)
units = ""
if dfp.name.units:
units = " (" + dfp.name.units + ")"
form_id = form_id.replace('/', '_s47_')
if dfp.name.isNumeric():
fields[form_id] = \
forms.DecimalField(label=dfp.name.full_name + units,
required=False,
initial=dfp.numerical_value)
elif dfp.name.isLongString():
fields[form_id] = \
forms.CharField(widget=forms.Textarea, label=dfp.name.full_name + units,
max_length=255,
required=False,
initial=dfp.string_value)
else:
fields[form_id] = \
forms.CharField(label=dfp.name.full_name + units,
max_length=255,
required=False,
initial=dfp.string_value)
if dfp.name.immutable or dfp.name.schema.immutable:
fields[form_id].widget.attrs['readonly'] = True
fields[form_id].label = \
fields[form_id].label + " (read only)"
return type('DynamicForm', (forms.BaseForm, ), {'base_fields': fields})
def _create_experiment_id_parameter_set(self, experiment, experimentId):
namespace = AtomImportSchemas.get_schema(Schema.EXPERIMENT).namespace
mgr = ParameterSetManager(parentObject=experiment, schema=namespace)
mgr.new_param(self.PARAM_EXPERIMENT_ID, experimentId)
def get_filename(ps):
try:
return ParameterSetManager(ps)\
.get_param('metadata-filename', True)
except DatasetParameter.DoesNotExist:
return None
def get_subject(ps, type_):
psm = ParameterSetManager(ps)
return { 'text': psm.get_param('code', True),
'type': type_ }
def _create_entry_parameter_set(self, dataset, entryId, updated):
namespace = AtomImportSchemas.get_schema(Schema.DATASET).namespace
mgr = ParameterSetManager(parentObject=dataset, schema=namespace)
mgr.new_param(self.PARAM_ENTRY_ID, entryId)
mgr.new_param(self.PARAM_UPDATED, iso8601.parse_date(updated))
