def test_instrument(self): group = Group(name="Test Manager Group") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() self.assertEqual(str(facility), "Test Facility") instrument = Instrument(name="Test Instrument", facility=facility) instrument.save() self.assertEqual(str(instrument), "Test Instrument") self.assertEqual(len(instrument.getParameterSets()), 0) schema = Schema(namespace='test instrument schema namespace', type=Schema.INSTRUMENT) schema.save() parname = ParameterName(schema=schema, name='name', full_name='full_name') parname.save() pset = InstrumentParameterSet.objects.create(instrument=instrument, schema=schema) pset.save() self.assertEqual(len(instrument.getParameterSets()), 1)
def save_rif_cs_profile(self, experiment, profile): """ Save selected profile choice as experiment parameter """ namespace = "http://monash.edu.au/rif-cs/profile/" schema = None try: schema = Schema.objects.get(namespace__exact=namespace) except Schema.DoesNotExist: logger.debug('Schema ' + namespace + ' does not exist. Creating.') schema = Schema(namespace=namespace) schema.save() try: parametername = ParameterName.objects.get( schema__namespace__exact=schema.namespace, name="profile") except ParameterName.DoesNotExist: logger.debug("profile does not exist. Creating.") parametername = ParameterName(name="profile", schema=schema) parametername.save() parameterset = None try: parameterset = \ ExperimentParameterSet.objects.get(\ schema=schema, experiment=experiment) except ExperimentParameterSet.DoesNotExist, e: parameterset = ExperimentParameterSet(\ schema=schema, experiment=experiment) parameterset.save()
def _make_param(self,schema,name,paramtype): try: param = ParameterName.objects.get(schema=schema, name=name,data_type=paramtype ) except ParameterName.DoesNotExist: logger.debug("%s does not exist. Creating." % name) param = ParameterName(name=name,schema=schema,data_type=paramtype) param.save() return param
def getParamaters(self, schema, metadata): """Return a list of the paramaters that will be saved. """ param_objects = ParameterName.objects.filter(schema=schema) parameters = [] for p in metadata: if self.tagsToFind and not p in self.tagsToFind: continue if p in self.tagsToExclude: continue parameter = filter(lambda x: x.name == p, param_objects) if parameter: parameters.append(parameter[0]) continue # detect type of parameter datatype = ParameterName.STRING # integer data type test try: int(metadata[p][0]) except (ValueError, TypeError): pass else: datatype = ParameterName.NUMERIC # float data type test try: float(metadata[p][0]) except (ValueError, TypeError): pass else: datatype = ParameterName.NUMERIC # fraction data type test if isinstance(metadata[p][0], Fraction): datatype = ParameterName.NUMERIC unit = "" if metadata[p][1]: unit = metadata[p][1] new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype, units=unit) new_param.save() parameters.append(new_param) return parameters
def _make_param(self, schema, name, paramtype): try: param = ParameterName.objects.get(schema=schema, name=name, data_type=paramtype) except ParameterName.DoesNotExist: logger.debug("%s does not exist. Creating." % name) param = ParameterName(name=name, schema=schema, data_type=paramtype) param.save() return param
def getParamaters(self, schema, metadata): """Return a list of the paramaters that will be saved. """ param_objects = ParameterName.objects.filter(schema=schema) parameters = [] for p in metadata: if self.tagsToFind and not p in self.tagsToFind: continue if p in self.tagsToExclude: continue parameter = filter(lambda x: x.name == p, param_objects) if parameter: parameters.append(parameter[0]) continue # detect type of parameter datatype = ParameterName.STRING # Int test try: int(metadata[p]) except ValueError: pass except TypeError: pass else: datatype = ParameterName.NUMERIC # Fraction test if isinstance(metadata[p], Fraction): datatype = ParameterName.NUMERIC # Float test try: float(metadata[p]) except ValueError: pass except TypeError: pass else: datatype = ParameterName.NUMERIC new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype) new_param.save() parameters.append(new_param) return parameters
def setUp(self): # Load schemas for test from django.core.management import call_command call_command('loaddata', 'as_schemas') self.client = Client() self.experiments = [] try: user = User.objects.get(username='******') except User.DoesNotExist: user = User.objects.create_user('test', '', 'test') user.save() self.userprofile = user.userprofile # base_path = path.abspath(path.dirname(__file__)) experiment = Experiment(title='SAXS Test', created_by=user) experiment.save() acl = ObjectACL(pluginId=django_user, entityId=str(user.id), content_object=experiment, canRead=True, canWrite=True, canDelete=True, isOwner=True) acl.save() self.experiments += [experiment] schema = Schema.objects.get(type=Schema.DATAFILE, subtype='saxs') parameter = ParameterName.objects.get(schema=schema, name='io') parameter.is_searchable = True parameter.save() self.io_param_name = parameter.getUniqueShortName() schema = Schema.objects.get(type=Schema.DATASET, subtype='saxs') parameter = ParameterName.objects.get(schema=schema, name='frqimn') parameter.is_searchable = True parameter.save() self.frqimn_param_name = parameter.getUniqueShortName() new_schema = Schema() new_schema.namespace = 'testschemawithduplicatename' new_schema.save() new_param = ParameterName( schema=new_schema, name='title', full_name='Duplicate title parametername', is_searchable=True) new_param.save()
def _get_parameters(schema,metadata): """ Returns set of parameters from schema matched to elements in metadata, or creates them based on the metadata values. Types are based on seen values, favouring numerics over strings. """ param_objects = ParameterName.objects.filter(schema=schema) logger.debug("param_objects=%s\n" % param_objects) parameters = [] for p in metadata.keys(): logger.debug("pp=%s\n" % p) logger.debug("metadata[pp][0] = %s" % metadata[p][0]) logger.debug("metadata[pp][1] = %s" % metadata[p][1]) parameter = filter(lambda x: x.name == p,param_objects) logger.debug("pparameter=%s\n" % parameter) if parameter: parameters.append(parameter[0]) continue # work out the best type from the value datatype = ParameterName.LONGSTRING units = None try: int(metadata[p][0]) except (ValueError,TypeError): pass else: datatype = ParameterName.NUMERIC units = str(metadata[p][1]) try: float(metadata[p][0]) except (ValueError,TypeError): pass else: datatype = ParameterName.NUMERIC units = str(metadata[p][1]) logger.debug("units=%s" % str(metadata[p][1])) if units: new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype, units=units) else: new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype) new_param.save() logger.debug("new_param=%s" % new_param) logger.debug("datatype=%s" % datatype) parameters.append(new_param) return parameters
def getOrCreateParameterNames(self, schema, paramnames): """ Takes a list of paramnames (defined in the __init__ method) to get or create new parameter names objects """ pn_objects = [] for paramname in paramnames: param_objects = ParameterName.objects.filter(schema=schema, name=paramname['name']) if len(param_objects) == 0: pn = ParameterName(schema=schema, name=paramname['name'], full_name=paramname['full_name'], data_type=paramname['data_type']) pn.save() else: pn = param_objects[0] pn_objects.append(pn) return pn_objects
def get_parameters(schema, metadata): param_objects = ParameterName.objects.filter(schema=schema) logger.debug("param_objects=%s\n" % param_objects) parameters = [] for p in metadata.keys(): logger.debug("pp=%s\n" % p) logger.debug("metadata[pp][0] = %s" % metadata[p][0]) logger.debug("metadata[pp][1] = %s" % metadata[p][1]) parameter = filter(lambda x: x.name == p, param_objects) logger.debug("pparameter=%s\n" % parameter) if parameter: parameters.append(parameter[0]) continue datatype = ParameterName.STRING units = None try: int(metadata[p][0]) except (ValueError, TypeError): pass else: datatype = ParameterName.NUMERIC units = str(metadata[p][1]) try: float(metadata[p][0]) except (ValueError, TypeError): pass else: datatype = ParameterName.NUMERIC units = str(metadata[p][1]) logger.debug("units=%s" % str(metadata[p][1])) if units: new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype, units=units) else: new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype) new_param.save() logger.debug("new_param=%s" % new_param) logger.debug("datatype=%s" % datatype) parameters.append(new_param) return parameters
def test_contextual_view(self): """ Given schema on dataset, check that image file created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() ds = _create_test_dataset( ds, exp.id, { "output.dat": 'test data\n', "grexp.dat": '1 2\n2 3\n3 7\n', "grfinal21.dat": '1 2\n 2 4\n4 9\n' }) sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME) param.save() dps = DatasetParameterSet(schema=sch, dataset=ds) dps.save() ds.experiments.add(exp) ds.save() client = Client() response = client.get('/dataset/%s' % ds.id) self.assertEqual(response.status_code, 200) param_sets = get_param_sets(ds) self.assertTrue(param_sets) dp = DatasetParameter.objects.get(parameterset=param_sets[0], name=param) self.assertTrue(dp) self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def test_hrmc_filter(self): """ Make an experiment, lood up grexp file and check dataset schema missing, then loadup grfinal and check dataset schema created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() _create_test_dataset(ds, exp.id, { "output.dat": 'hello', "grexp.dat": '2 5\n6 15\n' }) ds.experiments.add(exp) ds.save() sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME) param.save() param_sets = get_param_sets(ds) self.assertEquals(list(param_sets), []) _create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"}) df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat') df2.save() h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA) h(sender=Dataset_File, instance=df2) param_sets = get_param_sets(ds) self.assertEquals([x.schema.namespace for x in param_sets], [self.HRMCSCHEMA])
def test_contextual_view(self): """ Given schema on dataset, check that image file created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() ds = _create_test_dataset(ds, exp.id, { "output.dat": 'test data\n', "grexp.dat": '1 2\n2 3\n3 7\n', "grfinal21.dat": '1 2\n 2 4\n4 9\n'}) sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME ) param.save() dps = DatasetParameterSet(schema=sch, dataset=ds) dps.save() ds.experiments.add(exp) ds.save() client = Client() response = client.get('/dataset/%s' % ds.id) self.assertEqual(response.status_code, 200) param_sets = get_param_sets(ds) self.assertTrue(param_sets) dp = DatasetParameter.objects.get(parameterset=param_sets[0], name=param) self.assertTrue(dp) self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def getOrCreateParameterNames(self, schema, paramnames): """ Takes a list of paramnames (defined in the __init__ method) to get or create new parameter names objects """ pn_objects = [] for paramname in paramnames: param_objects = ParameterName.objects.filter( schema=schema, name=paramname['name']) if len(param_objects) == 0: pn = ParameterName(schema=schema, name=paramname['name'], full_name=paramname['full_name'], data_type=paramname['data_type']) pn.save() else: pn = param_objects[0] pn_objects.append(pn) return pn_objects
def test_hrmc_filter(self): """ Make an experiment, lood up grexp file and check dataset schema missing, then loadup grfinal and check dataset schema created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() _create_test_dataset(ds, exp.id, {"output.dat": 'hello', "grexp.dat": '2 5\n6 15\n'}) ds.experiments.add(exp) ds.save() sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME ) param.save() param_sets = get_param_sets(ds) self.assertEquals(list(param_sets), []) _create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"}) df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat') df2.save() h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA) h(sender=Dataset_File, instance=df2) param_sets = get_param_sets(ds) self.assertEquals([x.schema.namespace for x in param_sets], [self.HRMCSCHEMA])
def save_rif_cs_profile(self, experiment, profile): """ Save selected profile choice as experiment parameter """ namespace = "http://monash.edu.au/rif-cs/profile/" schema = None try: schema = Schema.objects.get( namespace__exact=namespace) except Schema.DoesNotExist: logger.debug('Schema ' + namespace + ' does not exist. Creating.') schema = Schema(namespace=namespace) schema.save() try: parametername = ParameterName.objects.get( schema__namespace__exact=schema.namespace, name="profile") except ParameterName.DoesNotExist: logger.debug("profile does not exist. Creating.") parametername = ParameterName(name="profile",schema=schema) parametername.save() parameterset = None try: parameterset = \ ExperimentParameterSet.objects.get(\ schema=schema, experiment=experiment) except ExperimentParameterSet.DoesNotExist, e: parameterset = ExperimentParameterSet(\ schema=schema, experiment=experiment) parameterset.save()
def _get_parameters(schema, metadata): """ Returns set of parameters from schema matched to elements in metadata, or creates them based on the metadata values. Types are based on seen values, favouring numerics over strings. """ param_objects = ParameterName.objects.filter(schema=schema) logger.debug("param_objects=%s\n" % param_objects) parameters = [] for p in metadata.keys(): logger.debug("pp=%s\n" % p) logger.debug("metadata[pp][0] = %s" % metadata[p][0]) logger.debug("metadata[pp][1] = %s" % metadata[p][1]) parameter = filter(lambda x: x.name == p, param_objects) logger.debug("pparameter=%s\n" % parameter) if parameter: parameters.append(parameter[0]) continue # work out the best type from the value datatype = ParameterName.LONGSTRING units = None try: int(metadata[p][0]) except (ValueError, TypeError): pass else: datatype = ParameterName.NUMERIC units = str(metadata[p][1]) try: float(metadata[p][0]) except (ValueError, TypeError): pass else: datatype = ParameterName.NUMERIC units = str(metadata[p][1]) logger.debug("units=%s" % str(metadata[p][1])) if units: new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype, units=units) else: new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype) new_param.save() logger.debug("new_param=%s" % new_param) logger.debug("datatype=%s" % datatype) parameters.append(new_param) return parameters
class ParameterSetManagerTestCase(TestCase): def setUp(self): from django.contrib.auth.models import User from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.dataset = Dataset(description="dataset description...") self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, filename="testfile.txt", size="42", md5sum='bogus') self.datafile.save() self.dfo = DataFileObject( datafile=self.datafile, storage_box=self.datafile.get_default_storage_box(), uri="1/testfile.txt") self.dfo.save() self.schema = Schema(namespace="http://localhost/psmtest/df/", name="Parameter Set Manager", type=3) self.schema.save() self.parametername1 = ParameterName(schema=self.schema, name="parameter1", full_name="Parameter 1") self.parametername1.save() self.parametername2 = ParameterName(schema=self.schema, name="parameter2", full_name="Parameter 2", data_type=ParameterName.NUMERIC) self.parametername2.save() self.parametername3 = ParameterName(schema=self.schema, name="parameter3", full_name="Parameter 3", data_type=ParameterName.DATETIME) self.parametername3.save() self.datafileparameterset = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset.save() self.datafileparameter1 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername1, string_value="test1") self.datafileparameter1.save() self.datafileparameter2 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername2, numerical_value=2) self.datafileparameter2.save() # Create a ParameterName and Parameter of type LINK to an experiment self.parametername_exp_link = ParameterName( schema=self.schema, name="exp_link", full_name="This parameter is a experiment LINK", data_type=ParameterName.LINK) self.parametername_exp_link.save() self.exp_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_exp_link) exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.exp_link_param.set_value(exp_url) self.exp_link_param.save() # Create a ParameterName and Parameter of type LINK to a dataset self.parametername_dataset_link = ParameterName( schema=self.schema, name="dataset_link", full_name="This parameter is a dataset LINK", data_type=ParameterName.LINK) self.parametername_dataset_link.save() self.dataset_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_dataset_link) dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.dataset_link_param.set_value(dataset_url) self.dataset_link_param.save() # Create a ParameterName type LINK to an unresolvable (non-URL) # free-text value self.parametername_unresolvable_link = ParameterName( schema=self.schema, name="freetext_link", full_name="This parameter is a non-URL LINK", data_type=ParameterName.LINK) self.parametername_unresolvable_link.save() def tearDown(self): self.exp.delete() self.user.delete() self.parametername1.delete() self.parametername2.delete() self.parametername3.delete() self.parametername_exp_link.delete() self.parametername_dataset_link.delete() self.parametername_unresolvable_link.delete() self.schema.delete() def test_existing_parameterset(self): psm = ParameterSetManager(parameterset=self.datafileparameterset) self.assertTrue( psm.get_schema().namespace == "http://localhost/psmtest/df/") self.assertTrue(psm.get_param("parameter1").string_value == "test1") self.assertTrue(psm.get_param("parameter2", True) == 2) def test_new_parameterset(self): psm = ParameterSetManager(parentObject=self.datafile, schema="http://localhost/psmtest/df2/") self.assertTrue( psm.get_schema().namespace == "http://localhost/psmtest/df2/") psm.set_param("newparam1", "test3", "New Parameter 1") self.assertTrue(psm.get_param("newparam1").string_value == "test3") self.assertTrue( psm.get_param("newparam1").name.full_name == "New Parameter 1") psm.new_param("newparam1", "test4") self.assertTrue(len(psm.get_params("newparam1", True)) == 2) psm.set_param_list("newparam2", ("a", "b", "c", "d")) self.assertTrue(len(psm.get_params("newparam2")) == 4) psm.set_params_from_dict({"newparam2": "test5", "newparam3": 3}) self.assertTrue(psm.get_param("newparam2", True) == "test5") # the newparam3 gets created and '3' is set to a string_value # since once cannot assume that an initial numeric value # will imply continuing numeric type for this new param self.assertTrue(psm.get_param("newparam3").string_value == '3') psm.delete_params("newparam1") self.assertTrue(len(psm.get_params("newparam1", True)) == 0) def test_link_parameter_type(self): """ Test that Parameter.link_gfk (GenericForeignKey) is correctly assigned after using Parameter.set_value(some_url) for a LINK Parameter. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) # Check link to experiment exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.assertTrue(psm.get_param("exp_link").string_value == exp_url) self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id) exp_ct = ContentType.objects.get(model__iexact="experiment") self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct) self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp) # Check link to dataset dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.assertTrue( psm.get_param("dataset_link").string_value == dataset_url) self.assertTrue( psm.get_param("dataset_link").link_id == self.dataset.id) dataset_ct = ContentType.objects.get(model__iexact="dataset") self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct) self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset) def test_link_parameter_type_extra(self): # make a second ParameterSet for testing some variations # in URL values self.datafileparameterset2 = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset2.save() psm = ParameterSetManager(parameterset=self.datafileparameterset2) self.dataset_link_param2 = DatafileParameter( parameterset=self.datafileparameterset2, name=self.parametername_dataset_link) # /dataset/1 - no trailing slash dataset_url = self.dataset.get_absolute_url() self.dataset_link_param2.set_value(dataset_url) self.dataset_link_param2.save() # Check link_id/link_ct/link_gfk to dataset self.assertTrue( psm.get_param("dataset_link").link_id == self.dataset.id) dataset_ct = ContentType.objects.get(model__iexact="dataset") self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct) self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset) # Test links of the form /api/v1/experiment/<experiment_id>/ self.exp_link_param2 = DatafileParameter( parameterset=self.datafileparameterset2, name=self.parametername_exp_link) exp_url = '/api/v1/experiment/%s/' % self.exp.id self.exp_link_param2.set_value(exp_url) self.exp_link_param2.save() # Check link_id/link_ct/link_gfk to experiment self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id) exp_ct = ContentType.objects.get(model__iexact="experiment") self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct) self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp) def test_unresolvable_link_parameter(self): """ Test that LINK Parameters that can't be resolved to a model (including non-URL values) still work. """ self.datafileparameterset3 = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset3.save() psm = ParameterSetManager(parameterset=self.datafileparameterset3) # Create a Parameter of type LINK to an unresolvable (non-URL) # free-text value self.freetext_link_param = DatafileParameter( parameterset=self.datafileparameterset3, name=self.parametername_unresolvable_link) self.assertRaises( SuspiciousOperation, lambda: self.freetext_link_param.set_value("FREETEXT_ID_123")) def test_tz_naive_date_handling(self): """ Ensure that dates are handling in a timezone-aware way. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) psm.new_param("parameter3", str(datetime(1970, 01, 01, 10, 0, 0))) expect(psm.get_param("parameter3", True))\ .to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc)) def test_tz_aware_date_handling(self): """ Ensure that dates are handling in a timezone-aware way. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) psm.new_param("parameter3", '1970-01-01T08:00:00+08:00') expect(psm.get_param("parameter3", True))\ .to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def setUp(self): from django.contrib.auth.models import User from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.dataset = Dataset(description="dataset description...") self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, filename="testfile.txt", size="42", md5sum='bogus') self.datafile.save() self.dfo = DataFileObject( datafile=self.datafile, storage_box=self.datafile.get_default_storage_box(), uri="1/testfile.txt") self.dfo.save() self.schema = Schema( namespace="http://localhost/psmtest/df/", name="Parameter Set Manager", type=3) self.schema.save() self.parametername1 = ParameterName( schema=self.schema, name="parameter1", full_name="Parameter 1") self.parametername1.save() self.parametername2 = ParameterName( schema=self.schema, name="parameter2", full_name="Parameter 2", data_type=ParameterName.NUMERIC) self.parametername2.save() self.parametername3 = ParameterName( schema=self.schema, name="parameter3", full_name="Parameter 3", data_type=ParameterName.DATETIME) self.parametername3.save() self.datafileparameterset = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset.save() self.datafileparameter1 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername1, string_value="test1") self.datafileparameter1.save() self.datafileparameter2 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername2, numerical_value=2) self.datafileparameter2.save() # Create a ParameterName and Parameter of type LINK to an experiment self.parametername_exp_link = ParameterName( schema=self.schema, name="exp_link", full_name="This parameter is a experiment LINK", data_type=ParameterName.LINK) self.parametername_exp_link.save() self.exp_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_exp_link) exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.exp_link_param.set_value(exp_url) self.exp_link_param.save() # Create a ParameterName and Parameter of type LINK to a dataset self.parametername_dataset_link = ParameterName( schema=self.schema, name="dataset_link", full_name="This parameter is a dataset LINK", data_type=ParameterName.LINK) self.parametername_dataset_link.save() self.dataset_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_dataset_link) dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.dataset_link_param.set_value(dataset_url) self.dataset_link_param.save() # Create a ParameterName type LINK to an unresolvable (non-URL) # free-text value self.parametername_unresolvable_link = ParameterName( schema=self.schema, name="freetext_link", full_name="This parameter is a non-URL LINK", data_type=ParameterName.LINK) self.parametername_unresolvable_link.save()
class ParameterSetManagerTestCase(TestCase): def setUp(self): from django.contrib.auth.models import User from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.dataset = Dataset(description="dataset description...") self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, filename="testfile.txt", size="42", md5sum='bogus') self.datafile.save() self.dfo = DataFileObject( datafile=self.datafile, storage_box=self.datafile.get_default_storage_box(), uri="1/testfile.txt") self.dfo.save() self.schema = Schema( namespace="http://localhost/psmtest/df/", name="Parameter Set Manager", type=3) self.schema.save() self.parametername1 = ParameterName( schema=self.schema, name="parameter1", full_name="Parameter 1") self.parametername1.save() self.parametername2 = ParameterName( schema=self.schema, name="parameter2", full_name="Parameter 2", data_type=ParameterName.NUMERIC) self.parametername2.save() self.parametername3 = ParameterName( schema=self.schema, name="parameter3", full_name="Parameter 3", data_type=ParameterName.DATETIME) self.parametername3.save() self.datafileparameterset = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset.save() self.datafileparameter1 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername1, string_value="test1") self.datafileparameter1.save() self.datafileparameter2 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername2, numerical_value=2) self.datafileparameter2.save() # Create a ParameterName and Parameter of type LINK to an experiment self.parametername_exp_link = ParameterName( schema=self.schema, name="exp_link", full_name="This parameter is a experiment LINK", data_type=ParameterName.LINK) self.parametername_exp_link.save() self.exp_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_exp_link) exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.exp_link_param.set_value(exp_url) self.exp_link_param.save() # Create a ParameterName and Parameter of type LINK to a dataset self.parametername_dataset_link = ParameterName( schema=self.schema, name="dataset_link", full_name="This parameter is a dataset LINK", data_type=ParameterName.LINK) self.parametername_dataset_link.save() self.dataset_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_dataset_link) dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.dataset_link_param.set_value(dataset_url) self.dataset_link_param.save() # Create a ParameterName type LINK to an unresolvable (non-URL) # free-text value self.parametername_unresolvable_link = ParameterName( schema=self.schema, name="freetext_link", full_name="This parameter is a non-URL LINK", data_type=ParameterName.LINK) self.parametername_unresolvable_link.save() def tearDown(self): self.exp.delete() self.user.delete() self.parametername1.delete() self.parametername2.delete() self.parametername3.delete() self.parametername_exp_link.delete() self.parametername_dataset_link.delete() self.parametername_unresolvable_link.delete() self.schema.delete() def test_existing_parameterset(self): psm = ParameterSetManager(parameterset=self.datafileparameterset) self.assertTrue(psm.get_schema().namespace == "http://localhost/psmtest/df/") self.assertTrue(psm.get_param("parameter1").string_value == "test1") self.assertTrue(psm.get_param("parameter2", True) == 2) def test_new_parameterset(self): psm = ParameterSetManager(parentObject=self.datafile, schema="http://localhost/psmtest/df2/") self.assertTrue(psm.get_schema().namespace == "http://localhost/psmtest/df2/") psm.set_param("newparam1", "test3", "New Parameter 1") self.assertTrue(psm.get_param("newparam1").string_value == "test3") self.assertTrue(psm.get_param("newparam1").name.full_name == "New Parameter 1") psm.new_param("newparam1", "test4") self.assertTrue(len(psm.get_params("newparam1", True)) == 2) psm.set_param_list("newparam2", ("a", "b", "c", "d")) self.assertTrue(len(psm.get_params("newparam2")) == 4) psm.set_params_from_dict( {"newparam2": "test5", "newparam3": 3}) self.assertTrue(psm.get_param("newparam2", True) == "test5") # the newparam3 gets created and '3' is set to a string_value # since once cannot assume that an initial numeric value # will imply continuing numeric type for this new param self.assertTrue(psm.get_param("newparam3").string_value == '3') psm.delete_params("newparam1") self.assertTrue(len(psm.get_params("newparam1", True)) == 0) def test_link_parameter_type(self): """ Test that Parameter.link_gfk (GenericForeignKey) is correctly assigned after using Parameter.set_value(some_url) for a LINK Parameter. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) # Check link to experiment exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.assertTrue(psm.get_param("exp_link").string_value == exp_url) self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id) exp_ct = ContentType.objects.get(model__iexact="experiment") self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct) self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp) # Check link to dataset dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.assertTrue(psm.get_param("dataset_link").string_value == dataset_url) self.assertTrue(psm.get_param("dataset_link").link_id == self.dataset.id) dataset_ct = ContentType.objects.get(model__iexact="dataset") self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct) self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset) def test_link_parameter_type_extra(self): # make a second ParameterSet for testing some variations # in URL values self.datafileparameterset2 = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset2.save() psm = ParameterSetManager(parameterset=self.datafileparameterset2) self.dataset_link_param2 = DatafileParameter( parameterset=self.datafileparameterset2, name=self.parametername_dataset_link) # /dataset/1 - no trailing slash dataset_url = self.dataset.get_absolute_url() self.dataset_link_param2.set_value(dataset_url) self.dataset_link_param2.save() # Check link_id/link_ct/link_gfk to dataset self.assertTrue(psm.get_param("dataset_link").link_id == self.dataset.id) dataset_ct = ContentType.objects.get(model__iexact="dataset") self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct) self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset) # Test links of the form /api/v1/experiment/<experiment_id>/ self.exp_link_param2 = DatafileParameter( parameterset=self.datafileparameterset2, name=self.parametername_exp_link) exp_url = '/api/v1/experiment/%s/' % self.exp.id self.exp_link_param2.set_value(exp_url) self.exp_link_param2.save() # Check link_id/link_ct/link_gfk to experiment self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id) exp_ct = ContentType.objects.get(model__iexact="experiment") self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct) self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp) def test_unresolvable_link_parameter(self): """ Test that LINK Parameters that can't be resolved to a model (including non-URL values) still work. """ self.datafileparameterset3 = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset3.save() psm = ParameterSetManager(parameterset=self.datafileparameterset3) # Create a Parameter of type LINK to an unresolvable (non-URL) # free-text value self.freetext_link_param = DatafileParameter( parameterset=self.datafileparameterset3, name=self.parametername_unresolvable_link) self.assertRaises(SuspiciousOperation, lambda: self.freetext_link_param.set_value( "FREETEXT_ID_123")) def test_tz_naive_date_handling(self): """ Ensure that dates are handling in a timezone-aware way. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) psm.new_param("parameter3", str(datetime(1970, 01, 01, 10, 0, 0))) expect(psm.get_param("parameter3", True))\ .to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc)) def test_tz_aware_date_handling(self): """ Ensure that dates are handling in a timezone-aware way. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) psm.new_param("parameter3", '1970-01-01T08:00:00+08:00') expect(psm.get_param("parameter3", True))\ .to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def _get_create_parname(self, parname, fullparname=None, example_value=None): try: paramName = ParameterName.objects.get(name=parname, schema__id=self.get_schema().id) except ObjectDoesNotExist: paramName = ParameterName() paramName.schema = self.get_schema() paramName.name = parname if fullparname: paramName.full_name = fullparname else: paramName.full_name = parname if example_value: try: float(example_value) paramName.data_type = ParameterName.NUMERIC except (TypeError, ValueError): paramName.data_type = ParameterName.STRING else: paramName.data_type = ParameterName.STRING paramName.is_searchable = True paramName.save() return paramName
def test_parameter(self): exp = Experiment( title='test exp1', institution_name='Australian Synchrotron', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE, ) exp.save() dataset = Dataset(description="dataset description") dataset.save() dataset.experiments.add(exp) dataset.save() df_file = DataFile(dataset=dataset, filename='file.txt', size=42, md5sum='bogus') df_file.save() df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml', type=Schema.DATAFILE) df_schema.save() ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml', type=Schema.DATASET) ds_schema.save() exp_schema = Schema( namespace='http://www.cern.ch/felzmann/schema3.xml', type=Schema.EXPERIMENT) exp_schema.save() df_parname = ParameterName(schema=df_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) df_parname.save() ds_parname = ParameterName(schema=ds_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) ds_parname.save() exp_parname = ParameterName(schema=exp_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) exp_parname.save() df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file) df_parset.save() ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset) ds_parset.save() exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp) exp_parset.save() with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)): filename = 'test.jpg' df_parameter = DatafileParameter(name=df_parname, parameterset=df_parset, string_value=filename) df_parameter.save() ds_parameter = DatasetParameter(name=ds_parname, parameterset=ds_parset, string_value=filename) ds_parameter.save() exp_parameter = ExperimentParameter(name=exp_parname, parameterset=exp_parset, string_value=filename) exp_parameter.save() self.assertEqual( "<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>" % # noqa (df_parameter.id, df_parameter.id), df_parameter.get()) self.assertEqual( "<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>" % # noqa (ds_parameter.id, ds_parameter.id), ds_parameter.get()) self.assertEqual( "<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>" % # noqa (exp_parameter.id, exp_parameter.id), exp_parameter.get())
def setUp(self): from django.contrib.auth.models import User from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.dataset = Dataset(description="dataset description...") self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, filename="testfile.txt", size="42", md5sum='bogus') self.datafile.save() self.dfo = DataFileObject( datafile=self.datafile, storage_box=self.datafile.get_default_storage_box(), uri="1/testfile.txt") self.dfo.save() self.schema = Schema(namespace="http://localhost/psmtest/df/", name="Parameter Set Manager", type=3) self.schema.save() self.parametername1 = ParameterName(schema=self.schema, name="parameter1", full_name="Parameter 1") self.parametername1.save() self.parametername2 = ParameterName(schema=self.schema, name="parameter2", full_name="Parameter 2", data_type=ParameterName.NUMERIC) self.parametername2.save() self.parametername3 = ParameterName(schema=self.schema, name="parameter3", full_name="Parameter 3", data_type=ParameterName.DATETIME) self.parametername3.save() self.datafileparameterset = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset.save() self.datafileparameter1 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername1, string_value="test1") self.datafileparameter1.save() self.datafileparameter2 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername2, numerical_value=2) self.datafileparameter2.save() # Create a ParameterName and Parameter of type LINK to an experiment self.parametername_exp_link = ParameterName( schema=self.schema, name="exp_link", full_name="This parameter is a experiment LINK", data_type=ParameterName.LINK) self.parametername_exp_link.save() self.exp_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_exp_link) exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.exp_link_param.set_value(exp_url) self.exp_link_param.save() # Create a ParameterName and Parameter of type LINK to a dataset self.parametername_dataset_link = ParameterName( schema=self.schema, name="dataset_link", full_name="This parameter is a dataset LINK", data_type=ParameterName.LINK) self.parametername_dataset_link.save() self.dataset_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_dataset_link) dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.dataset_link_param.set_value(dataset_url) self.dataset_link_param.save() # Create a ParameterName type LINK to an unresolvable (non-URL) # free-text value self.parametername_unresolvable_link = ParameterName( schema=self.schema, name="freetext_link", full_name="This parameter is a non-URL LINK", data_type=ParameterName.LINK) self.parametername_unresolvable_link.save()