def _get_parameters(schema, metadata): """ Returns set of parameters from schema matched to elements in metadata, or creates them based on the metadata values. Types are based on seen values, favouring numerics over strings. """ param_objects = ParameterName.objects.filter(schema=schema) logger.debug("param_objects=%s\n" % param_objects) parameters = [] for p in metadata.keys(): logger.debug("pp=%s\n" % p) logger.debug("metadata[pp][0] = %s" % metadata[p][0]) logger.debug("metadata[pp][1] = %s" % metadata[p][1]) parameter = filter(lambda x: x.name == p, param_objects) logger.debug("pparameter=%s\n" % parameter) if parameter: parameters.append(parameter[0]) continue # work out the best type from the value datatype = ParameterName.LONGSTRING units = None try: int(metadata[p][0]) except (ValueError, TypeError): pass else: datatype = ParameterName.NUMERIC units = str(metadata[p][1]) try: float(metadata[p][0]) except (ValueError, TypeError): pass else: datatype = ParameterName.NUMERIC units = str(metadata[p][1]) logger.debug("units=%s" % str(metadata[p][1])) if units: new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype, units=units) else: new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype) new_param.save() logger.debug("new_param=%s" % new_param) logger.debug("datatype=%s" % datatype) parameters.append(new_param) return parameters
def test_instrument(self): group = Group(name="Test Manager Group") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() self.assertEqual(str(facility), "Test Facility") instrument = Instrument(name="Test Instrument", facility=facility) instrument.save() self.assertEqual(str(instrument), "Test Instrument") self.assertEqual(len(instrument.getParameterSets()), 0) schema = Schema(namespace='test instrument schema namespace', type=Schema.INSTRUMENT) schema.save() parname = ParameterName(schema=schema, name='name', full_name='full_name') parname.save() pset = InstrumentParameterSet.objects.create(instrument=instrument, schema=schema) pset.save() self.assertEqual(len(instrument.getParameterSets()), 1)
def _get_create_parname(self, parname, fullparname=None, example_value=None): try: paramName = ParameterName.objects.get( name=parname, schema__id=self.get_schema().id) except ObjectDoesNotExist: paramName = ParameterName() paramName.schema = self.get_schema() paramName.name = parname if fullparname: paramName.full_name = fullparname else: paramName.full_name = parname if example_value: try: float(example_value) paramName.data_type = ParameterName.NUMERIC except (TypeError, ValueError): paramName.data_type = ParameterName.STRING else: paramName.data_type = ParameterName.STRING paramName.is_searchable = True paramName.save() return paramName
def save_rif_cs_profile(self, experiment, profile): """ Save selected profile choice as experiment parameter """ namespace = "http://monash.edu.au/rif-cs/profile/" schema = None try: schema = Schema.objects.get(namespace__exact=namespace) except Schema.DoesNotExist: logger.debug('Schema ' + namespace + ' does not exist. Creating.') schema = Schema(namespace=namespace) schema.save() try: parametername = ParameterName.objects.get( schema__namespace__exact=schema.namespace, name="profile") except ParameterName.DoesNotExist: logger.debug("profile does not exist. Creating.") parametername = ParameterName(name="profile", schema=schema) parametername.save() parameterset = None try: parameterset = \ ExperimentParameterSet.objects.get(\ schema=schema, experiment=experiment) except ExperimentParameterSet.DoesNotExist, e: parameterset = ExperimentParameterSet(\ schema=schema, experiment=experiment) parameterset.save()
def getParamaters(self, schema, metadata): """Return a list of the paramaters that will be saved. """ param_objects = ParameterName.objects.filter(schema=schema) parameters = [] for p in metadata: if self.tagsToFind and not p in self.tagsToFind: continue if p in self.tagsToExclude: continue parameter = filter(lambda x: x.name == p, param_objects) if parameter: parameters.append(parameter[0]) continue # detect type of parameter datatype = ParameterName.STRING # Int test try: int(metadata[p]) except ValueError: pass except TypeError: pass else: datatype = ParameterName.NUMERIC # Fraction test if isinstance(metadata[p], Fraction): datatype = ParameterName.NUMERIC # Float test try: float(metadata[p]) except ValueError: pass except TypeError: pass else: datatype = ParameterName.NUMERIC new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype) new_param.save() parameters.append(new_param) return parameters
def _make_param(self, schema, name, paramtype): try: param = ParameterName.objects.get(schema=schema, name=name, data_type=paramtype) except ParameterName.DoesNotExist: logger.debug("%s does not exist. Creating." % name) param = ParameterName(name=name, schema=schema, data_type=paramtype) param.save() return param
def test_contextual_view(self): """ Given schema on dataset, check that image file created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() ds = _create_test_dataset( ds, exp.id, { "output.dat": 'test data\n', "grexp.dat": '1 2\n2 3\n3 7\n', "grfinal21.dat": '1 2\n 2 4\n4 9\n' }) sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME) param.save() dps = DatasetParameterSet(schema=sch, dataset=ds) dps.save() ds.experiments.add(exp) ds.save() client = Client() response = client.get('/dataset/%s' % ds.id) self.assertEqual(response.status_code, 200) param_sets = get_param_sets(ds) self.assertTrue(param_sets) dp = DatasetParameter.objects.get(parameterset=param_sets[0], name=param) self.assertTrue(dp) self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def test_hrmc_filter(self): """ Make an experiment, lood up grexp file and check dataset schema missing, then loadup grfinal and check dataset schema created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() _create_test_dataset(ds, exp.id, { "output.dat": 'hello', "grexp.dat": '2 5\n6 15\n' }) ds.experiments.add(exp) ds.save() sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME) param.save() param_sets = get_param_sets(ds) self.assertEquals(list(param_sets), []) _create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"}) df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat') df2.save() h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA) h(sender=Dataset_File, instance=df2) param_sets = get_param_sets(ds) self.assertEquals([x.schema.namespace for x in param_sets], [self.HRMCSCHEMA])
def getOrCreateParameterNames(self, schema, paramnames): """ Takes a list of paramnames (defined in the __init__ method) to get or create new parameter names objects """ pn_objects = [] for paramname in paramnames: param_objects = ParameterName.objects.filter( schema=schema, name=paramname['name']) if len(param_objects) == 0: pn = ParameterName(schema=schema, name=paramname['name'], full_name=paramname['full_name'], data_type=paramname['data_type']) pn.save() else: pn = param_objects[0] pn_objects.append(pn) return pn_objects
def test_parameter(self): exp = Experiment( title='test exp1', institution_name='Australian Synchrotron', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE, ) exp.save() dataset = Dataset(description="dataset description") dataset.save() dataset.experiments.add(exp) dataset.save() df_file = DataFile(dataset=dataset, filename='file.txt', size=42, md5sum='bogus') df_file.save() df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml', type=Schema.DATAFILE) df_schema.save() ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml', type=Schema.DATASET) ds_schema.save() exp_schema = Schema( namespace='http://www.cern.ch/felzmann/schema3.xml', type=Schema.EXPERIMENT) exp_schema.save() df_parname = ParameterName(schema=df_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) df_parname.save() ds_parname = ParameterName(schema=ds_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) ds_parname.save() exp_parname = ParameterName(schema=exp_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) exp_parname.save() df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file) df_parset.save() ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset) ds_parset.save() exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp) exp_parset.save() with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)): filename = 'test.jpg' df_parameter = DatafileParameter(name=df_parname, parameterset=df_parset, string_value=filename) df_parameter.save() ds_parameter = DatasetParameter(name=ds_parname, parameterset=ds_parset, string_value=filename) ds_parameter.save() exp_parameter = ExperimentParameter(name=exp_parname, parameterset=exp_parset, string_value=filename) exp_parameter.save() self.assertEqual( "<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>" % # noqa (df_parameter.id, df_parameter.id), df_parameter.get()) self.assertEqual( "<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>" % # noqa (ds_parameter.id, ds_parameter.id), ds_parameter.get()) self.assertEqual( "<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>" % # noqa (exp_parameter.id, exp_parameter.id), exp_parameter.get())
def setUp(self): from django.contrib.auth.models import User from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.dataset = Dataset(description="dataset description...") self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, filename="testfile.txt", size="42", md5sum='bogus') self.datafile.save() self.dfo = DataFileObject( datafile=self.datafile, storage_box=self.datafile.get_default_storage_box(), uri="1/testfile.txt") self.dfo.save() self.schema = Schema(namespace="http://localhost/psmtest/df/", name="Parameter Set Manager", type=3) self.schema.save() self.parametername1 = ParameterName(schema=self.schema, name="parameter1", full_name="Parameter 1") self.parametername1.save() self.parametername2 = ParameterName(schema=self.schema, name="parameter2", full_name="Parameter 2", data_type=ParameterName.NUMERIC) self.parametername2.save() self.parametername3 = ParameterName(schema=self.schema, name="parameter3", full_name="Parameter 3", data_type=ParameterName.DATETIME) self.parametername3.save() self.datafileparameterset = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset.save() self.datafileparameter1 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername1, string_value="test1") self.datafileparameter1.save() self.datafileparameter2 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername2, numerical_value=2) self.datafileparameter2.save() # Create a ParameterName and Parameter of type LINK to an experiment self.parametername_exp_link = ParameterName( schema=self.schema, name="exp_link", full_name="This parameter is a experiment LINK", data_type=ParameterName.LINK) self.parametername_exp_link.save() self.exp_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_exp_link) exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.exp_link_param.set_value(exp_url) self.exp_link_param.save() # Create a ParameterName and Parameter of type LINK to a dataset self.parametername_dataset_link = ParameterName( schema=self.schema, name="dataset_link", full_name="This parameter is a dataset LINK", data_type=ParameterName.LINK) self.parametername_dataset_link.save() self.dataset_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_dataset_link) dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.dataset_link_param.set_value(dataset_url) self.dataset_link_param.save() # Create a ParameterName type LINK to an unresolvable (non-URL) # free-text value self.parametername_unresolvable_link = ParameterName( schema=self.schema, name="freetext_link", full_name="This parameter is a non-URL LINK", data_type=ParameterName.LINK) self.parametername_unresolvable_link.save()