def save_metadata(instance, schema, metadata): parameters = get_parameters(schema, metadata) logger.debug("parameters=%s" % parameters) if not parameters: return None try: ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance) # return ps except DatasetParameterSet.DoesNotExist: ps = DatasetParameterSet(schema=schema, dataset=instance) ps.save() logger.debug("ps=%s\n" % ps) logger.debug("metadata2=%s\n" % metadata) for p in parameters: logger.debug("p=%s\n" % p) if p.name in metadata: logger.debug("found p =%s %s\n" % (p.name, p.units)) try: dfp = DatasetParameter.objects.get(parameterset=ps, name=p) except DatasetParameter.DoesNotExist: dfp = DatasetParameter(parameterset=ps, name=p) # TODO: handle bad type if p.isNumeric(): dfp.numerical_value = metadata[p.name][0] logger.debug("numeric") else: dfp.string_value = metadata[p.name][0] logger.debug("dfp=%(dfp)s" % locals()) dfp.save()
def __init__(self, dataset=None, dataset_id=None, description="", experiment_id=None): """ instantiate new task or existing task :param dataset: optional parameter to instanciate task from metadata, will be tested for completeness and copied into new task if complete :type dataset: Dataset """ if dataset: self.dataset = dataset elif dataset_id: self.dataset = Dataset.objects.get(pk=dataset_id) else: if description == "": raise TypeError("No description given") if not experiment_id: raise TypeError("No experiment id given") self.dataset = Dataset() self.dataset.experiment_id = experiment_id self.dataset.description = description self.dataset.save() try: thisparameterset = DatasetParameterSet.objects.get( schema=self.get_schema(), dataset=self.dataset) except ObjectDoesNotExist: thisparameterset = DatasetParameterSet( schema=self.get_schema(), dataset=self.dataset) thisparameterset.save() super(Task, self).__init__(parameterset=thisparameterset)
def _save_metadata(instance,schema,metadataset): """ Creates schema from the metadataset and associates it with the instance. If metadata value is empty, then existing value is unchanged. """ parameters = _get_parameters(schema, metadataset) logger.debug("parameters=%s" % parameters) if not parameters: return None try: ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance) except DatasetParameterSet.DoesNotExist: ps = DatasetParameterSet(schema=schema,dataset=instance) ps.save() logger.debug("ps=%s\n" % ps) logger.debug("metadata2=%s\n" % metadataset) for p in parameters: logger.debug("p=%s\n" % p) if p.name in metadataset: logger.debug("found p =%s %s\n" % (p.name,p.units)) if p.isNumeric(): val = metadataset[p.name][0] if val: dfp = DatasetParameter.objects.filter(parameterset=ps, name=p) if not dfp: dfp = DatasetParameter(parameterset=ps, name=p) dfp.numerical_value = val logger.debug("new numeric") dfp.save() else: for dp in dfp: dp.numerical_value = val dp.save() logger.debug("numeric") else: val = metadataset[p.name][0] logger.debug("val=%s" % val) if val: dfp = DatasetParameter.objects.filter(parameterset=ps, name=p) if not dfp: dfp = DatasetParameter(parameterset=ps, name=p) dfp.string_value = metadataset[p.name][0] dfp.save() logger.debug("new string") else: for dp in dfp: dp.string_value = metadataset[p.name][0] dp.save() logger.debug("string") logger.debug("done")
def _save_metadata(instance, schema, metadataset): """ Creates schema from the metadataset and associates it with the instance. If metadata value is empty, then existing value is unchanged. """ parameters = _get_parameters(schema, metadataset) logger.debug("parameters=%s" % parameters) if not parameters: return None try: ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance) except DatasetParameterSet.DoesNotExist: ps = DatasetParameterSet(schema=schema, dataset=instance) ps.save() logger.debug("ps=%s\n" % ps) logger.debug("metadata2=%s\n" % metadataset) for p in parameters: logger.debug("p=%s\n" % p) if p.name in metadataset: logger.debug("found p =%s %s\n" % (p.name, p.units)) if p.isNumeric(): val = metadataset[p.name][0] if val: dfp = DatasetParameter.objects.filter(parameterset=ps, name=p) if not dfp: dfp = DatasetParameter(parameterset=ps, name=p) dfp.numerical_value = val logger.debug("new numeric") dfp.save() else: for dp in dfp: dp.numerical_value = val dp.save() logger.debug("numeric") else: val = metadataset[p.name][0] logger.debug("val=%s" % val) if val: dfp = DatasetParameter.objects.filter(parameterset=ps, name=p) if not dfp: dfp = DatasetParameter(parameterset=ps, name=p) dfp.string_value = metadataset[p.name][0] dfp.save() logger.debug("new string") else: for dp in dfp: dp.string_value = metadataset[p.name][0] dp.save() logger.debug("string") logger.debug("done")
def test_contextual_view(self): """ Given schema on dataset, check that image file created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() ds = _create_test_dataset( ds, exp.id, { "output.dat": 'test data\n', "grexp.dat": '1 2\n2 3\n3 7\n', "grfinal21.dat": '1 2\n 2 4\n4 9\n' }) sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME) param.save() dps = DatasetParameterSet(schema=sch, dataset=ds) dps.save() ds.experiments.add(exp) ds.save() client = Client() response = client.get('/dataset/%s' % ds.id) self.assertEqual(response.status_code, 200) param_sets = get_param_sets(ds) self.assertTrue(param_sets) dp = DatasetParameter.objects.get(parameterset=param_sets[0], name=param) self.assertTrue(dp) self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def test_contextual_view(self): """ Given schema on dataset, check that image file created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() ds = _create_test_dataset(ds, exp.id, { "output.dat": 'test data\n', "grexp.dat": '1 2\n2 3\n3 7\n', "grfinal21.dat": '1 2\n 2 4\n4 9\n'}) sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME ) param.save() dps = DatasetParameterSet(schema=sch, dataset=ds) dps.save() ds.experiments.add(exp) ds.save() client = Client() response = client.get('/dataset/%s' % ds.id) self.assertEqual(response.status_code, 200) param_sets = get_param_sets(ds) self.assertTrue(param_sets) dp = DatasetParameter.objects.get(parameterset=param_sets[0], name=param) self.assertTrue(dp) self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def __init__(self, parameterset=None, parentObject=None, schema=None): """ instantiate new task or existing ParameterSet :param dataset: optional parameter to instanciate task from metadata, will be tested for completeness and copied into new task if complete :type dataset: Dataset :param schema: Schema namespace :type schema: string """ if parameterset: self.parameterset = parameterset self.schema = self.parameterset.schema self.namespace = self.schema.namespace if isinstance(self.parameterset, DatafileParameterSet): self.parameters = DatafileParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatafileParameter elif isinstance(self.parameterset, DatasetParameterSet): self.parameters = DatasetParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatasetParameter elif isinstance(self.parameterset, ExperimentParameterSet): self.parameters = ExperimentParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = ExperimentParameter else: raise TypeError("Invalid parameterset object given.") elif parentObject and schema: self.namespace = schema if isinstance(parentObject, Dataset_File): self.parameterset = DatafileParameterSet( schema=self.get_schema(), dataset_file=parentObject) self.parameterset.save() self.parameters = DatafileParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatafileParameter elif isinstance(parentObject, Dataset): self.parameterset = DatasetParameterSet( schema=self.get_schema(), dataset=parentObject) self.parameterset.save() self.parameters = DatasetParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatasetParameter elif isinstance(parentObject, Experiment): self.parameterset = ExperimentParameterSet( schema=self.get_schema(), experiment=parentObject) self.parameterset.save() self.parameters = ExperimentParameter.objects.filter( parameterset=self.parameterset) self.blank_param = ExperimentParameter else: raise TypeError("Invalid parent object." + "Must be an experiment/dataset/datafile not " + str(type(parentObject))) else: raise TypeError("Missing arguments")
def __call__(self, sender, **kwargs): """post save callback entry point. :param sender: The model class. :param instance: The actual instance being saved. :param created: A boolean; True if a new record was created. :type created: bool """ logger.debug("hrmc __call__") datafile_instance = kwargs.get("instance") dataset_instance = datafile_instance.dataset try: sch = Schema.objects.get(namespace=self.schema) except Schema.DoesNotExist: logger.debug("no hrmc schema") return None except MultipleObjectsReturned: logger.error("multiple hrmc schemas returned") return None logger.debug("sch=%s" % sch) try: ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset_instance) except DatasetParameterSet.DoesNotExist: pass except MultipleObjectsReturned: logger.warn("Multiple dataset paramter sets for %s returned" % self.name) # NB: If multiple filters run in dataset could get race condition # which causes multiple DatasetParameterSets to be created. # However, we know that all data will be the same for this schema # so can safely delete any extras we find. pslist = [x.id for x in DatasetParameterSet.objects.filter(schema=sch, dataset=dataset_instance)] logger.debug("pslist=%s" % pslist) DatasetParameterSet.objects.filter(id__in=pslist[1:]).delete() return None else: logger.debug("parameterset already exists") return None logger.debug("dataset_instance=%s" % dataset_instance) filepath = datafile_instance.get_absolute_filepath() logger.debug("filepath=%s" % filepath) grexp_file = None grfinal_file = None for df in Dataset_File.objects.filter(dataset=dataset_instance): logger.debug("df=%s" % df.filename) if "grexp.dat" in df.filename: grexp_file = df if df.filename.startswith("grfinal"): grfinal_file = df if grexp_file and grfinal_file: logger.debug("found all files") try: ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset_instance) except DatasetParameterSet.DoesNotExist: ps = DatasetParameterSet(schema=sch, dataset=dataset_instance) ps.save() logger.debug("created new dataset") return None except MultipleObjectsReturned: logger.error("Multiple dataset paramter sets for %s returned" % self.name) return None else: logger.debug("parameterset already exists") return None else: logger.debug("one or more files missing")
def __call__(self, sender, **kwargs): """post save callback entry point. :param sender: The model class. :param instance: The actual instance being saved. :param created: A boolean; True if a new record was created. :type created: bool """ logger.debug("hrmc __call__") datafile_instance = kwargs.get('instance') dataset_instance = datafile_instance.dataset try: sch = Schema.objects.get(namespace=self.schema) except Schema.DoesNotExist: logger.debug("no hrmc schema") return None except MultipleObjectsReturned: logger.error("multiple hrmc schemas returned") return None logger.debug("sch=%s" % sch) try: ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset_instance) except DatasetParameterSet.DoesNotExist: pass except MultipleObjectsReturned: logger.warn("Multiple dataset paramter sets for %s returned" % self.name) # NB: If multiple filters run in dataset could get race condition # which causes multiple DatasetParameterSets to be created. # However, we know that all data will be the same for this schema # so can safely delete any extras we find. pslist = [ x.id for x in DatasetParameterSet.objects.filter( schema=sch, dataset=dataset_instance) ] logger.debug("pslist=%s" % pslist) DatasetParameterSet.objects.filter(id__in=pslist[1:]).delete() return None else: logger.debug("parameterset already exists") return None logger.debug("dataset_instance=%s" % dataset_instance) filepath = datafile_instance.get_absolute_filepath() logger.debug("filepath=%s" % filepath) grexp_file = None grfinal_file = None for df in Dataset_File.objects.filter(dataset=dataset_instance): logger.debug("df=%s" % df.filename) if "grexp.dat" in df.filename: grexp_file = df if df.filename.startswith("grfinal"): grfinal_file = df if grexp_file and grfinal_file: logger.debug("found all files") try: ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset_instance) except DatasetParameterSet.DoesNotExist: ps = DatasetParameterSet(schema=sch, dataset=dataset_instance) ps.save() logger.debug("created new dataset") return None except MultipleObjectsReturned: logger.error("Multiple dataset paramter sets for %s returned" % self.name) return None else: logger.debug("parameterset already exists") return None else: logger.debug("one or more files missing")
def test_parameter(self): exp = Experiment( title='test exp1', institution_name='Australian Synchrotron', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE, ) exp.save() dataset = Dataset(description="dataset description") dataset.save() dataset.experiments.add(exp) dataset.save() df_file = DataFile(dataset=dataset, filename='file.txt', size=42, md5sum='bogus') df_file.save() df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml', type=Schema.DATAFILE) df_schema.save() ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml', type=Schema.DATASET) ds_schema.save() exp_schema = Schema( namespace='http://www.cern.ch/felzmann/schema3.xml', type=Schema.EXPERIMENT) exp_schema.save() df_parname = ParameterName(schema=df_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) df_parname.save() ds_parname = ParameterName(schema=ds_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) ds_parname.save() exp_parname = ParameterName(schema=exp_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) exp_parname.save() df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file) df_parset.save() ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset) ds_parset.save() exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp) exp_parset.save() with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)): filename = 'test.jpg' df_parameter = DatafileParameter(name=df_parname, parameterset=df_parset, string_value=filename) df_parameter.save() ds_parameter = DatasetParameter(name=ds_parname, parameterset=ds_parset, string_value=filename) ds_parameter.save() exp_parameter = ExperimentParameter(name=exp_parname, parameterset=exp_parset, string_value=filename) exp_parameter.save() self.assertEqual( "<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>" % # noqa (df_parameter.id, df_parameter.id), df_parameter.get()) self.assertEqual( "<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>" % # noqa (ds_parameter.id, ds_parameter.id), ds_parameter.get()) self.assertEqual( "<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>" % # noqa (exp_parameter.id, exp_parameter.id), exp_parameter.get())