Пример #1
0
def save_metadata(instance, schema, metadata):
    parameters = get_parameters(schema, metadata)
    logger.debug("parameters=%s" % parameters)
    if not parameters:
        return None

    try:
        ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance)
        # return ps
    except DatasetParameterSet.DoesNotExist:
        ps = DatasetParameterSet(schema=schema, dataset=instance)
        ps.save()

    logger.debug("ps=%s\n" % ps)
    logger.debug("metadata2=%s\n" % metadata)

    for p in parameters:
        logger.debug("p=%s\n" % p)
        if p.name in metadata:
            logger.debug("found p =%s %s\n" % (p.name, p.units))
            try:
                dfp = DatasetParameter.objects.get(parameterset=ps, name=p)
            except DatasetParameter.DoesNotExist:
                dfp = DatasetParameter(parameterset=ps, name=p)

            # TODO: handle bad type
            if p.isNumeric():
                dfp.numerical_value = metadata[p.name][0]
                logger.debug("numeric")
            else:
                dfp.string_value = metadata[p.name][0]
            logger.debug("dfp=%(dfp)s" % locals())
            dfp.save()
Пример #2
0
 def __init__(self, dataset=None, dataset_id=None,
              description="", experiment_id=None):
     """
     instantiate new task or existing task
     :param dataset: optional parameter to instanciate task from
       metadata, will be tested for completeness and copied into
       new task if complete
     :type dataset: Dataset
     """
     if dataset:
         self.dataset = dataset
     elif dataset_id:
         self.dataset = Dataset.objects.get(pk=dataset_id)
     else:
         if description == "":
             raise TypeError("No description given")
         if not experiment_id:
             raise TypeError("No experiment id given")
         self.dataset = Dataset()
         self.dataset.experiment_id = experiment_id
         self.dataset.description = description
         self.dataset.save()
     try:
         thisparameterset = DatasetParameterSet.objects.get(
             schema=self.get_schema(),
             dataset=self.dataset)
     except ObjectDoesNotExist:
         thisparameterset = DatasetParameterSet(
             schema=self.get_schema(),
             dataset=self.dataset)
         thisparameterset.save()
     super(Task, self).__init__(parameterset=thisparameterset)
Пример #3
0
def _save_metadata(instance,schema,metadataset):
    """ Creates schema from the metadataset and associates it 
        with the instance.  If metadata value is empty, then 
        existing value is unchanged.  
    """
    parameters = _get_parameters(schema, metadataset)
    logger.debug("parameters=%s" % parameters)
    if not parameters:
        return None
    try:
        ps = DatasetParameterSet.objects.get(schema=schema,
                                                dataset=instance)
    except DatasetParameterSet.DoesNotExist:
        ps = DatasetParameterSet(schema=schema,dataset=instance)
        ps.save()
    logger.debug("ps=%s\n" % ps)
    logger.debug("metadata2=%s\n" % metadataset)
    for p in parameters:
        logger.debug("p=%s\n" % p)
        if p.name in metadataset:
            logger.debug("found p =%s %s\n" % (p.name,p.units))
            
            if p.isNumeric():
                val = metadataset[p.name][0]
                if val:
                        
                    dfp = DatasetParameter.objects.filter(parameterset=ps,
                                                           name=p)
                    if not dfp:
                        dfp = DatasetParameter(parameterset=ps,
                                               name=p)                    
                        dfp.numerical_value = val
                        logger.debug("new numeric")
                        dfp.save()    
                    else:
                        for dp in dfp:
                            dp.numerical_value = val
                            dp.save()    
                        logger.debug("numeric")
            else:
                val = metadataset[p.name][0]
                logger.debug("val=%s" % val)
                if val:
                    dfp = DatasetParameter.objects.filter(parameterset=ps,
                                                           name=p)
                    if not dfp:
                        dfp = DatasetParameter(parameterset=ps,
                                               name=p)
                        dfp.string_value = metadataset[p.name][0]
                        dfp.save()
                        logger.debug("new string")
                    else:
                        for dp in dfp:
                            dp.string_value = metadataset[p.name][0]
                            dp.save()
                            logger.debug("string")
                        logger.debug("done")
Пример #4
0
def _save_metadata(instance, schema, metadataset):
    """ Creates schema from the metadataset and associates it 
        with the instance.  If metadata value is empty, then 
        existing value is unchanged.  
    """
    parameters = _get_parameters(schema, metadataset)
    logger.debug("parameters=%s" % parameters)
    if not parameters:
        return None
    try:
        ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance)
    except DatasetParameterSet.DoesNotExist:
        ps = DatasetParameterSet(schema=schema, dataset=instance)
        ps.save()
    logger.debug("ps=%s\n" % ps)
    logger.debug("metadata2=%s\n" % metadataset)
    for p in parameters:
        logger.debug("p=%s\n" % p)
        if p.name in metadataset:
            logger.debug("found p =%s %s\n" % (p.name, p.units))

            if p.isNumeric():
                val = metadataset[p.name][0]
                if val:

                    dfp = DatasetParameter.objects.filter(parameterset=ps,
                                                          name=p)
                    if not dfp:
                        dfp = DatasetParameter(parameterset=ps, name=p)
                        dfp.numerical_value = val
                        logger.debug("new numeric")
                        dfp.save()
                    else:
                        for dp in dfp:
                            dp.numerical_value = val
                            dp.save()
                        logger.debug("numeric")
            else:
                val = metadataset[p.name][0]
                logger.debug("val=%s" % val)
                if val:
                    dfp = DatasetParameter.objects.filter(parameterset=ps,
                                                          name=p)
                    if not dfp:
                        dfp = DatasetParameter(parameterset=ps, name=p)
                        dfp.string_value = metadataset[p.name][0]
                        dfp.save()
                        logger.debug("new string")
                    else:
                        for dp in dfp:
                            dp.string_value = metadataset[p.name][0]
                            dp.save()
                            logger.debug("string")
                        logger.debug("done")
Пример #5
0
    def test_contextual_view(self):
        """
            Given schema on dataset, check that  image file created
        """
        user = _create_test_user()
        license = _create_license()
        exp = _create_test_experiment(user, license)
        ds = Dataset(description='happy snaps of plumage')
        ds.save()
        ds = _create_test_dataset(
            ds, exp.id, {
                "output.dat": 'test data\n',
                "grexp.dat": '1 2\n2 3\n3 7\n',
                "grfinal21.dat": '1 2\n 2 4\n4 9\n'
            })

        sch = Schema(namespace=self.HRMCSCHEMA,
                     name="hrmc_views",
                     type=Schema.DATASET)
        sch.save()

        param = ParameterName(schema=sch,
                              name="plot",
                              full_name="scatterplot",
                              units="image",
                              data_type=ParameterName.FILENAME)
        param.save()

        dps = DatasetParameterSet(schema=sch, dataset=ds)
        dps.save()

        ds.experiments.add(exp)
        ds.save()

        client = Client()
        response = client.get('/dataset/%s' % ds.id)
        self.assertEqual(response.status_code, 200)

        param_sets = get_param_sets(ds)
        self.assertTrue(param_sets)

        dp = DatasetParameter.objects.get(parameterset=param_sets[0],
                                          name=param)

        self.assertTrue(dp)
        self.assertNotEquals(dp.string_value, "")  # ie, it has a filename
    def test_contextual_view(self):
        """
            Given schema on dataset, check that  image file created
        """
        user = _create_test_user()
        license = _create_license()
        exp = _create_test_experiment(user, license)
        ds = Dataset(description='happy snaps of plumage')
        ds.save()
        ds = _create_test_dataset(ds, exp.id, {
            "output.dat": 'test data\n',
            "grexp.dat": '1 2\n2 3\n3 7\n',
            "grfinal21.dat": '1 2\n 2 4\n4 9\n'})

        sch = Schema(namespace=self.HRMCSCHEMA,
            name="hrmc_views", type=Schema.DATASET)
        sch.save()

        param = ParameterName(schema=sch, name="plot",
            full_name="scatterplot", units="image",
            data_type=ParameterName.FILENAME
            )
        param.save()

        dps = DatasetParameterSet(schema=sch, dataset=ds)
        dps.save()

        ds.experiments.add(exp)
        ds.save()

        client = Client()
        response = client.get('/dataset/%s' % ds.id)
        self.assertEqual(response.status_code, 200)

        param_sets = get_param_sets(ds)
        self.assertTrue(param_sets)

        dp = DatasetParameter.objects.get(parameterset=param_sets[0],
            name=param)

        self.assertTrue(dp)
        self.assertNotEquals(dp.string_value, "")  # ie, it has a filename
Пример #7
0
    def __init__(self, parameterset=None, parentObject=None,
                 schema=None):
        """
        instantiate new task or existing ParameterSet
        :param dataset: optional parameter to instanciate task from
          metadata, will be tested for completeness and copied into
          new task if complete
        :type dataset: Dataset
        :param schema: Schema namespace
        :type schema: string
        """

        if parameterset:
            self.parameterset = parameterset
            self.schema = self.parameterset.schema
            self.namespace = self.schema.namespace

            if isinstance(self.parameterset, DatafileParameterSet):
                self.parameters = DatafileParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = DatafileParameter

            elif isinstance(self.parameterset, DatasetParameterSet):
                self.parameters = DatasetParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = DatasetParameter

            elif isinstance(self.parameterset, ExperimentParameterSet):
                self.parameters = ExperimentParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = ExperimentParameter

            else:
                raise TypeError("Invalid parameterset object given.")

        elif parentObject and schema:

            self.namespace = schema

            if isinstance(parentObject, Dataset_File):
                self.parameterset = DatafileParameterSet(
                    schema=self.get_schema(), dataset_file=parentObject)

                self.parameterset.save()

                self.parameters = DatafileParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = DatafileParameter

            elif isinstance(parentObject, Dataset):
                self.parameterset = DatasetParameterSet(
                    schema=self.get_schema(), dataset=parentObject)

                self.parameterset.save()

                self.parameters = DatasetParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = DatasetParameter

            elif isinstance(parentObject, Experiment):
                self.parameterset = ExperimentParameterSet(
                    schema=self.get_schema(), experiment=parentObject)

                self.parameterset.save()

                self.parameters = ExperimentParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = ExperimentParameter

            else:
                raise TypeError("Invalid parent object." +
                    "Must be an experiment/dataset/datafile not " + str(type(parentObject)))

        else:
            raise TypeError("Missing arguments")
    def __call__(self, sender, **kwargs):
        """post save callback entry point.

        :param sender: The model class.
        :param instance: The actual instance being saved.
        :param created: A boolean; True if a new record was created.
        :type created: bool
        """
        logger.debug("hrmc __call__")

        datafile_instance = kwargs.get("instance")
        dataset_instance = datafile_instance.dataset
        try:
            sch = Schema.objects.get(namespace=self.schema)
        except Schema.DoesNotExist:
            logger.debug("no hrmc schema")
            return None
        except MultipleObjectsReturned:
            logger.error("multiple hrmc schemas returned")
            return None
        logger.debug("sch=%s" % sch)

        try:
            ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset_instance)
        except DatasetParameterSet.DoesNotExist:
            pass
        except MultipleObjectsReturned:
            logger.warn("Multiple dataset paramter sets for %s returned" % self.name)
            # NB: If multiple filters run in dataset could get race condition
            # which causes multiple DatasetParameterSets to be created.
            # However, we know that all data will be the same for this schema
            # so can safely delete any extras we find.
            pslist = [x.id for x in DatasetParameterSet.objects.filter(schema=sch, dataset=dataset_instance)]
            logger.debug("pslist=%s" % pslist)
            DatasetParameterSet.objects.filter(id__in=pslist[1:]).delete()
            return None
        else:
            logger.debug("parameterset already exists")
            return None

        logger.debug("dataset_instance=%s" % dataset_instance)
        filepath = datafile_instance.get_absolute_filepath()
        logger.debug("filepath=%s" % filepath)

        grexp_file = None
        grfinal_file = None
        for df in Dataset_File.objects.filter(dataset=dataset_instance):
            logger.debug("df=%s" % df.filename)
            if "grexp.dat" in df.filename:
                grexp_file = df
            if df.filename.startswith("grfinal"):
                grfinal_file = df

        if grexp_file and grfinal_file:
            logger.debug("found all files")
            try:
                ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset_instance)
            except DatasetParameterSet.DoesNotExist:
                ps = DatasetParameterSet(schema=sch, dataset=dataset_instance)
                ps.save()
                logger.debug("created new dataset")
                return None
            except MultipleObjectsReturned:
                logger.error("Multiple dataset paramter sets for %s returned" % self.name)
                return None
            else:
                logger.debug("parameterset already exists")
            return None
        else:
            logger.debug("one or more files missing")
Пример #9
0
    def __call__(self, sender, **kwargs):
        """post save callback entry point.

        :param sender: The model class.
        :param instance: The actual instance being saved.
        :param created: A boolean; True if a new record was created.
        :type created: bool
        """
        logger.debug("hrmc __call__")

        datafile_instance = kwargs.get('instance')
        dataset_instance = datafile_instance.dataset
        try:
            sch = Schema.objects.get(namespace=self.schema)
        except Schema.DoesNotExist:
            logger.debug("no hrmc schema")
            return None
        except MultipleObjectsReturned:
            logger.error("multiple hrmc schemas returned")
            return None
        logger.debug("sch=%s" % sch)

        try:
            ps = DatasetParameterSet.objects.get(schema=sch,
                                                 dataset=dataset_instance)
        except DatasetParameterSet.DoesNotExist:
            pass
        except MultipleObjectsReturned:
            logger.warn("Multiple dataset paramter sets for %s returned" %
                        self.name)
            # NB: If multiple filters run in dataset could get race condition
            # which causes multiple DatasetParameterSets to be created.
            # However, we know that all data will be the same for this schema
            # so can safely delete any extras we find.
            pslist = [
                x.id for x in DatasetParameterSet.objects.filter(
                    schema=sch, dataset=dataset_instance)
            ]
            logger.debug("pslist=%s" % pslist)
            DatasetParameterSet.objects.filter(id__in=pslist[1:]).delete()
            return None
        else:
            logger.debug("parameterset already exists")
            return None

        logger.debug("dataset_instance=%s" % dataset_instance)
        filepath = datafile_instance.get_absolute_filepath()
        logger.debug("filepath=%s" % filepath)

        grexp_file = None
        grfinal_file = None
        for df in Dataset_File.objects.filter(dataset=dataset_instance):
            logger.debug("df=%s" % df.filename)
            if "grexp.dat" in df.filename:
                grexp_file = df
            if df.filename.startswith("grfinal"):
                grfinal_file = df

        if grexp_file and grfinal_file:
            logger.debug("found all files")
            try:
                ps = DatasetParameterSet.objects.get(schema=sch,
                                                     dataset=dataset_instance)
            except DatasetParameterSet.DoesNotExist:
                ps = DatasetParameterSet(schema=sch, dataset=dataset_instance)
                ps.save()
                logger.debug("created new dataset")
                return None
            except MultipleObjectsReturned:
                logger.error("Multiple dataset paramter sets for %s returned" %
                             self.name)
                return None
            else:
                logger.debug("parameterset already exists")
            return None
        else:
            logger.debug("one or more files missing")
Пример #10
0
    def test_parameter(self):
        exp = Experiment(
            title='test exp1',
            institution_name='Australian Synchrotron',
            approved=True,
            created_by=self.user,
            public_access=Experiment.PUBLIC_ACCESS_NONE,
        )
        exp.save()

        dataset = Dataset(description="dataset description")
        dataset.save()
        dataset.experiments.add(exp)
        dataset.save()

        df_file = DataFile(dataset=dataset,
                           filename='file.txt',
                           size=42,
                           md5sum='bogus')
        df_file.save()

        df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml',
                           type=Schema.DATAFILE)
        df_schema.save()

        ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml',
                           type=Schema.DATASET)
        ds_schema.save()

        exp_schema = Schema(
            namespace='http://www.cern.ch/felzmann/schema3.xml',
            type=Schema.EXPERIMENT)
        exp_schema.save()

        df_parname = ParameterName(schema=df_schema,
                                   name='name',
                                   full_name='full_name',
                                   units='image/jpg',
                                   data_type=ParameterName.FILENAME)
        df_parname.save()

        ds_parname = ParameterName(schema=ds_schema,
                                   name='name',
                                   full_name='full_name',
                                   units='image/jpg',
                                   data_type=ParameterName.FILENAME)
        ds_parname.save()

        exp_parname = ParameterName(schema=exp_schema,
                                    name='name',
                                    full_name='full_name',
                                    units='image/jpg',
                                    data_type=ParameterName.FILENAME)
        exp_parname.save()

        df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file)
        df_parset.save()

        ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset)
        ds_parset.save()

        exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp)
        exp_parset.save()

        with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)):
            filename = 'test.jpg'
            df_parameter = DatafileParameter(name=df_parname,
                                             parameterset=df_parset,
                                             string_value=filename)
            df_parameter.save()

            ds_parameter = DatasetParameter(name=ds_parname,
                                            parameterset=ds_parset,
                                            string_value=filename)
            ds_parameter.save()

            exp_parameter = ExperimentParameter(name=exp_parname,
                                                parameterset=exp_parset,
                                                string_value=filename)
            exp_parameter.save()

            self.assertEqual(
                "<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>"
                %  # noqa
                (df_parameter.id, df_parameter.id),
                df_parameter.get())

            self.assertEqual(
                "<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>"
                %  # noqa
                (ds_parameter.id, ds_parameter.id),
                ds_parameter.get())

            self.assertEqual(
                "<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>"
                %  # noqa
                (exp_parameter.id, exp_parameter.id),
                exp_parameter.get())