Ejemplo n.º 1
0
def map_form_to_schemas(extraInfo, publication):
    for form_id, form in extraInfo.iteritems():
        try:  # Ignore form if no schema exists with this name
            schema = Schema.objects.get(namespace=form['schema'])
        except Schema.DoesNotExist:
            continue
        parameter_set = ExperimentParameterSet(
            schema=schema, experiment=publication)
        parameter_set.save()
        for key, value in form.iteritems():
            if key != 'schema':
                try:  # Ignore field if parameter name (key) doesn't match
                    parameter_name = ParameterName.objects.get(
                        schema=schema, name=key)
                    if parameter_name.isNumeric():
                        parameter = ExperimentParameter(
                            name=parameter_name,
                            parameterset=parameter_set,
                            numerical_value=float(value))
                    elif parameter_name.isLongString() or \
                            parameter_name.isString() or \
                            parameter_name.isURL() or \
                            parameter_name.isLink() or \
                            parameter_name.isFilename():
                        parameter = ExperimentParameter(
                            name=parameter_name,
                            parameterset=parameter_set,
                            string_value=str(value))
                    else:
                        # Shouldn't happen, but here in case the parameter type
                        # is non-standard
                        continue
                    parameter.save()
                except ParameterName.DoesNotExist:
                    pass
Ejemplo n.º 2
0
    def save_rif_cs_profile(self, experiment, profile):
        """
        Save selected profile choice as experiment parameter
        """
        namespace = "http://monash.edu.au/rif-cs/profile/"
        schema = None
        try:
            schema = Schema.objects.get(
                namespace__exact=namespace)
        except Schema.DoesNotExist:
            logger.debug('Schema ' + namespace +
            ' does not exist. Creating.')
            schema = Schema(namespace=namespace)
            schema.save()

        parametername = ParameterName.objects.get(
            schema__namespace__exact=schema.namespace,
            name="profile")

        parameterset = None
        try:
            parameterset = \
                         ExperimentParameterSet.objects.get(\
                                schema=schema,
                                experiment=experiment)

        except ExperimentParameterSet.DoesNotExist, e:
            parameterset = ExperimentParameterSet(\
                                schema=schema,
                                experiment=experiment)

            parameterset.save()
Ejemplo n.º 3
0
def create_draft_publication(user, publication_title, publication_description):
    # Note: Maybe this logic can be taken from the tardis_portal/views.py?

    experiment = Experiment(created_by=user,
                            title=publication_title,
                            description=publication_description)
    experiment.save()

    ObjectACL(content_object=experiment,
              pluginId=django_user,
              entityId=str(user.id),
              canRead=True,
              canWrite=False,
              canDelete=False,
              isOwner=True,
              aclOwnershipType=ObjectACL.OWNER_OWNED).save()

    ObjectACL(content_object=experiment,
              pluginId=django_group,
              entityId=str(
                  Group.objects.get_or_create(
                      name=getattr(
                          settings, 'PUBLICATION_OWNER_GROUP',
                          default_settings.PUBLICATION_OWNER_GROUP))[0].id),
              canRead=True,
              canWrite=True,
              canDelete=True,
              isOwner=True,
              aclOwnershipType=ObjectACL.OWNER_OWNED).save()

    publication_schema = Schema.objects.get(
        namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
                          default_settings.PUBLICATION_SCHEMA_ROOT))

    # Attach draft schema
    draft_publication_schema = Schema.objects.get(
        namespace=getattr(settings, 'PUBLICATION_DRAFT_SCHEMA',
                          default_settings.PUBLICATION_DRAFT_SCHEMA))
    ExperimentParameterSet(schema=draft_publication_schema,
                           experiment=experiment).save()

    # Attach root schema and blank form_state parameter
    publication_root_schema = Schema.objects.get(
        namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
                          default_settings.PUBLICATION_SCHEMA_ROOT))
    publication_root_parameter_set = ExperimentParameterSet(
        schema=publication_schema,
        experiment=experiment)
    publication_root_parameter_set.save()
    form_state_param_name = ParameterName.objects.get(
        schema=publication_root_schema, name='form_state')
    ExperimentParameter(name=form_state_param_name,
                        parameterset=publication_root_parameter_set).save()

    return experiment
Ejemplo n.º 4
0
def _create_related_info(request, experiment_id):
    if not authz.has_write_permissions(request, experiment_id):
        return return_response_error(request)
    form = RelatedInfoForm(json.loads(request.body))
    if not form.is_valid():
        return HttpResponse('', status=400)
    ps = ExperimentParameterSet(experiment_id=experiment_id,
                                schema=_get_schema())
    ps.save()
    ParameterSetManager(ps).set_params_from_dict(form.cleaned_data)
    return HttpResponse(json.dumps(_get_dict_from_ps(ps)),
                        content_type='application/json; charset=utf-8',
                        status=201)
Ejemplo n.º 5
0
 def _create(self, request, experiment_id):
     from tardis.tardis_portal.auth.decorators import has_experiment_write
     if not has_experiment_write(request, experiment_id):
         return return_response_error(request)
     form = self.form_cls(json.loads(request.body))
     if not form.is_valid():
         return HttpResponse('', status=400)
     ps = ExperimentParameterSet(experiment_id=experiment_id,
                                 schema=self.schema)
     ps.save()
     ParameterSetManager(ps).set_params_from_dict(form.cleaned_data)
     return HttpResponse(json.dumps(self._get_dict_from_ps(ps)),
                         content_type='application/json; charset=utf-8',
                         status=201)
Ejemplo n.º 6
0
def set_embargo_release_date(publication, release_date):
    pub_schema_root = Schema.objects.get(
        namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
                          default_settings.PUBLICATION_SCHEMA_ROOT))
    pub_schema_root_parameter_set = ExperimentParameterSet(
        schema=pub_schema_root,
        experiment=publication)
    pub_schema_root_parameter_set.save()
    embargo_parameter_name = ParameterName.objects.get(
        schema=pub_schema_root,
        name='embargo')
    ExperimentParameter(name=embargo_parameter_name,
                        parameterset=pub_schema_root_parameter_set,
                        datetime_value=release_date).save()
Ejemplo n.º 7
0
    def add_info(self, cleaned_data):
        logger.debug('adding info')
        logger.debug(cleaned_data)

        type = cleaned_data['type']
        identifier_type = cleaned_data['identifier_type']
        identifier = cleaned_data['identifier']
        title = cleaned_data['title']
        notes = cleaned_data['notes']

        eps = ExperimentParameterSet(experiment_id=self.experiment_id, schema=self.schema)
        eps.save()

        _maybe_add(eps, self.type_name, type)
        _maybe_add(eps, self.identifier_type_name, identifier_type, force=True)
        _maybe_add(eps, self.identifier_name, identifier, force=True)
        _maybe_add(eps, self.title_name, title)
        _maybe_add(eps, self.notes_name, notes)
Ejemplo n.º 8
0
 def _save_party_refs(self,  party, party_relation):
     """ Save party and party relation information as parameters on the 
         experiment
     """
     namespace = "http://rmit.edu.au/rif-cs/party/1.0/"
     logger.debug("saving party")
     schema = None
     try:
         schema = Schema.objects.get(
             namespace__exact=namespace)
     except Schema.DoesNotExist:
         logger.debug('Schema ' + namespace +
         ' does not exist. Creating.')
         schema = Schema(namespace=namespace)
         schema.save()
     exp = Experiment.objects.get(pk=self.experiment_id)    
     party_id_param = self._make_param(schema=schema, 
                                       name="party_id",
                                       paramtype=ParameterName.NUMERIC)
     relation_param = self._make_param(schema=schema, 
                                       name="relationtocollection_id",
                                       paramtype=ParameterName.STRING)                    
     parameterset = ExperimentParameterSet(schema=schema, experiment=exp)
     parameterset.save()    
     ep = ExperimentParameter.objects.filter(name=party_id_param,
         parameterset=parameterset,
         parameterset__experiment=exp)
     for p in ep:
         p.delete()
     ep = ExperimentParameter(
         parameterset=parameterset,
         name=party_id_param,
         numerical_value=party.pk)
     ep.save()                        
     ep = ExperimentParameter.objects.filter(name=relation_param,
         parameterset=parameterset,
         parameterset__experiment=exp)
     for p in ep:
         p.delete()
     ep = ExperimentParameter(
         parameterset=parameterset,
         name=relation_param,
         string_value=party_relation)
     ep.save()
Ejemplo n.º 9
0
def synchrotron_search_epn(publication):
    # *** Synchrotron specific ***
    # Search for beamline/EPN information associated with each dataset
    # and add to the publication.
    try:
        synch_epn_schema = Schema.objects.get(
            namespace='http://www.tardis.edu.au/schemas/as/'
                      'experiment/2010/09/21')
        datasets = Dataset.objects.filter(experiments=publication)
        synch_experiments = Experiment.objects.filter(
            datasets__in=datasets,
            experimentparameterset__schema=synch_epn_schema).exclude(
            pk=publication.pk).distinct()
        for exp in [s for s in
                    synch_experiments if not s.is_publication()]:
            epn = ExperimentParameter.objects.get(
                name__name='EPN',
                name__schema=synch_epn_schema,
                parameterset__experiment=exp).string_value
            beamline = ExperimentParameter.objects.get(
                name__name='beamline',
                name__schema=synch_epn_schema,
                parameterset__experiment=exp).string_value

            epn_parameter_set = ExperimentParameterSet(
                schema=synch_epn_schema,
                experiment=publication)
            epn_parameter_set.save()
            epn_copy = ExperimentParameter(
                name=ParameterName.objects.get(
                    name='EPN', schema=synch_epn_schema),
                parameterset=epn_parameter_set)
            epn_copy.string_value = epn
            epn_copy.save()
            beamline_copy = ExperimentParameter(
                name=ParameterName.objects.get(
                    name='beamline', schema=synch_epn_schema),
                parameterset=epn_parameter_set)
            beamline_copy.string_value = beamline
            beamline_copy.save()
    except Schema.DoesNotExist:
        pass
class ParameterSetManager(object):

    parameterset = None
    parameters = None       # queryset of parameters
    blank_param = None

    # parameterset OR schema / datafile / dataset / experiment
    # delete dataset creation code
    # make parameterset / object arguments generic and test type
    # create function to return generic parameter type for setting/getting
    #
    # 2011/07/20 (Ryan Braganza)
    # changed type checking from:
    #     type(self.parameterset).__name__ == 'ClassName'
    # to:
    #     isinstance(self.parameterset, Class)
    # This is to solve an issue with models using deferred fields.
    # When using deferred fields, a subclass is used, so the class name
    # will not match when using type(self.parameterset).__name__
    # isinstance deals with inheritance appropriately here
    def __init__(self, parameterset=None, parentObject=None,
                 schema=None):
        """
        instantiate new task or existing ParameterSet
        :param dataset: optional parameter to instanciate task from
          metadata, will be tested for completeness and copied into
          new task if complete
        :type dataset: Dataset
        :param schema: Schema namespace
        :type schema: string
        """

        if parameterset:
            self.parameterset = parameterset
            self.schema = self.parameterset.schema
            self.namespace = self.schema.namespace

            if isinstance(self.parameterset, DatafileParameterSet):
                self.parameters = DatafileParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = DatafileParameter

            elif isinstance(self.parameterset, DatasetParameterSet):
                self.parameters = DatasetParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = DatasetParameter

            elif isinstance(self.parameterset, ExperimentParameterSet):
                self.parameters = ExperimentParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = ExperimentParameter

            else:
                raise TypeError("Invalid parameterset object given.")

        elif parentObject and schema:

            self.namespace = schema

            if isinstance(parentObject, Dataset_File):
                self.parameterset = DatafileParameterSet(
                    schema=self.get_schema(), dataset_file=parentObject)

                self.parameterset.save()

                self.parameters = DatafileParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = DatafileParameter

            elif isinstance(parentObject, Dataset):
                self.parameterset = DatasetParameterSet(
                    schema=self.get_schema(), dataset=parentObject)

                self.parameterset.save()

                self.parameters = DatasetParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = DatasetParameter

            elif isinstance(parentObject, Experiment):
                self.parameterset = ExperimentParameterSet(
                    schema=self.get_schema(), experiment=parentObject)

                self.parameterset.save()

                self.parameters = ExperimentParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = ExperimentParameter

            else:
                raise TypeError("Invalid parent object." +
                    "Must be an experiment/dataset/datafile not " + str(type(parentObject)))

        else:
            raise TypeError("Missing arguments")

    def get_schema(self):
        try:
            schema = Schema.objects.get(
                namespace=self.namespace)
        except ObjectDoesNotExist:
            schema = Schema()
            schema.namespace = self.namespace
            schema.save()
            self.schema = schema
        return schema

    def get_param(self, parname, value=False):
        par = self.parameters.get(name__name=parname)
        if value:
            if par.name.isNumeric():
                return par.numerical_value
            elif par.name.isDateTime():
                if is_aware(par.datetime_value):
                    return par.datetime_value
                return make_aware(par.datetime_value, LOCAL_TZ)
            else:
                return par.string_value
        return par

    def get_params(self, parname, value=False):
        pars = self.parameters.filter(name__name=parname)
        if value:
            if len(pars) > 0 and pars[0].name.isNumeric():
                return [par.numerical_value
                        for par in pars]
            else:
                return [par.string_value
                        for par in pars]
        return pars

    def set_param(self, parname, value, fullparname=None,
                  example_value=None):
        try:
            param = self.get_param(parname)
        except ObjectDoesNotExist:
            param = self.blank_param()
            param.parameterset = self.parameterset
            param.name = self._get_create_parname(parname, fullparname,
                example_value=example_value)
            #param.string_value = value
            #param.save()
        if param.name.isNumeric():
            param.numerical_value = float(value)
        elif param.name.isDateTime():
            if settings.USE_TZ:
                if (is_naive(value)):
                    value = make_aware(value, LOCAL_TZ)
                param.datetime_value = value
            else:
                if (is_aware(value)):
                    value = make_naive(value, LOCAL_TZ)
                param.datetime_value = value
        else:
            param.string_value = unicode(value)
        param.save()
        return param.id

    def new_param(self, parname, value, fullparname=None):
        param = self.blank_param()
        param.parameterset = self.parameterset
        param.name = self._get_create_parname(parname, fullparname)
        param.string_value = value
        param.save()
        if param.name.isNumeric():
            param.numerical_value = float(value)
        elif param.name.isDateTime():
            if settings.USE_TZ:
                if (is_naive(value)):
                    value = make_aware(value, LOCAL_TZ)
                param.datetime_value = value
            else:
                if (is_aware(value)):
                    value = make_naive(value, LOCAL_TZ)
                param.datetime_value = value
        else:
            param.string_value = unicode(value)
        param.save()
        return param.id

        # use this one from post data
    def set_param_list(self, parname, value_list, fullparname=None):
        self.delete_params(parname)
        for value in value_list:
            if value != None:
                self.new_param(parname, value, fullparname)

    def set_params_from_dict(self, dict):
        for (key, value) in dict.iteritems():
            if type(value) is list:
                self.set_param_list(key, value)
            else:
                if value != None:
                    self.delete_params(key)
                    self.set_param(key, value)

    def delete_params(self, parname):
        params = self.get_params(parname)
        for param in params:
            param.delete()

    def delete_all_params(self):
        for param in self.parameters:
            param.delete()

    def _get_create_parname(self, parname,
                            fullparname=None, example_value=None):
        try:
            paramName = ParameterName.objects.get(name=parname,
                               schema__id=self.get_schema().id)
        except ObjectDoesNotExist:
            paramName = ParameterName()
            paramName.schema = self.get_schema()
            paramName.name = parname
            if fullparname:
                paramName.full_name = fullparname
            else:
                paramName.full_name = parname
            if example_value:
                try:
                    float(example_value)
                    paramName.data_type = ParameterName.NUMERIC
                except (TypeError, ValueError):
                    paramName.data_type = ParameterName.STRING
            else:
                paramName.data_type = ParameterName.STRING
            paramName.is_searchable = True
            paramName.save()
        return paramName
Ejemplo n.º 11
0
class EmbargoHandler(object):

    def __init__(self, experiment_id, create=False):
        self.experiment = Experiment.objects.get(pk=experiment_id)

        parametersets = ExperimentParameterSet.objects.filter(
            schema__namespace=NAMESPACE, experiment__id=experiment_id)

        self.schema, _ = Schema.objects.get_or_create(namespace=NAMESPACE, name='Embargo Details')
        self.expiry_date, _ = ParameterName.objects.get_or_create(schema=self.schema, name=EXPIRY_DATE_KEY, full_name='Expiry', immutable=True, data_type=ParameterName.DATETIME)
        self.never_expire, _ = ParameterName.objects.get_or_create(schema=self.schema, name=NEVER_EXPIRE_KEY, full_name='Never Expires', immutable=True, data_type=ParameterName.STRING)

        if len(parametersets) == 1:
            self.parameterset = parametersets[0]
        elif create:
            self.parameterset = ExperimentParameterSet(experiment=self.experiment, schema=self.schema)
            self.parameterset.save()
        else:
            self.parameterset = None

    def never_expires(self):
        never_expire = self._get_or_none(NEVER_EXPIRE_KEY)
        if never_expire:
            return True
        expiry_date = self._get_or_none(EXPIRY_DATE_KEY)
        if expiry_date:
            return False
        else:
            return self.experiment.end_time == None

    def has_any_expiry(self):
        return not self.never_expires()

    def can_be_defaulted(self):
        expiry_date = self._get_or_none(EXPIRY_DATE_KEY)
        never_expires = self._get_or_none(NEVER_EXPIRE_KEY)
        return never_expires or expiry_date

    def because_no_end_date(self):
        if self._get_or_none(NEVER_EXPIRE_KEY):
            return False
        return self.experiment.end_time == None

    def get_expiry_date(self):
        ''' returns calculated or explicit expiry or None '''
        import datetime
        if self.never_expires():
            return None

        explicit_expiry = self._get_or_none(EXPIRY_DATE_KEY)
        if explicit_expiry:
            return explicit_expiry.datetime_value
        else:
            return self.experiment.end_time + datetime.timedelta(settings.EMBARGO_DAYS)

    def _get_or_none(self, name):
        if not self.parameterset:
            return None
        params = self.parameterset.experimentparameter_set.filter(name__name=name)
        if params.count() == 0:
            return None
        else:
            return params[0]

    def prevent_expiry(self):
        # delete any current expiry dates
        # set never_expires = True
        logger.fatal('preventing expiry')
        if not self.parameterset:
            raise Exception('incorrectly initialised, call with create=True')
        params = self.parameterset.experimentparameter_set
        params.all().delete()
        param = ExperimentParameter(name=self.never_expire, string_value='True', parameterset=self.parameterset)
        param.save()

        self.experiment.public = False
        self.experiment.save()

    def reset_to_default(self):
        import datetime
        if self.parameterset:
            self.parameterset.delete()

            expiry_date = self.get_expiry_date()
            if expiry_date and expiry_date.date() < datetime.date.today():
                self.experiment.public = True
            else:
                self.experiment.public = False
            self.experiment.save()

        else:
            logger.warn('tried to delete parameterset that does not exist')

    def set_expiry(self, date_string):
        if not self.parameterset:
            raise Exception('incorrectly initialised, call with create=True')

        params = self.parameterset.experimentparameter_set
        params.all().delete()
        import datetime
        expiry_date = datetime.datetime.strptime(date_string, '%Y/%m/%d')
        param = ExperimentParameter(name=self.expiry_date, datetime_value=expiry_date, parameterset=self.parameterset)
        param.save()

        if expiry_date.date() < datetime.date.today():
            self.experiment.public = True
        else:
            self.experiment.public = False
        self.experiment.save()
Ejemplo n.º 12
0
def process_form(request):
    # Decode the form data
    form_state = json.loads(request.body)

    def validation_error(error=None):
        if error is None:
            error = 'Invalid form data was submitted ' \
                    '(server-side validation failed)'
        return HttpResponse(
            json.dumps({
                'error': error}),
            content_type="application/json")

    # Check if the form data contains a publication ID
    # If it does, then this publication needs to be updated
    # rather than created.
    if 'publicationId' not in form_state:
        if not form_state['publicationTitle'].strip():
            return validation_error()
        publication = create_draft_publication(
            request.user, form_state['publicationTitle'],
            form_state['publicationDescription'])
        form_state['publicationId'] = publication.id
    else:
        publication = get_draft_publication(
            request.user, form_state['publicationId'])
        # Check if the publication is finalised (i.e. not in draft)
        # if it is, then refuse to process the form.
        if publication is None or not publication.is_publication_draft():
            return HttpResponseForbidden()

    # Get the form state database object
    form_state_parameter = ExperimentParameter.objects.get(
        name__name='form_state',
        name__schema__namespace=getattr(
            settings, 'PUBLICATION_SCHEMA_ROOT',
            default_settings.PUBLICATION_SCHEMA_ROOT),
        parameterset__experiment=publication)

    # Check if the form state needs to be loaded (i.e. a publication draft
    # is resumed)
    # no database changes are made if the form is resumed
    if form_state['action'] == 'resume':
        form_state = json.loads(form_state_parameter.string_value)
        return HttpResponse(json.dumps(form_state),
                            content_type="application/json")

    if form_state['action'] == 'update-dataset-selection':
        # Update the publication title/description if changed.
        # Must not be blank.
        if not form_state['publicationTitle'].strip() or \
                not form_state['publicationDescription'].strip():
            return validation_error()

        if publication.title != form_state['publicationTitle']:
            publication.title = form_state['publicationTitle']
            publication.save()
        if publication.description != form_state['publicationDescription']:
            publication.description = form_state['publicationDescription']
            publication.save()

        # Update associated datasets
        # (note: might not be efficient re: db queries)
        # ... first clear all current associations
        current_datasets = Dataset.objects.filter(experiments=publication)
        for current_dataset in current_datasets:
            current_dataset.experiments.remove(publication)
        # ... now (re)add all datasets
        selected_datasets = [ds['dataset']['id']
                             for ds in form_state['addedDatasets']]
        datasets = Dataset.objects.filter(
            experiments__in=Experiment.safe.owned_and_shared(request.user),
            pk__in=selected_datasets).distinct()

        for dataset in datasets:
            dataset.experiments.add(publication)

        # --- Get data for the next page --- #
        # Construct the disclipline-specific form based on the
        # selected datasets
        selected_forms = select_forms(datasets)
        if 'disciplineSpecificFormTemplates' in form_state:
            # clear extraInfo if the selected forms differ
            # (i.e. datasets have changed)
            if json.dumps(selected_forms) != json.dumps(
                    form_state['disciplineSpecificFormTemplates']):
                form_state['extraInfo'] = {}
        form_state['disciplineSpecificFormTemplates'] = selected_forms

    elif form_state['action'] == 'update-extra-info':
        # Clear any current parameter sets except for those belonging
        # to the publication draft schema or containing the form_state
        # parameter
        clear_publication_metadata(publication)

        # Loop through form data and create associates parameter sets
        # Any unrecognised fields or schemas are ignored!
        map_form_to_schemas(form_state['extraInfo'], publication)

        # *** Synchrotron specific ***
        # Search for beamline/EPN information associated with each dataset
        # and add to the publication.
        synchrotron_search_epn(publication)

        # --- Get data for the next page --- #
        licenses_json = get_licenses()
        form_state['licenses'] = licenses_json

        # Select the first license as default
        if licenses_json:
            if 'selectedLicenseId' not in form_state:
                form_state['selectedLicenseId'] = licenses_json[0]['id']
        else:  # No licenses configured...
            form_state['selectedLicenseId'] = -1

        # Set a default author (current user) if no previously saved data
        # By default, the form sends a list of authors of one element
        # with blank fields
        if len(form_state['authors']) == 1 and \
                not form_state['authors'][0]['name']:
            form_state['authors'] = [
                {'name': ' '.join([request.user.first_name,
                                   request.user.last_name]),
                 'institution': getattr(settings, 'DEFAULT_INSTITUTION', ''),
                 'email': request.user.email}]

    elif form_state['action'] == 'submit':
        # any final form validation should occur here
        # and specific error messages can be returned
        # to the browser before the publication's draft
        # status is removed.

        if 'acknowledge' not in form_state or not form_state['acknowledge']:
            return validation_error('You must confirm that you are '
                                    'authorised to submit this publication')

        set_publication_authors(form_state['authors'], publication)

        institutions = '; '.join(
            set([author['institution'] for author in form_state['authors']]))
        publication.institution_name = institutions

        # Attach the publication details schema
        pub_details_schema = Schema.objects.get(
            namespace=getattr(settings, 'PUBLICATION_DETAILS_SCHEMA',
                              default_settings.PUBLICATION_DETAILS_SCHEMA))
        pub_details_parameter_set = ExperimentParameterSet(
            schema=pub_details_schema,
            experiment=publication)
        pub_details_parameter_set.save()

        # Add the acknowledgements
        acknowledgements_parameter_name = ParameterName.objects.get(
            schema=pub_details_schema,
            name='acknowledgements')
        ExperimentParameter(name=acknowledgements_parameter_name,
                            parameterset=pub_details_parameter_set,
                            string_value=form_state['acknowledgements']).save()

        # Set the release date
        set_embargo_release_date(
            publication,
            dateutil.parser.parse(
                form_state[
                    'embargo']))

        # Set the license
        try:
            publication.license = License.objects.get(
                pk=form_state['selectedLicenseId'],
                is_active=True,
                allows_distribution=True)
        except License.DoesNotExist:
            publication.license = License.get_none_option_license()

        publication.save()

        # Send emails about publication in draft
        subject, message_content = email_pub_requires_authorisation(
            request.user.username,
            request.build_absolute_uri(
                reverse('tardis_portal.view_experiment',
                        args=(publication.id,))),
            request.build_absolute_uri(
                '/apps/publication-forms/approvals/'))

        try:
            send_mail(subject,
                      message_content,
                      getattr(
                          settings, 'PUBLICATION_NOTIFICATION_SENDER_EMAIL',
                          default_settings.PUBLICATION_NOTIFICATION_SENDER_EMAIL),
                      get_pub_admin_email_addresses(),
                      fail_silently=False)

            subject, message_content = email_pub_awaiting_approval(
                publication.title)
            send_mail_to_authors(publication, subject, message_content,
                                 fail_silently=False)
        except Exception as e:
            logger.error(
                "failed to send publication notification email(s): %s" %
                repr(e)
            )
            return HttpResponse(
                json.dumps({
                    'error': 'Failed to send notification email - please '
                             'contact the %s administrator (%s), '
                             'or try again later. Your draft is saved.'
                             % (get_site_admin_email(),
                                getattr(settings, 'SITE_TITLE', 'MyTardis'))
                }),
                content_type="application/json")

        # Remove the draft status
        remove_draft_status(publication)

        # Automatically approve publications if approval is not required
        if not getattr(settings, 'PUBLICATIONS_REQUIRE_APPROVAL',
                       default_settings.PUBLICATIONS_REQUIRE_APPROVAL):
            approve_publication(request, publication, message=None,
                                send_email=False)
            # approve_publication will delete the form state, so don't
            # bother saving is and return.
            form_state['action'] = ''
            return HttpResponse(json.dumps(form_state),
                                content_type="appication/json")

        # Trigger publication record update
        tasks.update_publication_records.delay()

    # Clear the form action and save the state
    form_state['action'] = ''
    form_state_parameter.string_value = json.dumps(form_state)
    form_state_parameter.save()

    return HttpResponse(json.dumps(form_state), content_type="appication/json")
Ejemplo n.º 13
0
def populate_pdb_pub_records():
    PUB_SCHEMA = getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
                         default_settings.PUBLICATION_SCHEMA_ROOT)
    PUB_SCHEMA_DRAFT = getattr(settings, 'PUBLICATION_DRAFT_SCHEMA',
                               default_settings.PUBLICATION_DRAFT_SCHEMA)
    PDB_SCHEMA = getattr(settings, 'PDB_PUBLICATION_SCHEMA_ROOT',
                         default_settings.PDB_PUBLICATION_SCHEMA_ROOT)
    publications = Experiment.objects \
        .filter(experimentparameterset__schema__namespace=PDB_SCHEMA) \
        .filter(experimentparameterset__schema__namespace=PUB_SCHEMA) \
        .exclude(experimentparameterset__schema__namespace=PUB_SCHEMA_DRAFT) \
        .distinct()

    last_update_parameter_name = ParameterName.objects.get(
        name='pdb-last-sync', schema__namespace=PUB_SCHEMA)

    def add_if_missing(parameterset,
                       name,
                       string_value=None,
                       numerical_value=None,
                       datetime_value=None):
        try:
            ExperimentParameter.objects.get(name__name=name,
                                            parameterset=parameterset)
        except ExperimentParameter.DoesNotExist:
            param_name = ParameterName.objects.get(name=name,
                                                   schema=parameterset.schema)
            param = ExperimentParameter(name=param_name,
                                        parameterset=parameterset)
            param.string_value = string_value
            param.numerical_value = numerical_value
            param.datetime_value = datetime_value
            param.save()

    for pub in publications:
        try:
            # try to get the last update time for the PDB data
            pdb_last_update_parameter = ExperimentParameter.objects.get(
                parameterset__schema__namespace=PUB_SCHEMA,
                name=last_update_parameter_name,
                parameterset__experiment=pub)
            last_update = pdb_last_update_parameter.datetime_value
            needs_update = last_update + \
                getattr(settings,
                        'PDB_REFRESH_INTERVAL',
                        default_settings.PDB_REFRESH_INTERVAL) \
                < timezone.now()

        except ExperimentParameter.DoesNotExist:
            # if the PDB last update time parameter doesn't exist,
            # we definitely need to update the data and create a last
            # update entry
            needs_update = True
            pdb_last_update_parameter = None

        # If an update needs to happen...
        if needs_update:
            # 1. get the PDB info
            pdb_parameter_set = ExperimentParameterSet.objects.get(
                schema__namespace=getattr(
                    settings, 'PDB_PUBLICATION_SCHEMA_ROOT',
                    default_settings.PDB_PUBLICATION_SCHEMA_ROOT),
                experiment=pub)
            pdb = ExperimentParameter.objects.get(
                name__name='pdb-id', parameterset=pdb_parameter_set)
            pdb_id = pdb.string_value
            # 1a. cosmetic change of case for PDB ID, if entered incorrectly
            if pdb_id != pdb_id.upper():
                pdb.string_value = pdb_id.upper()
                pdb.save()

            try:
                # 2. fetch the info from pdb.org
                pdb = PDBCifHelper(pdb_id)

                # 3. insert all standard pdb parameters
                add_if_missing(pdb_parameter_set,
                               'title',
                               string_value=pdb.get_pdb_title())
                add_if_missing(pdb_parameter_set,
                               'url',
                               string_value=pdb.get_pdb_url())
                try:
                    add_if_missing(pdb_parameter_set,
                                   'resolution',
                                   numerical_value=pdb.get_resolution())
                except ValueError:
                    logger.error('PDB field "resolution" could not be set for '
                                 'publication Id %i \n %s' %
                                 (pub.id, traceback.format_exc()))

                try:
                    add_if_missing(pdb_parameter_set,
                                   'r-value',
                                   numerical_value=pdb.get_obs_r_value())
                except ValueError:
                    logger.error('PDB field "r-value" could not be set for '
                                 'publication Id %i \n %s' %
                                 (pub.id, traceback.format_exc()))

                try:
                    add_if_missing(pdb_parameter_set,
                                   'r-free',
                                   numerical_value=pdb.get_free_r_value())
                except ValueError:
                    logger.error('PDB field "r-free" could not be set for '
                                 'publication Id %i \n %s' %
                                 (pub.id, traceback.format_exc()))

                add_if_missing(pdb_parameter_set,
                               'space-group',
                               string_value=pdb.get_spacegroup())
                add_if_missing(pdb_parameter_set,
                               'unit-cell',
                               string_value=pdb.get_unit_cell())

                # 4. insert sequence info (lazy checking)
                pdb_seq_parameter_sets = ExperimentParameterSet.objects.filter(
                    schema__namespace=getattr(
                        settings, 'PDB_SEQUENCE_PUBLICATION_SCHEMA',
                        default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA),
                    experiment=pub)
                if pdb_seq_parameter_sets.count() == 0:
                    # insert seqences
                    for seq in pdb.get_sequence_info():
                        seq_ps_namespace = getattr(
                            settings, 'PDB_SEQUENCE_PUBLICATION_SCHEMA',
                            default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA)
                        seq_parameter_set = ExperimentParameterSet(
                            schema=Schema.objects.get(
                                namespace=seq_ps_namespace),
                            experiment=pub)
                        seq_parameter_set.save()
                        add_if_missing(seq_parameter_set,
                                       'organism',
                                       string_value=seq['organism'])
                        add_if_missing(seq_parameter_set,
                                       'expression-system',
                                       string_value=seq['expression_system'])
                        add_if_missing(seq_parameter_set,
                                       'sequence',
                                       string_value=seq['sequence'])

                # 5. insert/update citation info (aggressive)
                ExperimentParameterSet.objects.filter(
                    schema__namespace=getattr(
                        settings, 'PDB_CITATION_PUBLICATION_SCHEMA',
                        default_settings.PDB_CITATION_PUBLICATION_SCHEMA),
                    experiment=pub).delete()
                for citation in pdb.get_citations():
                    cit_ps_namespace = getattr(
                        settings, 'PDB_CITATION_PUBLICATION_SCHEMA',
                        default_settings.PDB_CITATION_PUBLICATION_SCHEMA)
                    cit_parameter_set = ExperimentParameterSet(
                        schema=Schema.objects.get(namespace=cit_ps_namespace),
                        experiment=pub)
                    cit_parameter_set.save()
                    add_if_missing(cit_parameter_set,
                                   'title',
                                   string_value=citation['title'])
                    add_if_missing(cit_parameter_set,
                                   'authors',
                                   string_value='; '.join(citation['authors']))
                    add_if_missing(cit_parameter_set,
                                   'journal',
                                   string_value=citation['journal'])
                    add_if_missing(cit_parameter_set,
                                   'volume',
                                   string_value=citation['volume'])
                    add_if_missing(cit_parameter_set,
                                   'page-range',
                                   string_value='-'.join([
                                       citation['page_first'],
                                       citation['page_last']
                                   ]))
                    add_if_missing(cit_parameter_set,
                                   'doi',
                                   string_value='http://dx.doi.org/' +
                                   citation['doi'])

                # 6. Remove the PDB embargo if set, since the update has
                # occurred and therefore the PDB must have been relased.
                try:
                    ExperimentParameter.objects.get(
                        name__name='pdb-embargo',
                        parameterset__schema__namespace=getattr(
                            settings, 'PUBLICATION_SCHEMA_ROOT',
                            default_settings.PUBLICATION_SCHEMA_ROOT)).delete(
                            )
                except ExperimentParameter.DoesNotExist:
                    pass

                # 7. Set the last update parameter to be now
                if pdb_last_update_parameter is None:
                    pub_parameter_set = ExperimentParameterSet(
                        schema=Schema.objects.get(namespace=PUB_SCHEMA),
                        experiment=pub)
                    pub_parameter_set.save()
                    pdb_last_update_parameter = ExperimentParameter(
                        name=last_update_parameter_name,
                        parameterset=pub_parameter_set,
                        datetime_value=timezone.now())
                else:
                    pdb_last_update_parameter.datetime_value = timezone.now()
                pdb_last_update_parameter.save()

            except CifFile.StarError:
                # PDB is either unavailable or invalid
                # (maybe notify the user somehow?)
                continue
class ParameterSetManager(object):

    parameterset = None
    parameters = None       # queryset of parameters
    blank_param = None

    # parameterset OR schema / datafile / dataset / experiment
    # delete dataset creation code
    # make parameterset / object arguments generic and test type
    # create function to return generic parameter type for setting/getting
    def __init__(self, parameterset=None, parentObject=None,
                 schema=None):
        """
        instantiate new task or existing ParameterSet
        :param dataset: optional parameter to instanciate task from
          metadata, will be tested for completeness and copied into
          new task if complete
        :type dataset: Dataset
        """

        if parameterset:
            self.parameterset = parameterset
            self.schema = self.parameterset.schema
            self.namespace = self.schema.namespace

            if type(self.parameterset).__name__ == "DatafileParameterSet":
                self.parameters = DatafileParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = DatafileParameter

            elif type(self.parameterset).__name__ == "DatasetParameterSet":
                self.parameters = DatasetParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = DatasetParameter

            elif type(self.parameterset).__name__ == "ExperimentParameterSet":
                self.parameters = ExperimentParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = ExperimentParameter

            else:
                raise TypeError("Invalid parameterset object given.")

        elif parentObject and schema:

            self.namespace = schema

            if type(parentObject).__name__ == "Dataset_File":
                self.parameterset = DatafileParameterSet(
                    schema=self.get_schema(), dataset_file=parentObject)

                self.parameterset.save()

                self.parameters = DatafileParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = DatafileParameter

            elif type(parentObject).__name__ == "Dataset":
                self.parameterset = DatasetParameterSet(
                    schema=self.get_schema(), dataset=parentObject)

                self.parameterset.save()

                self.parameters = DatasetParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = DatasetParameter

            elif type(parentObject).__name__ == "Experiment":
                self.parameterset = ExperimentParameterSet(
                    schema=self.get_schema(), experiment=parentObject)

                self.parameterset.save()

                self.parameters = ExperimentParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = ExperimentParameter

            else:
                raise TypeError("Invalid parent object." +
                    "Must be an experiment/dataset/datafile")

        else:
            raise TypeError("Missing arguments")

    def get_schema(self):
        try:
            schema = Schema.objects.get(
                namespace=self.namespace)
        except ObjectDoesNotExist:
            schema = Schema()
            schema.namespace = self.namespace
            schema.save()
            self.schema = schema
        return schema

    def get_param(self, parname, value=False):
        par = self.parameters.get(name__name=parname)
        if value:
            if par.name.isNumeric():
                return par.numerical_value
            else:
                return par.string_value
        return par

    def get_params(self, parname, value=False):
        pars = self.parameters.filter(name__name=parname)
        if value:
            if len(pars) > 0 and pars[0].name.isNumeric():
                return [par.numerical_value
                        for par in pars]
            else:
                return [par.string_value
                        for par in pars]
        return pars

    def set_param(self, parname, value, fullparname=None,
                  example_value=None):
        try:
            param = self.get_param(parname)
        except ObjectDoesNotExist:
            param = self.blank_param()
            param.parameterset = self.parameterset
            param.name = self._get_create_parname(parname, fullparname,
                example_value=example_value)
            #param.string_value = value
            #param.save()
        if param.name.isNumeric():
            param.numerical_value = float(value)
        else:
            param.string_value = str(value)
        param.save()

    def new_param(self, parname, value, fullparname=None):
        param = self.blank_param()
        param.parameterset = self.parameterset
        param.name = self._get_create_parname(parname, fullparname)
        param.string_value = value
        param.save()
        if param.name.isNumeric():
            param.numerical_value = float(value)
        else:
            param.string_value = str(value)
        param.save()

        # use this one from post data
    def set_param_list(self, parname, value_list, fullparname=None):
        self.delete_params(parname)
        for value in value_list:
            if value != None:
                self.new_param(parname, value, fullparname)

    def set_params_from_dict(self, dict):
        print type(dict)
        for (key, value) in dict.iteritems():
            if type(value) is list:
                self.set_param_list(key, value)
            else:
                if value != None:
                    self.delete_params(key)
                    self.set_param(key, value)

    def delete_params(self, parname):
        params = self.get_params(parname)
        for param in params:
            param.delete()

    def delete_all_params(self):
        for param in self.parameters:
            param.delete()

    def _get_create_parname(self, parname,
                            fullparname=None, example_value=None):
        try:
            paramName = ParameterName.objects.get(name=parname,
                               schema__id=self.get_schema().id)
        except ObjectDoesNotExist:
            paramName = ParameterName()
            paramName.schema = self.get_schema()
            paramName.name = parname
            if fullparname:
                paramName.full_name = fullparname
            else:
                paramName.full_name = parname
            if example_value:
                try:
                    float(example_value)
                    paramName.data_type = ParameterName.NUMERIC
                except (TypeError, ValueError):
                    paramName.data_type = ParameterName.STRING
            else:
                paramName.data_type = ParameterName.STRING
            paramName.is_searchable = True
            paramName.save()
        return paramName
Ejemplo n.º 15
0
def process_form(request):
    # Decode the form data
    form_state = json.loads(request.body)

    def validation_error(error=None):
        if error is None:
            error = 'Invalid form data was submitted ' \
                    '(server-side validation failed)'
        return HttpResponse(json.dumps({'error': error}),
                            content_type="application/json")

    # Check if the form data contains a publication ID
    # If it does, then this publication needs to be updated
    # rather than created.
    if 'publicationId' not in form_state:
        if not form_state['publicationTitle'].strip():
            return validation_error()
        publication = create_draft_publication(
            request.user, form_state['publicationTitle'],
            form_state['publicationDescription'])
        form_state['publicationId'] = publication.id
    else:
        publication = get_draft_publication(request.user,
                                            form_state['publicationId'])
        # Check if the publication is finalised (i.e. not in draft)
        # if it is, then refuse to process the form.
        if publication is None or not publication.is_publication_draft():
            return HttpResponseForbidden()

    # Get the form state database object
    form_state_parameter = ExperimentParameter.objects.get(
        name__name='form_state',
        name__schema__namespace=getattr(
            settings, 'PUBLICATION_SCHEMA_ROOT',
            default_settings.PUBLICATION_SCHEMA_ROOT),
        parameterset__experiment=publication)

    # Check if the form state needs to be loaded (i.e. a publication draft
    # is resumed)
    # no database changes are made if the form is resumed
    if form_state['action'] == 'resume':
        form_state = json.loads(form_state_parameter.string_value)
        return HttpResponse(json.dumps(form_state),
                            content_type="application/json")

    if form_state['action'] == 'update-dataset-selection':
        # Update the publication title/description if changed.
        # Must not be blank.
        if not form_state['publicationTitle'].strip() or \
                not form_state['publicationDescription'].strip():
            return validation_error()

        if publication.title != form_state['publicationTitle']:
            publication.title = form_state['publicationTitle']
            publication.save()
        if publication.description != form_state['publicationDescription']:
            publication.description = form_state['publicationDescription']
            publication.save()

        # Update associated datasets
        # (note: might not be efficient re: db queries)
        # ... first clear all current associations
        current_datasets = Dataset.objects.filter(experiments=publication)
        for current_dataset in current_datasets:
            current_dataset.experiments.remove(publication)
        # ... now (re)add all datasets
        selected_datasets = [
            ds['dataset']['id'] for ds in form_state['addedDatasets']
        ]
        datasets = Dataset.objects.filter(
            experiments__in=Experiment.safe.owned_and_shared(request.user),
            pk__in=selected_datasets).distinct()

        for dataset in datasets:
            dataset.experiments.add(publication)

        # --- Get data for the next page --- #
        # Construct the disclipline-specific form based on the
        # selected datasets
        selected_forms = select_forms(datasets)
        if 'disciplineSpecificFormTemplates' in form_state:
            # clear extraInfo if the selected forms differ
            # (i.e. datasets have changed)
            if json.dumps(selected_forms) != json.dumps(
                    form_state['disciplineSpecificFormTemplates']):
                form_state['extraInfo'] = {}
        form_state['disciplineSpecificFormTemplates'] = selected_forms

    elif form_state['action'] == 'update-extra-info':
        # Clear any current parameter sets except for those belonging
        # to the publication draft schema or containing the form_state
        # parameter
        clear_publication_metadata(publication)

        # Loop through form data and create associates parameter sets
        # Any unrecognised fields or schemas are ignored!
        map_form_to_schemas(form_state['extraInfo'], publication)

        # *** Synchrotron specific ***
        # Search for beamline/EPN information associated with each dataset
        # and add to the publication.
        synchrotron_search_epn(publication)

        # --- Get data for the next page --- #
        licenses_json = get_licenses()
        form_state['licenses'] = licenses_json

        # Select the first license as default
        if licenses_json:
            if 'selectedLicenseId' not in form_state:
                form_state['selectedLicenseId'] = licenses_json[0]['id']
        else:  # No licenses configured...
            form_state['selectedLicenseId'] = -1

        # Set a default author (current user) if no previously saved data
        # By default, the form sends a list of authors of one element
        # with blank fields
        if len(form_state['authors']) == 1 and \
                not form_state['authors'][0]['name']:
            form_state['authors'] = [{
                'name':
                ' '.join([request.user.first_name, request.user.last_name]),
                'institution':
                getattr(settings, 'DEFAULT_INSTITUTION', ''),
                'email':
                request.user.email
            }]

    elif form_state['action'] == 'submit':
        # any final form validation should occur here
        # and specific error messages can be returned
        # to the browser before the publication's draft
        # status is removed.

        if 'acknowledge' not in form_state or not form_state['acknowledge']:
            return validation_error('You must confirm that you are '
                                    'authorised to submit this publication')

        set_publication_authors(form_state['authors'], publication)

        institutions = '; '.join(
            set([author['institution'] for author in form_state['authors']]))
        publication.institution_name = institutions

        # Attach the publication details schema
        pub_details_schema = Schema.objects.get(
            namespace=getattr(settings, 'PUBLICATION_DETAILS_SCHEMA',
                              default_settings.PUBLICATION_DETAILS_SCHEMA))
        pub_details_parameter_set = ExperimentParameterSet(
            schema=pub_details_schema, experiment=publication)
        pub_details_parameter_set.save()

        # Add the acknowledgements
        acknowledgements_parameter_name = ParameterName.objects.get(
            schema=pub_details_schema, name='acknowledgements')
        ExperimentParameter(
            name=acknowledgements_parameter_name,
            parameterset=pub_details_parameter_set,
            string_value=form_state['acknowledgements']).save()

        # Set the release date
        set_embargo_release_date(publication,
                                 dateutil.parser.parse(form_state['embargo']))

        # Set the license
        try:
            publication.license = License.objects.get(
                pk=form_state['selectedLicenseId'],
                is_active=True,
                allows_distribution=True)
        except License.DoesNotExist:
            publication.license = License.get_none_option_license()

        publication.save()

        # Send emails about publication in draft
        subject, message_content = email_pub_requires_authorisation(
            request.user.username,
            request.build_absolute_uri(
                reverse('tardis_portal.view_experiment',
                        args=(publication.id, ))),
            request.build_absolute_uri('/apps/publication-forms/approvals/'))

        try:
            send_mail(
                subject,
                message_content,
                getattr(
                    settings, 'PUBLICATION_NOTIFICATION_SENDER_EMAIL',
                    default_settings.PUBLICATION_NOTIFICATION_SENDER_EMAIL),
                get_pub_admin_email_addresses(),
                fail_silently=False)

            subject, message_content = email_pub_awaiting_approval(
                publication.title)
            send_mail_to_authors(publication,
                                 subject,
                                 message_content,
                                 fail_silently=False)
        except Exception as e:
            logger.error(
                "failed to send publication notification email(s): %s" %
                repr(e))
            return HttpResponse(json.dumps({
                'error':
                'Failed to send notification email - please '
                'contact the %s administrator (%s), '
                'or try again later. Your draft is saved.' %
                (get_site_admin_email(),
                 getattr(settings, 'SITE_TITLE', 'MyTardis'))
            }),
                                content_type="application/json")

        # Remove the draft status
        remove_draft_status(publication)

        # Automatically approve publications if approval is not required
        if not getattr(settings, 'PUBLICATIONS_REQUIRE_APPROVAL',
                       default_settings.PUBLICATIONS_REQUIRE_APPROVAL):
            approve_publication(request,
                                publication,
                                message=None,
                                send_email=False)
            # approve_publication will delete the form state, so don't
            # bother saving is and return.
            form_state['action'] = ''
            return HttpResponse(json.dumps(form_state),
                                content_type="appication/json")

        # Trigger publication record update
        tasks.update_publication_records.delay()

    # Clear the form action and save the state
    form_state['action'] = ''
    form_state_parameter.string_value = json.dumps(form_state)
    form_state_parameter.save()

    return HttpResponse(json.dumps(form_state), content_type="appication/json")
Ejemplo n.º 16
0
def populate_pdb_pub_records():
    PUB_SCHEMA = getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
                         default_settings.PUBLICATION_SCHEMA_ROOT)
    PUB_SCHEMA_DRAFT = getattr(settings, 'PUBLICATION_DRAFT_SCHEMA',
                               default_settings.PUBLICATION_DRAFT_SCHEMA)
    PDB_SCHEMA = getattr(settings, 'PDB_PUBLICATION_SCHEMA_ROOT',
                         default_settings.PDB_PUBLICATION_SCHEMA_ROOT)
    publications = Experiment.objects \
        .filter(experimentparameterset__schema__namespace=PDB_SCHEMA) \
        .filter(experimentparameterset__schema__namespace=PUB_SCHEMA) \
        .exclude(experimentparameterset__schema__namespace=PUB_SCHEMA_DRAFT) \
        .distinct()

    last_update_parameter_name = ParameterName.objects.get(
        name='pdb-last-sync',
        schema__namespace=PUB_SCHEMA)

    def add_if_missing(parameterset, name, string_value=None,
                       numerical_value=None, datetime_value=None):
        try:
            ExperimentParameter.objects.get(
                name__name=name, parameterset=parameterset)
        except ExperimentParameter.DoesNotExist:
            param_name = ParameterName.objects.get(
                name=name, schema=parameterset.schema)
            param = ExperimentParameter(name=param_name,
                                        parameterset=parameterset)
            param.string_value = string_value
            param.numerical_value = numerical_value
            param.datetime_value = datetime_value
            param.save()

    for pub in publications:
        try:
            # try to get the last update time for the PDB data
            pdb_last_update_parameter = ExperimentParameter.objects.get(
                parameterset__schema__namespace=PUB_SCHEMA,
                name=last_update_parameter_name,
                parameterset__experiment=pub
            )
            last_update = pdb_last_update_parameter.datetime_value
            needs_update = last_update + \
                getattr(settings,
                        'PDB_REFRESH_INTERVAL',
                        default_settings.PDB_REFRESH_INTERVAL) \
                < timezone.now()

        except ExperimentParameter.DoesNotExist:
            # if the PDB last update time parameter doesn't exist,
            # we definitely need to update the data and create a last
            # update entry
            needs_update = True
            pdb_last_update_parameter = None

        # If an update needs to happen...
        if needs_update:
            # 1. get the PDB info
            pdb_parameter_set = ExperimentParameterSet.objects.get(
                schema__namespace=getattr(
                    settings,
                    'PDB_PUBLICATION_SCHEMA_ROOT',
                    default_settings.PDB_PUBLICATION_SCHEMA_ROOT),
                experiment=pub)
            pdb = ExperimentParameter.objects.get(
                name__name='pdb-id',
                parameterset=pdb_parameter_set)
            pdb_id = pdb.string_value
            # 1a. cosmetic change of case for PDB ID, if entered incorrectly
            if pdb_id != pdb_id.upper():
                pdb.string_value = pdb_id.upper()
                pdb.save()

            try:
                # 2. fetch the info from pdb.org
                pdb = PDBCifHelper(pdb_id)

                # 3. insert all standard pdb parameters
                add_if_missing(pdb_parameter_set, 'title',
                               string_value=pdb.get_pdb_title())
                add_if_missing(pdb_parameter_set, 'url',
                               string_value=pdb.get_pdb_url())
                try:
                    add_if_missing(pdb_parameter_set, 'resolution',
                                   numerical_value=pdb.get_resolution())
                except ValueError:
                    logger.error(
                        'PDB field "resolution" could not be set for '
                        'publication Id %i \n %s' %
                        (pub.id, traceback.format_exc()))

                try:
                    add_if_missing(pdb_parameter_set, 'r-value',
                                   numerical_value=pdb.get_obs_r_value())
                except ValueError:
                    logger.error(
                        'PDB field "r-value" could not be set for '
                        'publication Id %i \n %s' %
                        (pub.id, traceback.format_exc()))

                try:
                    add_if_missing(pdb_parameter_set, 'r-free',
                                   numerical_value=pdb.get_free_r_value())
                except ValueError:
                    logger.error(
                        'PDB field "r-free" could not be set for '
                        'publication Id %i \n %s' %
                        (pub.id, traceback.format_exc()))

                add_if_missing(pdb_parameter_set, 'space-group',
                               string_value=pdb.get_spacegroup())
                add_if_missing(pdb_parameter_set, 'unit-cell',
                               string_value=pdb.get_unit_cell())

                # 4. insert sequence info (lazy checking)
                pdb_seq_parameter_sets = ExperimentParameterSet.objects.filter(
                    schema__namespace=getattr(
                        settings,
                        'PDB_SEQUENCE_PUBLICATION_SCHEMA',
                        default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA),
                    experiment=pub)
                if pdb_seq_parameter_sets.count() == 0:
                    # insert seqences
                    for seq in pdb.get_sequence_info():
                        seq_ps_namespace = getattr(
                            settings,
                            'PDB_SEQUENCE_PUBLICATION_SCHEMA',
                            default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA)
                        seq_parameter_set = ExperimentParameterSet(
                            schema=Schema.objects.get(
                                namespace=seq_ps_namespace),
                            experiment=pub)
                        seq_parameter_set.save()
                        add_if_missing(seq_parameter_set, 'organism',
                                       string_value=seq['organism'])
                        add_if_missing(seq_parameter_set, 'expression-system',
                                       string_value=seq['expression_system'])
                        add_if_missing(seq_parameter_set, 'sequence',
                                       string_value=seq['sequence'])

                # 5. insert/update citation info (aggressive)
                ExperimentParameterSet.objects.filter(
                    schema__namespace=getattr(
                        settings,
                        'PDB_CITATION_PUBLICATION_SCHEMA',
                        default_settings.PDB_CITATION_PUBLICATION_SCHEMA),
                    experiment=pub).delete()
                for citation in pdb.get_citations():
                    cit_ps_namespace = getattr(
                        settings,
                        'PDB_CITATION_PUBLICATION_SCHEMA',
                        default_settings.PDB_CITATION_PUBLICATION_SCHEMA)
                    cit_parameter_set = ExperimentParameterSet(
                        schema=Schema.objects.get(namespace=cit_ps_namespace),
                        experiment=pub)
                    cit_parameter_set.save()
                    add_if_missing(cit_parameter_set, 'title',
                                   string_value=citation['title'])
                    add_if_missing(cit_parameter_set, 'authors',
                                   string_value='; '.join(citation['authors']))
                    add_if_missing(cit_parameter_set, 'journal',
                                   string_value=citation['journal'])
                    add_if_missing(cit_parameter_set, 'volume',
                                   string_value=citation['volume'])
                    add_if_missing(cit_parameter_set, 'page-range',
                                   string_value='-'.join(
                                       [citation['page_first'],
                                        citation['page_last']]))
                    add_if_missing(cit_parameter_set, 'doi',
                                   string_value='http://dx.doi.org/' +
                                                citation['doi'])

                # 6. Remove the PDB embargo if set, since the update has
                # occurred and therefore the PDB must have been relased.
                try:
                    ExperimentParameter.objects.get(
                        name__name='pdb-embargo',
                        parameterset__schema__namespace=getattr(
                            settings,
                            'PUBLICATION_SCHEMA_ROOT',
                            default_settings.PUBLICATION_SCHEMA_ROOT)).delete()
                except ExperimentParameter.DoesNotExist:
                    pass

                # 7. Set the last update parameter to be now
                if pdb_last_update_parameter is None:
                    pub_parameter_set = ExperimentParameterSet(
                        schema=Schema.objects.get(namespace=PUB_SCHEMA),
                        experiment=pub)
                    pub_parameter_set.save()
                    pdb_last_update_parameter = ExperimentParameter(
                        name=last_update_parameter_name,
                        parameterset=pub_parameter_set,
                        datetime_value=timezone.now())
                else:
                    pdb_last_update_parameter.datetime_value = timezone.now()
                pdb_last_update_parameter.save()

            except CifFile.StarError:
                # PDB is either unavailable or invalid
                # (maybe notify the user somehow?)
                continue
Ejemplo n.º 17
0
class ParameterSetManager(object):

    parameterset = None
    parameters = None       # queryset of parameters
    blank_param = None

    # parameterset OR schema / datafile / dataset / experiment
    # delete dataset creation code
    # make parameterset / object arguments generic and test type
    # create function to return generic parameter type for setting/getting
    #
    # 2011/07/20 (Ryan Braganza)
    # changed type checking from:
    #     type(self.parameterset).__name__ == 'ClassName'
    # to:
    #     isinstance(self.parameterset, Class)
    # This is to solve an issue with models using deferred fields.
    # When using deferred fields, a subclass is used, so the class name
    # will not match when using type(self.parameterset).__name__
    # isinstance deals with inheritance appropriately here
    def __init__(self, parameterset=None, parentObject=None,
                 schema=None):
        """
        instantiate new task or existing ParameterSet
        :param dataset: optional parameter to instanciate task from
          metadata, will be tested for completeness and copied into
          new task if complete
        :type dataset: Dataset
        :param schema: Schema namespace
        :type schema: string
        """

        if parameterset:
            self.parameterset = parameterset
            self.schema = self.parameterset.schema
            self.namespace = self.schema.namespace

            if isinstance(self.parameterset, DatafileParameterSet):
                self.parameters = DatafileParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = DatafileParameter

            elif isinstance(self.parameterset, DatasetParameterSet):
                self.parameters = DatasetParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = DatasetParameter

            elif isinstance(self.parameterset, ExperimentParameterSet):
                self.parameters = ExperimentParameter.objects.filter(\
                   parameterset=self.parameterset).order_by('name__full_name')

                self.blank_param = ExperimentParameter

            else:
                raise TypeError("Invalid parameterset object given.")

        elif parentObject and schema:

            self.namespace = schema

            if isinstance(parentObject, Dataset_File):
                self.parameterset = DatafileParameterSet(
                    schema=self.get_schema(), dataset_file=parentObject)

                self.parameterset.save()

                self.parameters = DatafileParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = DatafileParameter

            elif isinstance(parentObject, Dataset):
                self.parameterset = DatasetParameterSet(
                    schema=self.get_schema(), dataset=parentObject)

                self.parameterset.save()

                self.parameters = DatasetParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = DatasetParameter

            elif isinstance(parentObject, Experiment):
                self.parameterset = ExperimentParameterSet(
                    schema=self.get_schema(), experiment=parentObject)

                self.parameterset.save()

                self.parameters = ExperimentParameter.objects.filter(
                    parameterset=self.parameterset)

                self.blank_param = ExperimentParameter

            else:
                raise TypeError("Invalid parent object." +
                    "Must be an experiment/dataset/datafile not " + str(type(parentObject)))

        else:
            raise TypeError("Missing arguments")

    def get_schema(self):
        try:
            schema = Schema.objects.get(
                namespace=self.namespace)
        except ObjectDoesNotExist:
            schema = Schema()
            schema.namespace = self.namespace
            schema.save()
            self.schema = schema
        return schema

    def get_param(self, parname, value=False):
        par = self.parameters.get(name__name=parname)
        if value:
            if par.name.isNumeric():
                return par.numerical_value
            else:
                return par.string_value
        return par

    def get_params(self, parname, value=False):
        pars = self.parameters.filter(name__name=parname)
        if value:
            if len(pars) > 0 and pars[0].name.isNumeric():
                return [par.numerical_value
                        for par in pars]
            else:
                return [par.string_value
                        for par in pars]
        return pars

    def set_param(self, parname, value, fullparname=None,
                  example_value=None):
        try:
            param = self.get_param(parname)
        except ObjectDoesNotExist:
            param = self.blank_param()
            param.parameterset = self.parameterset
            param.name = self._get_create_parname(parname, fullparname,
                example_value=example_value)
            #param.string_value = value
            #param.save()
        if param.name.isNumeric():
            param.numerical_value = float(value)
        else:
            param.string_value = unicode(value)
        param.save()
        return param.id

    def new_param(self, parname, value, fullparname=None):
        param = self.blank_param()
        param.parameterset = self.parameterset
        param.name = self._get_create_parname(parname, fullparname)
        param.string_value = value
        param.save()
        if param.name.isNumeric():
            param.numerical_value = float(value)
        else:
            param.string_value = unicode(value)
        param.save()
        return param.id

        # use this one from post data
    def set_param_list(self, parname, value_list, fullparname=None):
        self.delete_params(parname)
        for value in value_list:
            if value != None:
                self.new_param(parname, value, fullparname)

    def set_params_from_dict(self, dict):
        for (key, value) in dict.iteritems():
            if type(value) is list:
                self.set_param_list(key, value)
            else:
                if value != None:
                    self.delete_params(key)
                    self.set_param(key, value)

    def delete_params(self, parname):
        params = self.get_params(parname)
        for param in params:
            param.delete()

    def delete_all_params(self):
        for param in self.parameters:
            param.delete()

    def _get_create_parname(self, parname,
                            fullparname=None, example_value=None):
        try:
            paramName = ParameterName.objects.get(name=parname,
                               schema__id=self.get_schema().id)
        except ObjectDoesNotExist:
            paramName = ParameterName()
            paramName.schema = self.get_schema()
            paramName.name = parname
            if fullparname:
                paramName.full_name = fullparname
            else:
                paramName.full_name = parname
            if example_value:
                try:
                    float(example_value)
                    paramName.data_type = ParameterName.NUMERIC
                except (TypeError, ValueError):
                    paramName.data_type = ParameterName.STRING
            else:
                paramName.data_type = ParameterName.STRING
            paramName.is_searchable = True
            paramName.save()
        return paramName
Ejemplo n.º 18
0
    def test_parameter(self):
        exp = Experiment(
            title='test exp1',
            institution_name='Australian Synchrotron',
            approved=True,
            created_by=self.user,
            public_access=Experiment.PUBLIC_ACCESS_NONE,
        )
        exp.save()

        dataset = Dataset(description="dataset description")
        dataset.save()
        dataset.experiments.add(exp)
        dataset.save()

        df_file = DataFile(dataset=dataset,
                           filename='file.txt',
                           size=42,
                           md5sum='bogus')
        df_file.save()

        df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml',
                           type=Schema.DATAFILE)
        df_schema.save()

        ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml',
                           type=Schema.DATASET)
        ds_schema.save()

        exp_schema = Schema(
            namespace='http://www.cern.ch/felzmann/schema3.xml',
            type=Schema.EXPERIMENT)
        exp_schema.save()

        df_parname = ParameterName(schema=df_schema,
                                   name='name',
                                   full_name='full_name',
                                   units='image/jpg',
                                   data_type=ParameterName.FILENAME)
        df_parname.save()

        ds_parname = ParameterName(schema=ds_schema,
                                   name='name',
                                   full_name='full_name',
                                   units='image/jpg',
                                   data_type=ParameterName.FILENAME)
        ds_parname.save()

        exp_parname = ParameterName(schema=exp_schema,
                                    name='name',
                                    full_name='full_name',
                                    units='image/jpg',
                                    data_type=ParameterName.FILENAME)
        exp_parname.save()

        df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file)
        df_parset.save()

        ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset)
        ds_parset.save()

        exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp)
        exp_parset.save()

        with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)):
            filename = 'test.jpg'
            df_parameter = DatafileParameter(name=df_parname,
                                             parameterset=df_parset,
                                             string_value=filename)
            df_parameter.save()

            ds_parameter = DatasetParameter(name=ds_parname,
                                            parameterset=ds_parset,
                                            string_value=filename)
            ds_parameter.save()

            exp_parameter = ExperimentParameter(name=exp_parname,
                                                parameterset=exp_parset,
                                                string_value=filename)
            exp_parameter.save()

            self.assertEqual(
                "<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>"
                %  # noqa
                (df_parameter.id, df_parameter.id),
                df_parameter.get())

            self.assertEqual(
                "<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>"
                %  # noqa
                (ds_parameter.id, ds_parameter.id),
                ds_parameter.get())

            self.assertEqual(
                "<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>"
                %  # noqa
                (exp_parameter.id, exp_parameter.id),
                exp_parameter.get())