def map_form_to_schemas(extraInfo, publication): for form_id, form in extraInfo.iteritems(): try: # Ignore form if no schema exists with this name schema = Schema.objects.get(namespace=form['schema']) except Schema.DoesNotExist: continue parameter_set = ExperimentParameterSet( schema=schema, experiment=publication) parameter_set.save() for key, value in form.iteritems(): if key != 'schema': try: # Ignore field if parameter name (key) doesn't match parameter_name = ParameterName.objects.get( schema=schema, name=key) if parameter_name.isNumeric(): parameter = ExperimentParameter( name=parameter_name, parameterset=parameter_set, numerical_value=float(value)) elif parameter_name.isLongString() or \ parameter_name.isString() or \ parameter_name.isURL() or \ parameter_name.isLink() or \ parameter_name.isFilename(): parameter = ExperimentParameter( name=parameter_name, parameterset=parameter_set, string_value=str(value)) else: # Shouldn't happen, but here in case the parameter type # is non-standard continue parameter.save() except ParameterName.DoesNotExist: pass
def save_rif_cs_profile(self, experiment, profile): """ Save selected profile choice as experiment parameter """ namespace = "http://monash.edu.au/rif-cs/profile/" schema = None try: schema = Schema.objects.get( namespace__exact=namespace) except Schema.DoesNotExist: logger.debug('Schema ' + namespace + ' does not exist. Creating.') schema = Schema(namespace=namespace) schema.save() parametername = ParameterName.objects.get( schema__namespace__exact=schema.namespace, name="profile") parameterset = None try: parameterset = \ ExperimentParameterSet.objects.get(\ schema=schema, experiment=experiment) except ExperimentParameterSet.DoesNotExist, e: parameterset = ExperimentParameterSet(\ schema=schema, experiment=experiment) parameterset.save()
def create_draft_publication(user, publication_title, publication_description): # Note: Maybe this logic can be taken from the tardis_portal/views.py? experiment = Experiment(created_by=user, title=publication_title, description=publication_description) experiment.save() ObjectACL(content_object=experiment, pluginId=django_user, entityId=str(user.id), canRead=True, canWrite=False, canDelete=False, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED).save() ObjectACL(content_object=experiment, pluginId=django_group, entityId=str( Group.objects.get_or_create( name=getattr( settings, 'PUBLICATION_OWNER_GROUP', default_settings.PUBLICATION_OWNER_GROUP))[0].id), canRead=True, canWrite=True, canDelete=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED).save() publication_schema = Schema.objects.get( namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT)) # Attach draft schema draft_publication_schema = Schema.objects.get( namespace=getattr(settings, 'PUBLICATION_DRAFT_SCHEMA', default_settings.PUBLICATION_DRAFT_SCHEMA)) ExperimentParameterSet(schema=draft_publication_schema, experiment=experiment).save() # Attach root schema and blank form_state parameter publication_root_schema = Schema.objects.get( namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT)) publication_root_parameter_set = ExperimentParameterSet( schema=publication_schema, experiment=experiment) publication_root_parameter_set.save() form_state_param_name = ParameterName.objects.get( schema=publication_root_schema, name='form_state') ExperimentParameter(name=form_state_param_name, parameterset=publication_root_parameter_set).save() return experiment
def _create_related_info(request, experiment_id): if not authz.has_write_permissions(request, experiment_id): return return_response_error(request) form = RelatedInfoForm(json.loads(request.body)) if not form.is_valid(): return HttpResponse('', status=400) ps = ExperimentParameterSet(experiment_id=experiment_id, schema=_get_schema()) ps.save() ParameterSetManager(ps).set_params_from_dict(form.cleaned_data) return HttpResponse(json.dumps(_get_dict_from_ps(ps)), content_type='application/json; charset=utf-8', status=201)
def _create(self, request, experiment_id): from tardis.tardis_portal.auth.decorators import has_experiment_write if not has_experiment_write(request, experiment_id): return return_response_error(request) form = self.form_cls(json.loads(request.body)) if not form.is_valid(): return HttpResponse('', status=400) ps = ExperimentParameterSet(experiment_id=experiment_id, schema=self.schema) ps.save() ParameterSetManager(ps).set_params_from_dict(form.cleaned_data) return HttpResponse(json.dumps(self._get_dict_from_ps(ps)), content_type='application/json; charset=utf-8', status=201)
def set_embargo_release_date(publication, release_date): pub_schema_root = Schema.objects.get( namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT)) pub_schema_root_parameter_set = ExperimentParameterSet( schema=pub_schema_root, experiment=publication) pub_schema_root_parameter_set.save() embargo_parameter_name = ParameterName.objects.get( schema=pub_schema_root, name='embargo') ExperimentParameter(name=embargo_parameter_name, parameterset=pub_schema_root_parameter_set, datetime_value=release_date).save()
def add_info(self, cleaned_data): logger.debug('adding info') logger.debug(cleaned_data) type = cleaned_data['type'] identifier_type = cleaned_data['identifier_type'] identifier = cleaned_data['identifier'] title = cleaned_data['title'] notes = cleaned_data['notes'] eps = ExperimentParameterSet(experiment_id=self.experiment_id, schema=self.schema) eps.save() _maybe_add(eps, self.type_name, type) _maybe_add(eps, self.identifier_type_name, identifier_type, force=True) _maybe_add(eps, self.identifier_name, identifier, force=True) _maybe_add(eps, self.title_name, title) _maybe_add(eps, self.notes_name, notes)
def _save_party_refs(self, party, party_relation): """ Save party and party relation information as parameters on the experiment """ namespace = "http://rmit.edu.au/rif-cs/party/1.0/" logger.debug("saving party") schema = None try: schema = Schema.objects.get( namespace__exact=namespace) except Schema.DoesNotExist: logger.debug('Schema ' + namespace + ' does not exist. Creating.') schema = Schema(namespace=namespace) schema.save() exp = Experiment.objects.get(pk=self.experiment_id) party_id_param = self._make_param(schema=schema, name="party_id", paramtype=ParameterName.NUMERIC) relation_param = self._make_param(schema=schema, name="relationtocollection_id", paramtype=ParameterName.STRING) parameterset = ExperimentParameterSet(schema=schema, experiment=exp) parameterset.save() ep = ExperimentParameter.objects.filter(name=party_id_param, parameterset=parameterset, parameterset__experiment=exp) for p in ep: p.delete() ep = ExperimentParameter( parameterset=parameterset, name=party_id_param, numerical_value=party.pk) ep.save() ep = ExperimentParameter.objects.filter(name=relation_param, parameterset=parameterset, parameterset__experiment=exp) for p in ep: p.delete() ep = ExperimentParameter( parameterset=parameterset, name=relation_param, string_value=party_relation) ep.save()
def synchrotron_search_epn(publication): # *** Synchrotron specific *** # Search for beamline/EPN information associated with each dataset # and add to the publication. try: synch_epn_schema = Schema.objects.get( namespace='http://www.tardis.edu.au/schemas/as/' 'experiment/2010/09/21') datasets = Dataset.objects.filter(experiments=publication) synch_experiments = Experiment.objects.filter( datasets__in=datasets, experimentparameterset__schema=synch_epn_schema).exclude( pk=publication.pk).distinct() for exp in [s for s in synch_experiments if not s.is_publication()]: epn = ExperimentParameter.objects.get( name__name='EPN', name__schema=synch_epn_schema, parameterset__experiment=exp).string_value beamline = ExperimentParameter.objects.get( name__name='beamline', name__schema=synch_epn_schema, parameterset__experiment=exp).string_value epn_parameter_set = ExperimentParameterSet( schema=synch_epn_schema, experiment=publication) epn_parameter_set.save() epn_copy = ExperimentParameter( name=ParameterName.objects.get( name='EPN', schema=synch_epn_schema), parameterset=epn_parameter_set) epn_copy.string_value = epn epn_copy.save() beamline_copy = ExperimentParameter( name=ParameterName.objects.get( name='beamline', schema=synch_epn_schema), parameterset=epn_parameter_set) beamline_copy.string_value = beamline beamline_copy.save() except Schema.DoesNotExist: pass
def __init__(self, experiment_id, create=False): self.experiment = Experiment.objects.get(pk=experiment_id) parametersets = ExperimentParameterSet.objects.filter( schema__namespace=NAMESPACE, experiment__id=experiment_id) self.schema, _ = Schema.objects.get_or_create(namespace=NAMESPACE, name='Embargo Details') self.expiry_date, _ = ParameterName.objects.get_or_create(schema=self.schema, name=EXPIRY_DATE_KEY, full_name='Expiry', immutable=True, data_type=ParameterName.DATETIME) self.never_expire, _ = ParameterName.objects.get_or_create(schema=self.schema, name=NEVER_EXPIRE_KEY, full_name='Never Expires', immutable=True, data_type=ParameterName.STRING) if len(parametersets) == 1: self.parameterset = parametersets[0] elif create: self.parameterset = ExperimentParameterSet(experiment=self.experiment, schema=self.schema) self.parameterset.save() else: self.parameterset = None
class ParameterSetManager(object): parameterset = None parameters = None # queryset of parameters blank_param = None # parameterset OR schema / datafile / dataset / experiment # delete dataset creation code # make parameterset / object arguments generic and test type # create function to return generic parameter type for setting/getting # # 2011/07/20 (Ryan Braganza) # changed type checking from: # type(self.parameterset).__name__ == 'ClassName' # to: # isinstance(self.parameterset, Class) # This is to solve an issue with models using deferred fields. # When using deferred fields, a subclass is used, so the class name # will not match when using type(self.parameterset).__name__ # isinstance deals with inheritance appropriately here def __init__(self, parameterset=None, parentObject=None, schema=None): """ instantiate new task or existing ParameterSet :param dataset: optional parameter to instanciate task from metadata, will be tested for completeness and copied into new task if complete :type dataset: Dataset """ if parameterset: self.parameterset = parameterset self.schema = self.parameterset.schema self.namespace = self.schema.namespace if isinstance(self.parameterset, DatafileParameterSet): self.parameters = DatafileParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatafileParameter elif isinstance(self.parameterset, DatasetParameterSet): self.parameters = DatasetParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatasetParameter elif isinstance(self.parameterset, ExperimentParameterSet): self.parameters = ExperimentParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = ExperimentParameter else: raise TypeError("Invalid parameterset object given.") elif parentObject and schema: self.namespace = schema if isinstance(parentObject, Dataset_File): self.parameterset = DatafileParameterSet( schema=self.get_schema(), dataset_file=parentObject) self.parameterset.save() self.parameters = DatafileParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatafileParameter elif isinstance(parentObject, Dataset): self.parameterset = DatasetParameterSet( schema=self.get_schema(), dataset=parentObject) self.parameterset.save() self.parameters = DatasetParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatasetParameter elif isinstance(parentObject, Experiment): self.parameterset = ExperimentParameterSet( schema=self.get_schema(), experiment=parentObject) self.parameterset.save() self.parameters = ExperimentParameter.objects.filter( parameterset=self.parameterset) self.blank_param = ExperimentParameter else: raise TypeError("Invalid parent object." + "Must be an experiment/dataset/datafile not " + str(type(parentObject))) else: raise TypeError("Missing arguments") def get_schema(self): try: schema = Schema.objects.get(namespace=self.namespace) except ObjectDoesNotExist: schema = Schema() schema.namespace = self.namespace schema.save() self.schema = schema return schema def get_param(self, parname, value=False): par = self.parameters.get(name__name=parname) if value: if par.name.isNumeric(): return par.numerical_value else: return par.string_value return par def get_params(self, parname, value=False): pars = self.parameters.filter(name__name=parname) if value: if len(pars) > 0 and pars[0].name.isNumeric(): return [par.numerical_value for par in pars] else: return [par.string_value for par in pars] return pars def set_param(self, parname, value, fullparname=None, example_value=None): try: param = self.get_param(parname) except ObjectDoesNotExist: param = self.blank_param() param.parameterset = self.parameterset param.name = self._get_create_parname(parname, fullparname, example_value=example_value) #param.string_value = value #param.save() if param.name.isNumeric(): param.numerical_value = float(value) else: param.string_value = str(value) param.save() return param.id def new_param(self, parname, value, fullparname=None): param = self.blank_param() param.parameterset = self.parameterset param.name = self._get_create_parname(parname, fullparname) param.string_value = value param.save() if param.name.isNumeric(): param.numerical_value = float(value) else: param.string_value = str(value) param.save() return param.id # use this one from post data def set_param_list(self, parname, value_list, fullparname=None): self.delete_params(parname) for value in value_list: if value != None: self.new_param(parname, value, fullparname) def set_params_from_dict(self, dict): for (key, value) in dict.iteritems(): if type(value) is list: self.set_param_list(key, value) else: if value != None: self.delete_params(key) self.set_param(key, value) def delete_params(self, parname): params = self.get_params(parname) for param in params: param.delete() def delete_all_params(self): for param in self.parameters: param.delete() def _get_create_parname(self, parname, fullparname=None, example_value=None): try: paramName = ParameterName.objects.get( name=parname, schema__id=self.get_schema().id) except ObjectDoesNotExist: paramName = ParameterName() paramName.schema = self.get_schema() paramName.name = parname if fullparname: paramName.full_name = fullparname else: paramName.full_name = parname if example_value: try: float(example_value) paramName.data_type = ParameterName.NUMERIC except (TypeError, ValueError): paramName.data_type = ParameterName.STRING else: paramName.data_type = ParameterName.STRING paramName.is_searchable = True paramName.save() return paramName
def process_form(request): # Decode the form data form_state = json.loads(request.body) def validation_error(error=None): if error is None: error = 'Invalid form data was submitted ' \ '(server-side validation failed)' return HttpResponse( json.dumps({ 'error': error}), content_type="application/json") # Check if the form data contains a publication ID # If it does, then this publication needs to be updated # rather than created. if 'publicationId' not in form_state: if not form_state['publicationTitle'].strip(): return validation_error() publication = create_draft_publication( request.user, form_state['publicationTitle'], form_state['publicationDescription']) form_state['publicationId'] = publication.id else: publication = get_draft_publication( request.user, form_state['publicationId']) # Check if the publication is finalised (i.e. not in draft) # if it is, then refuse to process the form. if publication is None or not publication.is_publication_draft(): return HttpResponseForbidden() # Get the form state database object form_state_parameter = ExperimentParameter.objects.get( name__name='form_state', name__schema__namespace=getattr( settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT), parameterset__experiment=publication) # Check if the form state needs to be loaded (i.e. a publication draft # is resumed) # no database changes are made if the form is resumed if form_state['action'] == 'resume': form_state = json.loads(form_state_parameter.string_value) return HttpResponse(json.dumps(form_state), content_type="application/json") if form_state['action'] == 'update-dataset-selection': # Update the publication title/description if changed. # Must not be blank. if not form_state['publicationTitle'].strip() or \ not form_state['publicationDescription'].strip(): return validation_error() if publication.title != form_state['publicationTitle']: publication.title = form_state['publicationTitle'] publication.save() if publication.description != form_state['publicationDescription']: publication.description = form_state['publicationDescription'] publication.save() # Update associated datasets # (note: might not be efficient re: db queries) # ... first clear all current associations current_datasets = Dataset.objects.filter(experiments=publication) for current_dataset in current_datasets: current_dataset.experiments.remove(publication) # ... now (re)add all datasets selected_datasets = [ds['dataset']['id'] for ds in form_state['addedDatasets']] datasets = Dataset.objects.filter( experiments__in=Experiment.safe.owned_and_shared(request.user), pk__in=selected_datasets).distinct() for dataset in datasets: dataset.experiments.add(publication) # --- Get data for the next page --- # # Construct the disclipline-specific form based on the # selected datasets selected_forms = select_forms(datasets) if 'disciplineSpecificFormTemplates' in form_state: # clear extraInfo if the selected forms differ # (i.e. datasets have changed) if json.dumps(selected_forms) != json.dumps( form_state['disciplineSpecificFormTemplates']): form_state['extraInfo'] = {} form_state['disciplineSpecificFormTemplates'] = selected_forms elif form_state['action'] == 'update-extra-info': # Clear any current parameter sets except for those belonging # to the publication draft schema or containing the form_state # parameter clear_publication_metadata(publication) # Loop through form data and create associates parameter sets # Any unrecognised fields or schemas are ignored! map_form_to_schemas(form_state['extraInfo'], publication) # *** Synchrotron specific *** # Search for beamline/EPN information associated with each dataset # and add to the publication. synchrotron_search_epn(publication) # --- Get data for the next page --- # licenses_json = get_licenses() form_state['licenses'] = licenses_json # Select the first license as default if licenses_json: if 'selectedLicenseId' not in form_state: form_state['selectedLicenseId'] = licenses_json[0]['id'] else: # No licenses configured... form_state['selectedLicenseId'] = -1 # Set a default author (current user) if no previously saved data # By default, the form sends a list of authors of one element # with blank fields if len(form_state['authors']) == 1 and \ not form_state['authors'][0]['name']: form_state['authors'] = [ {'name': ' '.join([request.user.first_name, request.user.last_name]), 'institution': getattr(settings, 'DEFAULT_INSTITUTION', ''), 'email': request.user.email}] elif form_state['action'] == 'submit': # any final form validation should occur here # and specific error messages can be returned # to the browser before the publication's draft # status is removed. if 'acknowledge' not in form_state or not form_state['acknowledge']: return validation_error('You must confirm that you are ' 'authorised to submit this publication') set_publication_authors(form_state['authors'], publication) institutions = '; '.join( set([author['institution'] for author in form_state['authors']])) publication.institution_name = institutions # Attach the publication details schema pub_details_schema = Schema.objects.get( namespace=getattr(settings, 'PUBLICATION_DETAILS_SCHEMA', default_settings.PUBLICATION_DETAILS_SCHEMA)) pub_details_parameter_set = ExperimentParameterSet( schema=pub_details_schema, experiment=publication) pub_details_parameter_set.save() # Add the acknowledgements acknowledgements_parameter_name = ParameterName.objects.get( schema=pub_details_schema, name='acknowledgements') ExperimentParameter(name=acknowledgements_parameter_name, parameterset=pub_details_parameter_set, string_value=form_state['acknowledgements']).save() # Set the release date set_embargo_release_date( publication, dateutil.parser.parse( form_state[ 'embargo'])) # Set the license try: publication.license = License.objects.get( pk=form_state['selectedLicenseId'], is_active=True, allows_distribution=True) except License.DoesNotExist: publication.license = License.get_none_option_license() publication.save() # Send emails about publication in draft subject, message_content = email_pub_requires_authorisation( request.user.username, request.build_absolute_uri( reverse('tardis_portal.view_experiment', args=(publication.id,))), request.build_absolute_uri( '/apps/publication-forms/approvals/')) try: send_mail(subject, message_content, getattr( settings, 'PUBLICATION_NOTIFICATION_SENDER_EMAIL', default_settings.PUBLICATION_NOTIFICATION_SENDER_EMAIL), get_pub_admin_email_addresses(), fail_silently=False) subject, message_content = email_pub_awaiting_approval( publication.title) send_mail_to_authors(publication, subject, message_content, fail_silently=False) except Exception as e: logger.error( "failed to send publication notification email(s): %s" % repr(e) ) return HttpResponse( json.dumps({ 'error': 'Failed to send notification email - please ' 'contact the %s administrator (%s), ' 'or try again later. Your draft is saved.' % (get_site_admin_email(), getattr(settings, 'SITE_TITLE', 'MyTardis')) }), content_type="application/json") # Remove the draft status remove_draft_status(publication) # Automatically approve publications if approval is not required if not getattr(settings, 'PUBLICATIONS_REQUIRE_APPROVAL', default_settings.PUBLICATIONS_REQUIRE_APPROVAL): approve_publication(request, publication, message=None, send_email=False) # approve_publication will delete the form state, so don't # bother saving is and return. form_state['action'] = '' return HttpResponse(json.dumps(form_state), content_type="appication/json") # Trigger publication record update tasks.update_publication_records.delay() # Clear the form action and save the state form_state['action'] = '' form_state_parameter.string_value = json.dumps(form_state) form_state_parameter.save() return HttpResponse(json.dumps(form_state), content_type="appication/json")
def populate_pdb_pub_records(): PUB_SCHEMA = getattr(settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT) PUB_SCHEMA_DRAFT = getattr(settings, 'PUBLICATION_DRAFT_SCHEMA', default_settings.PUBLICATION_DRAFT_SCHEMA) PDB_SCHEMA = getattr(settings, 'PDB_PUBLICATION_SCHEMA_ROOT', default_settings.PDB_PUBLICATION_SCHEMA_ROOT) publications = Experiment.objects \ .filter(experimentparameterset__schema__namespace=PDB_SCHEMA) \ .filter(experimentparameterset__schema__namespace=PUB_SCHEMA) \ .exclude(experimentparameterset__schema__namespace=PUB_SCHEMA_DRAFT) \ .distinct() last_update_parameter_name = ParameterName.objects.get( name='pdb-last-sync', schema__namespace=PUB_SCHEMA) def add_if_missing(parameterset, name, string_value=None, numerical_value=None, datetime_value=None): try: ExperimentParameter.objects.get( name__name=name, parameterset=parameterset) except ExperimentParameter.DoesNotExist: param_name = ParameterName.objects.get( name=name, schema=parameterset.schema) param = ExperimentParameter(name=param_name, parameterset=parameterset) param.string_value = string_value param.numerical_value = numerical_value param.datetime_value = datetime_value param.save() for pub in publications: try: # try to get the last update time for the PDB data pdb_last_update_parameter = ExperimentParameter.objects.get( parameterset__schema__namespace=PUB_SCHEMA, name=last_update_parameter_name, parameterset__experiment=pub ) last_update = pdb_last_update_parameter.datetime_value needs_update = last_update + \ getattr(settings, 'PDB_REFRESH_INTERVAL', default_settings.PDB_REFRESH_INTERVAL) \ < timezone.now() except ExperimentParameter.DoesNotExist: # if the PDB last update time parameter doesn't exist, # we definitely need to update the data and create a last # update entry needs_update = True pdb_last_update_parameter = None # If an update needs to happen... if needs_update: # 1. get the PDB info pdb_parameter_set = ExperimentParameterSet.objects.get( schema__namespace=getattr( settings, 'PDB_PUBLICATION_SCHEMA_ROOT', default_settings.PDB_PUBLICATION_SCHEMA_ROOT), experiment=pub) pdb = ExperimentParameter.objects.get( name__name='pdb-id', parameterset=pdb_parameter_set) pdb_id = pdb.string_value # 1a. cosmetic change of case for PDB ID, if entered incorrectly if pdb_id != pdb_id.upper(): pdb.string_value = pdb_id.upper() pdb.save() try: # 2. fetch the info from pdb.org pdb = PDBCifHelper(pdb_id) # 3. insert all standard pdb parameters add_if_missing(pdb_parameter_set, 'title', string_value=pdb.get_pdb_title()) add_if_missing(pdb_parameter_set, 'url', string_value=pdb.get_pdb_url()) try: add_if_missing(pdb_parameter_set, 'resolution', numerical_value=pdb.get_resolution()) except ValueError: logger.error( 'PDB field "resolution" could not be set for ' 'publication Id %i \n %s' % (pub.id, traceback.format_exc())) try: add_if_missing(pdb_parameter_set, 'r-value', numerical_value=pdb.get_obs_r_value()) except ValueError: logger.error( 'PDB field "r-value" could not be set for ' 'publication Id %i \n %s' % (pub.id, traceback.format_exc())) try: add_if_missing(pdb_parameter_set, 'r-free', numerical_value=pdb.get_free_r_value()) except ValueError: logger.error( 'PDB field "r-free" could not be set for ' 'publication Id %i \n %s' % (pub.id, traceback.format_exc())) add_if_missing(pdb_parameter_set, 'space-group', string_value=pdb.get_spacegroup()) add_if_missing(pdb_parameter_set, 'unit-cell', string_value=pdb.get_unit_cell()) # 4. insert sequence info (lazy checking) pdb_seq_parameter_sets = ExperimentParameterSet.objects.filter( schema__namespace=getattr( settings, 'PDB_SEQUENCE_PUBLICATION_SCHEMA', default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA), experiment=pub) if pdb_seq_parameter_sets.count() == 0: # insert seqences for seq in pdb.get_sequence_info(): seq_ps_namespace = getattr( settings, 'PDB_SEQUENCE_PUBLICATION_SCHEMA', default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA) seq_parameter_set = ExperimentParameterSet( schema=Schema.objects.get( namespace=seq_ps_namespace), experiment=pub) seq_parameter_set.save() add_if_missing(seq_parameter_set, 'organism', string_value=seq['organism']) add_if_missing(seq_parameter_set, 'expression-system', string_value=seq['expression_system']) add_if_missing(seq_parameter_set, 'sequence', string_value=seq['sequence']) # 5. insert/update citation info (aggressive) ExperimentParameterSet.objects.filter( schema__namespace=getattr( settings, 'PDB_CITATION_PUBLICATION_SCHEMA', default_settings.PDB_CITATION_PUBLICATION_SCHEMA), experiment=pub).delete() for citation in pdb.get_citations(): cit_ps_namespace = getattr( settings, 'PDB_CITATION_PUBLICATION_SCHEMA', default_settings.PDB_CITATION_PUBLICATION_SCHEMA) cit_parameter_set = ExperimentParameterSet( schema=Schema.objects.get(namespace=cit_ps_namespace), experiment=pub) cit_parameter_set.save() add_if_missing(cit_parameter_set, 'title', string_value=citation['title']) add_if_missing(cit_parameter_set, 'authors', string_value='; '.join(citation['authors'])) add_if_missing(cit_parameter_set, 'journal', string_value=citation['journal']) add_if_missing(cit_parameter_set, 'volume', string_value=citation['volume']) add_if_missing(cit_parameter_set, 'page-range', string_value='-'.join( [citation['page_first'], citation['page_last']])) add_if_missing(cit_parameter_set, 'doi', string_value='http://dx.doi.org/' + citation['doi']) # 6. Remove the PDB embargo if set, since the update has # occurred and therefore the PDB must have been relased. try: ExperimentParameter.objects.get( name__name='pdb-embargo', parameterset__schema__namespace=getattr( settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT)).delete() except ExperimentParameter.DoesNotExist: pass # 7. Set the last update parameter to be now if pdb_last_update_parameter is None: pub_parameter_set = ExperimentParameterSet( schema=Schema.objects.get(namespace=PUB_SCHEMA), experiment=pub) pub_parameter_set.save() pdb_last_update_parameter = ExperimentParameter( name=last_update_parameter_name, parameterset=pub_parameter_set, datetime_value=timezone.now()) else: pdb_last_update_parameter.datetime_value = timezone.now() pdb_last_update_parameter.save() except CifFile.StarError: # PDB is either unavailable or invalid # (maybe notify the user somehow?) continue
class ParameterSetManager(object): parameterset = None parameters = None # queryset of parameters blank_param = None # parameterset OR schema / datafile / dataset / experiment # delete dataset creation code # make parameterset / object arguments generic and test type # create function to return generic parameter type for setting/getting def __init__(self, parameterset=None, parentObject=None, schema=None): """ instantiate new task or existing ParameterSet :param dataset: optional parameter to instanciate task from metadata, will be tested for completeness and copied into new task if complete :type dataset: Dataset """ if parameterset: self.parameterset = parameterset self.schema = self.parameterset.schema self.namespace = self.schema.namespace if type(self.parameterset).__name__ == "DatafileParameterSet": self.parameters = DatafileParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatafileParameter elif type(self.parameterset).__name__ == "DatasetParameterSet": self.parameters = DatasetParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatasetParameter elif type(self.parameterset).__name__ == "ExperimentParameterSet": self.parameters = ExperimentParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = ExperimentParameter else: raise TypeError("Invalid parameterset object given.") elif parentObject and schema: self.namespace = schema if type(parentObject).__name__ == "Dataset_File": self.parameterset = DatafileParameterSet( schema=self.get_schema(), dataset_file=parentObject) self.parameterset.save() self.parameters = DatafileParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatafileParameter elif type(parentObject).__name__ == "Dataset": self.parameterset = DatasetParameterSet( schema=self.get_schema(), dataset=parentObject) self.parameterset.save() self.parameters = DatasetParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatasetParameter elif type(parentObject).__name__ == "Experiment": self.parameterset = ExperimentParameterSet( schema=self.get_schema(), experiment=parentObject) self.parameterset.save() self.parameters = ExperimentParameter.objects.filter( parameterset=self.parameterset) self.blank_param = ExperimentParameter else: raise TypeError("Invalid parent object." + "Must be an experiment/dataset/datafile") else: raise TypeError("Missing arguments") def get_schema(self): try: schema = Schema.objects.get( namespace=self.namespace) except ObjectDoesNotExist: schema = Schema() schema.namespace = self.namespace schema.save() self.schema = schema return schema def get_param(self, parname, value=False): par = self.parameters.get(name__name=parname) if value: if par.name.isNumeric(): return par.numerical_value else: return par.string_value return par def get_params(self, parname, value=False): pars = self.parameters.filter(name__name=parname) if value: if len(pars) > 0 and pars[0].name.isNumeric(): return [par.numerical_value for par in pars] else: return [par.string_value for par in pars] return pars def set_param(self, parname, value, fullparname=None, example_value=None): try: param = self.get_param(parname) except ObjectDoesNotExist: param = self.blank_param() param.parameterset = self.parameterset param.name = self._get_create_parname(parname, fullparname, example_value=example_value) #param.string_value = value #param.save() if param.name.isNumeric(): param.numerical_value = float(value) else: param.string_value = str(value) param.save() def new_param(self, parname, value, fullparname=None): param = self.blank_param() param.parameterset = self.parameterset param.name = self._get_create_parname(parname, fullparname) param.string_value = value param.save() if param.name.isNumeric(): param.numerical_value = float(value) else: param.string_value = str(value) param.save() # use this one from post data def set_param_list(self, parname, value_list, fullparname=None): self.delete_params(parname) for value in value_list: if value != None: self.new_param(parname, value, fullparname) def set_params_from_dict(self, dict): print type(dict) for (key, value) in dict.iteritems(): if type(value) is list: self.set_param_list(key, value) else: if value != None: self.delete_params(key) self.set_param(key, value) def delete_params(self, parname): params = self.get_params(parname) for param in params: param.delete() def delete_all_params(self): for param in self.parameters: param.delete() def _get_create_parname(self, parname, fullparname=None, example_value=None): try: paramName = ParameterName.objects.get(name=parname, schema__id=self.get_schema().id) except ObjectDoesNotExist: paramName = ParameterName() paramName.schema = self.get_schema() paramName.name = parname if fullparname: paramName.full_name = fullparname else: paramName.full_name = parname if example_value: try: float(example_value) paramName.data_type = ParameterName.NUMERIC except (TypeError, ValueError): paramName.data_type = ParameterName.STRING else: paramName.data_type = ParameterName.STRING paramName.is_searchable = True paramName.save() return paramName
def delete_info(self, parameterset_id): eps = ExperimentParameterSet(experiment_id=self.experiment_id, schema=self.schema, pk=parameterset_id) eps.delete()
def process_form(request): # Decode the form data form_state = json.loads(request.body) def validation_error(error=None): if error is None: error = 'Invalid form data was submitted ' \ '(server-side validation failed)' return HttpResponse(json.dumps({'error': error}), content_type="application/json") # Check if the form data contains a publication ID # If it does, then this publication needs to be updated # rather than created. if 'publicationId' not in form_state: if not form_state['publicationTitle'].strip(): return validation_error() publication = create_draft_publication( request.user, form_state['publicationTitle'], form_state['publicationDescription']) form_state['publicationId'] = publication.id else: publication = get_draft_publication(request.user, form_state['publicationId']) # Check if the publication is finalised (i.e. not in draft) # if it is, then refuse to process the form. if publication is None or not publication.is_publication_draft(): return HttpResponseForbidden() # Get the form state database object form_state_parameter = ExperimentParameter.objects.get( name__name='form_state', name__schema__namespace=getattr( settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT), parameterset__experiment=publication) # Check if the form state needs to be loaded (i.e. a publication draft # is resumed) # no database changes are made if the form is resumed if form_state['action'] == 'resume': form_state = json.loads(form_state_parameter.string_value) return HttpResponse(json.dumps(form_state), content_type="application/json") if form_state['action'] == 'update-dataset-selection': # Update the publication title/description if changed. # Must not be blank. if not form_state['publicationTitle'].strip() or \ not form_state['publicationDescription'].strip(): return validation_error() if publication.title != form_state['publicationTitle']: publication.title = form_state['publicationTitle'] publication.save() if publication.description != form_state['publicationDescription']: publication.description = form_state['publicationDescription'] publication.save() # Update associated datasets # (note: might not be efficient re: db queries) # ... first clear all current associations current_datasets = Dataset.objects.filter(experiments=publication) for current_dataset in current_datasets: current_dataset.experiments.remove(publication) # ... now (re)add all datasets selected_datasets = [ ds['dataset']['id'] for ds in form_state['addedDatasets'] ] datasets = Dataset.objects.filter( experiments__in=Experiment.safe.owned_and_shared(request.user), pk__in=selected_datasets).distinct() for dataset in datasets: dataset.experiments.add(publication) # --- Get data for the next page --- # # Construct the disclipline-specific form based on the # selected datasets selected_forms = select_forms(datasets) if 'disciplineSpecificFormTemplates' in form_state: # clear extraInfo if the selected forms differ # (i.e. datasets have changed) if json.dumps(selected_forms) != json.dumps( form_state['disciplineSpecificFormTemplates']): form_state['extraInfo'] = {} form_state['disciplineSpecificFormTemplates'] = selected_forms elif form_state['action'] == 'update-extra-info': # Clear any current parameter sets except for those belonging # to the publication draft schema or containing the form_state # parameter clear_publication_metadata(publication) # Loop through form data and create associates parameter sets # Any unrecognised fields or schemas are ignored! map_form_to_schemas(form_state['extraInfo'], publication) # *** Synchrotron specific *** # Search for beamline/EPN information associated with each dataset # and add to the publication. synchrotron_search_epn(publication) # --- Get data for the next page --- # licenses_json = get_licenses() form_state['licenses'] = licenses_json # Select the first license as default if licenses_json: if 'selectedLicenseId' not in form_state: form_state['selectedLicenseId'] = licenses_json[0]['id'] else: # No licenses configured... form_state['selectedLicenseId'] = -1 # Set a default author (current user) if no previously saved data # By default, the form sends a list of authors of one element # with blank fields if len(form_state['authors']) == 1 and \ not form_state['authors'][0]['name']: form_state['authors'] = [{ 'name': ' '.join([request.user.first_name, request.user.last_name]), 'institution': getattr(settings, 'DEFAULT_INSTITUTION', ''), 'email': request.user.email }] elif form_state['action'] == 'submit': # any final form validation should occur here # and specific error messages can be returned # to the browser before the publication's draft # status is removed. if 'acknowledge' not in form_state or not form_state['acknowledge']: return validation_error('You must confirm that you are ' 'authorised to submit this publication') set_publication_authors(form_state['authors'], publication) institutions = '; '.join( set([author['institution'] for author in form_state['authors']])) publication.institution_name = institutions # Attach the publication details schema pub_details_schema = Schema.objects.get( namespace=getattr(settings, 'PUBLICATION_DETAILS_SCHEMA', default_settings.PUBLICATION_DETAILS_SCHEMA)) pub_details_parameter_set = ExperimentParameterSet( schema=pub_details_schema, experiment=publication) pub_details_parameter_set.save() # Add the acknowledgements acknowledgements_parameter_name = ParameterName.objects.get( schema=pub_details_schema, name='acknowledgements') ExperimentParameter( name=acknowledgements_parameter_name, parameterset=pub_details_parameter_set, string_value=form_state['acknowledgements']).save() # Set the release date set_embargo_release_date(publication, dateutil.parser.parse(form_state['embargo'])) # Set the license try: publication.license = License.objects.get( pk=form_state['selectedLicenseId'], is_active=True, allows_distribution=True) except License.DoesNotExist: publication.license = License.get_none_option_license() publication.save() # Send emails about publication in draft subject, message_content = email_pub_requires_authorisation( request.user.username, request.build_absolute_uri( reverse('tardis_portal.view_experiment', args=(publication.id, ))), request.build_absolute_uri('/apps/publication-forms/approvals/')) try: send_mail( subject, message_content, getattr( settings, 'PUBLICATION_NOTIFICATION_SENDER_EMAIL', default_settings.PUBLICATION_NOTIFICATION_SENDER_EMAIL), get_pub_admin_email_addresses(), fail_silently=False) subject, message_content = email_pub_awaiting_approval( publication.title) send_mail_to_authors(publication, subject, message_content, fail_silently=False) except Exception as e: logger.error( "failed to send publication notification email(s): %s" % repr(e)) return HttpResponse(json.dumps({ 'error': 'Failed to send notification email - please ' 'contact the %s administrator (%s), ' 'or try again later. Your draft is saved.' % (get_site_admin_email(), getattr(settings, 'SITE_TITLE', 'MyTardis')) }), content_type="application/json") # Remove the draft status remove_draft_status(publication) # Automatically approve publications if approval is not required if not getattr(settings, 'PUBLICATIONS_REQUIRE_APPROVAL', default_settings.PUBLICATIONS_REQUIRE_APPROVAL): approve_publication(request, publication, message=None, send_email=False) # approve_publication will delete the form state, so don't # bother saving is and return. form_state['action'] = '' return HttpResponse(json.dumps(form_state), content_type="appication/json") # Trigger publication record update tasks.update_publication_records.delay() # Clear the form action and save the state form_state['action'] = '' form_state_parameter.string_value = json.dumps(form_state) form_state_parameter.save() return HttpResponse(json.dumps(form_state), content_type="appication/json")
class EmbargoHandler(object): def __init__(self, experiment_id, create=False): self.experiment = Experiment.objects.get(pk=experiment_id) parametersets = ExperimentParameterSet.objects.filter( schema__namespace=NAMESPACE, experiment__id=experiment_id) self.schema, _ = Schema.objects.get_or_create(namespace=NAMESPACE, name='Embargo Details') self.expiry_date, _ = ParameterName.objects.get_or_create(schema=self.schema, name=EXPIRY_DATE_KEY, full_name='Expiry', immutable=True, data_type=ParameterName.DATETIME) self.never_expire, _ = ParameterName.objects.get_or_create(schema=self.schema, name=NEVER_EXPIRE_KEY, full_name='Never Expires', immutable=True, data_type=ParameterName.STRING) if len(parametersets) == 1: self.parameterset = parametersets[0] elif create: self.parameterset = ExperimentParameterSet(experiment=self.experiment, schema=self.schema) self.parameterset.save() else: self.parameterset = None def never_expires(self): never_expire = self._get_or_none(NEVER_EXPIRE_KEY) if never_expire: return True expiry_date = self._get_or_none(EXPIRY_DATE_KEY) if expiry_date: return False else: return self.experiment.end_time == None def has_any_expiry(self): return not self.never_expires() def can_be_defaulted(self): expiry_date = self._get_or_none(EXPIRY_DATE_KEY) never_expires = self._get_or_none(NEVER_EXPIRE_KEY) return never_expires or expiry_date def because_no_end_date(self): if self._get_or_none(NEVER_EXPIRE_KEY): return False return self.experiment.end_time == None def get_expiry_date(self): ''' returns calculated or explicit expiry or None ''' import datetime if self.never_expires(): return None explicit_expiry = self._get_or_none(EXPIRY_DATE_KEY) if explicit_expiry: return explicit_expiry.datetime_value else: return self.experiment.end_time + datetime.timedelta(settings.EMBARGO_DAYS) def _get_or_none(self, name): if not self.parameterset: return None params = self.parameterset.experimentparameter_set.filter(name__name=name) if params.count() == 0: return None else: return params[0] def prevent_expiry(self): # delete any current expiry dates # set never_expires = True logger.fatal('preventing expiry') if not self.parameterset: raise Exception('incorrectly initialised, call with create=True') params = self.parameterset.experimentparameter_set params.all().delete() param = ExperimentParameter(name=self.never_expire, string_value='True', parameterset=self.parameterset) param.save() self.experiment.public = False self.experiment.save() def reset_to_default(self): import datetime if self.parameterset: self.parameterset.delete() expiry_date = self.get_expiry_date() if expiry_date and expiry_date.date() < datetime.date.today(): self.experiment.public = True else: self.experiment.public = False self.experiment.save() else: logger.warn('tried to delete parameterset that does not exist') def set_expiry(self, date_string): if not self.parameterset: raise Exception('incorrectly initialised, call with create=True') params = self.parameterset.experimentparameter_set params.all().delete() import datetime expiry_date = datetime.datetime.strptime(date_string, '%Y/%m/%d') param = ExperimentParameter(name=self.expiry_date, datetime_value=expiry_date, parameterset=self.parameterset) param.save() if expiry_date.date() < datetime.date.today(): self.experiment.public = True else: self.experiment.public = False self.experiment.save()
class ParameterSetManager(object): parameterset = None parameters = None # queryset of parameters blank_param = None # parameterset OR schema / datafile / dataset / experiment # delete dataset creation code # make parameterset / object arguments generic and test type # create function to return generic parameter type for setting/getting # # 2011/07/20 (Ryan Braganza) # changed type checking from: # type(self.parameterset).__name__ == 'ClassName' # to: # isinstance(self.parameterset, Class) # This is to solve an issue with models using deferred fields. # When using deferred fields, a subclass is used, so the class name # will not match when using type(self.parameterset).__name__ # isinstance deals with inheritance appropriately here def __init__(self, parameterset=None, parentObject=None, schema=None): """ instantiate new task or existing ParameterSet :param dataset: optional parameter to instanciate task from metadata, will be tested for completeness and copied into new task if complete :type dataset: Dataset :param schema: Schema namespace :type schema: string """ if parameterset: self.parameterset = parameterset self.schema = self.parameterset.schema self.namespace = self.schema.namespace if isinstance(self.parameterset, DatafileParameterSet): self.parameters = DatafileParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatafileParameter elif isinstance(self.parameterset, DatasetParameterSet): self.parameters = DatasetParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatasetParameter elif isinstance(self.parameterset, ExperimentParameterSet): self.parameters = ExperimentParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = ExperimentParameter else: raise TypeError("Invalid parameterset object given.") elif parentObject and schema: self.namespace = schema if isinstance(parentObject, Dataset_File): self.parameterset = DatafileParameterSet( schema=self.get_schema(), dataset_file=parentObject) self.parameterset.save() self.parameters = DatafileParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatafileParameter elif isinstance(parentObject, Dataset): self.parameterset = DatasetParameterSet( schema=self.get_schema(), dataset=parentObject) self.parameterset.save() self.parameters = DatasetParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatasetParameter elif isinstance(parentObject, Experiment): self.parameterset = ExperimentParameterSet( schema=self.get_schema(), experiment=parentObject) self.parameterset.save() self.parameters = ExperimentParameter.objects.filter( parameterset=self.parameterset) self.blank_param = ExperimentParameter else: raise TypeError("Invalid parent object." + "Must be an experiment/dataset/datafile not " + str(type(parentObject))) else: raise TypeError("Missing arguments") def get_schema(self): try: schema = Schema.objects.get( namespace=self.namespace) except ObjectDoesNotExist: schema = Schema() schema.namespace = self.namespace schema.save() self.schema = schema return schema def get_param(self, parname, value=False): par = self.parameters.get(name__name=parname) if value: if par.name.isNumeric(): return par.numerical_value else: return par.string_value return par def get_params(self, parname, value=False): pars = self.parameters.filter(name__name=parname) if value: if len(pars) > 0 and pars[0].name.isNumeric(): return [par.numerical_value for par in pars] else: return [par.string_value for par in pars] return pars def set_param(self, parname, value, fullparname=None, example_value=None): try: param = self.get_param(parname) except ObjectDoesNotExist: param = self.blank_param() param.parameterset = self.parameterset param.name = self._get_create_parname(parname, fullparname, example_value=example_value) #param.string_value = value #param.save() if param.name.isNumeric(): param.numerical_value = float(value) else: param.string_value = unicode(value) param.save() return param.id def new_param(self, parname, value, fullparname=None): param = self.blank_param() param.parameterset = self.parameterset param.name = self._get_create_parname(parname, fullparname) param.string_value = value param.save() if param.name.isNumeric(): param.numerical_value = float(value) else: param.string_value = unicode(value) param.save() return param.id # use this one from post data def set_param_list(self, parname, value_list, fullparname=None): self.delete_params(parname) for value in value_list: if value != None: self.new_param(parname, value, fullparname) def set_params_from_dict(self, dict): for (key, value) in dict.iteritems(): if type(value) is list: self.set_param_list(key, value) else: if value != None: self.delete_params(key) self.set_param(key, value) def delete_params(self, parname): params = self.get_params(parname) for param in params: param.delete() def delete_all_params(self): for param in self.parameters: param.delete() def _get_create_parname(self, parname, fullparname=None, example_value=None): try: paramName = ParameterName.objects.get(name=parname, schema__id=self.get_schema().id) except ObjectDoesNotExist: paramName = ParameterName() paramName.schema = self.get_schema() paramName.name = parname if fullparname: paramName.full_name = fullparname else: paramName.full_name = parname if example_value: try: float(example_value) paramName.data_type = ParameterName.NUMERIC except (TypeError, ValueError): paramName.data_type = ParameterName.STRING else: paramName.data_type = ParameterName.STRING paramName.is_searchable = True paramName.save() return paramName
def test_parameter(self): exp = Experiment( title='test exp1', institution_name='Australian Synchrotron', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE, ) exp.save() dataset = Dataset(description="dataset description") dataset.save() dataset.experiments.add(exp) dataset.save() df_file = DataFile(dataset=dataset, filename='file.txt', size=42, md5sum='bogus') df_file.save() df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml', type=Schema.DATAFILE) df_schema.save() ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml', type=Schema.DATASET) ds_schema.save() exp_schema = Schema( namespace='http://www.cern.ch/felzmann/schema3.xml', type=Schema.EXPERIMENT) exp_schema.save() df_parname = ParameterName(schema=df_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) df_parname.save() ds_parname = ParameterName(schema=ds_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) ds_parname.save() exp_parname = ParameterName(schema=exp_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) exp_parname.save() df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file) df_parset.save() ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset) ds_parset.save() exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp) exp_parset.save() with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)): filename = 'test.jpg' df_parameter = DatafileParameter(name=df_parname, parameterset=df_parset, string_value=filename) df_parameter.save() ds_parameter = DatasetParameter(name=ds_parname, parameterset=ds_parset, string_value=filename) ds_parameter.save() exp_parameter = ExperimentParameter(name=exp_parname, parameterset=exp_parset, string_value=filename) exp_parameter.save() self.assertEqual( "<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>" % # noqa (df_parameter.id, df_parameter.id), df_parameter.get()) self.assertEqual( "<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>" % # noqa (ds_parameter.id, ds_parameter.id), ds_parameter.get()) self.assertEqual( "<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>" % # noqa (exp_parameter.id, exp_parameter.id), exp_parameter.get())
def populate_pdb_pub_records(): PUB_SCHEMA = getattr(settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT) PUB_SCHEMA_DRAFT = getattr(settings, 'PUBLICATION_DRAFT_SCHEMA', default_settings.PUBLICATION_DRAFT_SCHEMA) PDB_SCHEMA = getattr(settings, 'PDB_PUBLICATION_SCHEMA_ROOT', default_settings.PDB_PUBLICATION_SCHEMA_ROOT) publications = Experiment.objects \ .filter(experimentparameterset__schema__namespace=PDB_SCHEMA) \ .filter(experimentparameterset__schema__namespace=PUB_SCHEMA) \ .exclude(experimentparameterset__schema__namespace=PUB_SCHEMA_DRAFT) \ .distinct() last_update_parameter_name = ParameterName.objects.get( name='pdb-last-sync', schema__namespace=PUB_SCHEMA) def add_if_missing(parameterset, name, string_value=None, numerical_value=None, datetime_value=None): try: ExperimentParameter.objects.get(name__name=name, parameterset=parameterset) except ExperimentParameter.DoesNotExist: param_name = ParameterName.objects.get(name=name, schema=parameterset.schema) param = ExperimentParameter(name=param_name, parameterset=parameterset) param.string_value = string_value param.numerical_value = numerical_value param.datetime_value = datetime_value param.save() for pub in publications: try: # try to get the last update time for the PDB data pdb_last_update_parameter = ExperimentParameter.objects.get( parameterset__schema__namespace=PUB_SCHEMA, name=last_update_parameter_name, parameterset__experiment=pub) last_update = pdb_last_update_parameter.datetime_value needs_update = last_update + \ getattr(settings, 'PDB_REFRESH_INTERVAL', default_settings.PDB_REFRESH_INTERVAL) \ < timezone.now() except ExperimentParameter.DoesNotExist: # if the PDB last update time parameter doesn't exist, # we definitely need to update the data and create a last # update entry needs_update = True pdb_last_update_parameter = None # If an update needs to happen... if needs_update: # 1. get the PDB info pdb_parameter_set = ExperimentParameterSet.objects.get( schema__namespace=getattr( settings, 'PDB_PUBLICATION_SCHEMA_ROOT', default_settings.PDB_PUBLICATION_SCHEMA_ROOT), experiment=pub) pdb = ExperimentParameter.objects.get( name__name='pdb-id', parameterset=pdb_parameter_set) pdb_id = pdb.string_value # 1a. cosmetic change of case for PDB ID, if entered incorrectly if pdb_id != pdb_id.upper(): pdb.string_value = pdb_id.upper() pdb.save() try: # 2. fetch the info from pdb.org pdb = PDBCifHelper(pdb_id) # 3. insert all standard pdb parameters add_if_missing(pdb_parameter_set, 'title', string_value=pdb.get_pdb_title()) add_if_missing(pdb_parameter_set, 'url', string_value=pdb.get_pdb_url()) try: add_if_missing(pdb_parameter_set, 'resolution', numerical_value=pdb.get_resolution()) except ValueError: logger.error('PDB field "resolution" could not be set for ' 'publication Id %i \n %s' % (pub.id, traceback.format_exc())) try: add_if_missing(pdb_parameter_set, 'r-value', numerical_value=pdb.get_obs_r_value()) except ValueError: logger.error('PDB field "r-value" could not be set for ' 'publication Id %i \n %s' % (pub.id, traceback.format_exc())) try: add_if_missing(pdb_parameter_set, 'r-free', numerical_value=pdb.get_free_r_value()) except ValueError: logger.error('PDB field "r-free" could not be set for ' 'publication Id %i \n %s' % (pub.id, traceback.format_exc())) add_if_missing(pdb_parameter_set, 'space-group', string_value=pdb.get_spacegroup()) add_if_missing(pdb_parameter_set, 'unit-cell', string_value=pdb.get_unit_cell()) # 4. insert sequence info (lazy checking) pdb_seq_parameter_sets = ExperimentParameterSet.objects.filter( schema__namespace=getattr( settings, 'PDB_SEQUENCE_PUBLICATION_SCHEMA', default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA), experiment=pub) if pdb_seq_parameter_sets.count() == 0: # insert seqences for seq in pdb.get_sequence_info(): seq_ps_namespace = getattr( settings, 'PDB_SEQUENCE_PUBLICATION_SCHEMA', default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA) seq_parameter_set = ExperimentParameterSet( schema=Schema.objects.get( namespace=seq_ps_namespace), experiment=pub) seq_parameter_set.save() add_if_missing(seq_parameter_set, 'organism', string_value=seq['organism']) add_if_missing(seq_parameter_set, 'expression-system', string_value=seq['expression_system']) add_if_missing(seq_parameter_set, 'sequence', string_value=seq['sequence']) # 5. insert/update citation info (aggressive) ExperimentParameterSet.objects.filter( schema__namespace=getattr( settings, 'PDB_CITATION_PUBLICATION_SCHEMA', default_settings.PDB_CITATION_PUBLICATION_SCHEMA), experiment=pub).delete() for citation in pdb.get_citations(): cit_ps_namespace = getattr( settings, 'PDB_CITATION_PUBLICATION_SCHEMA', default_settings.PDB_CITATION_PUBLICATION_SCHEMA) cit_parameter_set = ExperimentParameterSet( schema=Schema.objects.get(namespace=cit_ps_namespace), experiment=pub) cit_parameter_set.save() add_if_missing(cit_parameter_set, 'title', string_value=citation['title']) add_if_missing(cit_parameter_set, 'authors', string_value='; '.join(citation['authors'])) add_if_missing(cit_parameter_set, 'journal', string_value=citation['journal']) add_if_missing(cit_parameter_set, 'volume', string_value=citation['volume']) add_if_missing(cit_parameter_set, 'page-range', string_value='-'.join([ citation['page_first'], citation['page_last'] ])) add_if_missing(cit_parameter_set, 'doi', string_value='http://dx.doi.org/' + citation['doi']) # 6. Remove the PDB embargo if set, since the update has # occurred and therefore the PDB must have been relased. try: ExperimentParameter.objects.get( name__name='pdb-embargo', parameterset__schema__namespace=getattr( settings, 'PUBLICATION_SCHEMA_ROOT', default_settings.PUBLICATION_SCHEMA_ROOT)).delete( ) except ExperimentParameter.DoesNotExist: pass # 7. Set the last update parameter to be now if pdb_last_update_parameter is None: pub_parameter_set = ExperimentParameterSet( schema=Schema.objects.get(namespace=PUB_SCHEMA), experiment=pub) pub_parameter_set.save() pdb_last_update_parameter = ExperimentParameter( name=last_update_parameter_name, parameterset=pub_parameter_set, datetime_value=timezone.now()) else: pdb_last_update_parameter.datetime_value = timezone.now() pdb_last_update_parameter.save() except CifFile.StarError: # PDB is either unavailable or invalid # (maybe notify the user somehow?) continue
def __init__(self, parameterset=None, parentObject=None, schema=None): """ instantiate new task or existing ParameterSet :param dataset: optional parameter to instanciate task from metadata, will be tested for completeness and copied into new task if complete :type dataset: Dataset :param schema: Schema namespace :type schema: string """ if parameterset: self.parameterset = parameterset self.schema = self.parameterset.schema self.namespace = self.schema.namespace if isinstance(self.parameterset, DatafileParameterSet): self.parameters = DatafileParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatafileParameter elif isinstance(self.parameterset, DatasetParameterSet): self.parameters = DatasetParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatasetParameter elif isinstance(self.parameterset, ExperimentParameterSet): self.parameters = ExperimentParameter.objects.filter(\ parameterset=self.parameterset).order_by('name__full_name') self.blank_param = ExperimentParameter else: raise TypeError("Invalid parameterset object given.") elif parentObject and schema: self.namespace = schema if isinstance(parentObject, Dataset_File): self.parameterset = DatafileParameterSet( schema=self.get_schema(), dataset_file=parentObject) self.parameterset.save() self.parameters = DatafileParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatafileParameter elif isinstance(parentObject, Dataset): self.parameterset = DatasetParameterSet( schema=self.get_schema(), dataset=parentObject) self.parameterset.save() self.parameters = DatasetParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatasetParameter elif isinstance(parentObject, Experiment): self.parameterset = ExperimentParameterSet( schema=self.get_schema(), experiment=parentObject) self.parameterset.save() self.parameters = ExperimentParameter.objects.filter( parameterset=self.parameterset) self.blank_param = ExperimentParameter else: raise TypeError("Invalid parent object." + "Must be an experiment/dataset/datafile not " + str(type(parentObject))) else: raise TypeError("Missing arguments")
def __init__(self, parameterset=None, parentObject=None, schema=None): """ instantiate new task or existing ParameterSet :param ParameterSet parameterset: optional parameter to instanciate task from metadata, will be tested for completeness and copied into new task if complete :param object parentObject: :param string schema: Schema namespace :raises TypeError: """ from tardis.tardis_portal.models import DatasetParameterSet from tardis.tardis_portal.models import DatafileParameterSet from tardis.tardis_portal.models import ExperimentParameterSet from tardis.tardis_portal.models import DatasetParameter from tardis.tardis_portal.models import DatafileParameter from tardis.tardis_portal.models import ExperimentParameter from tardis.tardis_portal.models.parameters import ParameterSet if issubclass(type(self), ParameterSet): pass elif parameterset: self.parameterset = parameterset self.schema = self.parameterset.schema self.namespace = self.schema.namespace if isinstance(self.parameterset, DatafileParameterSet): self.parameters = DatafileParameter.objects.filter( parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatafileParameter elif isinstance(self.parameterset, DatasetParameterSet): self.parameters = DatasetParameter.objects.filter( parameterset=self.parameterset).order_by('name__full_name') self.blank_param = DatasetParameter elif isinstance(self.parameterset, ExperimentParameterSet): self.parameters = ExperimentParameter.objects.filter( parameterset=self.parameterset).order_by('name__full_name') self.blank_param = ExperimentParameter else: raise TypeError("Invalid parameterset object given.") elif parentObject and schema: self.namespace = schema if isinstance(parentObject, DataFile): self.parameterset = DatafileParameterSet( schema=self.get_schema(), datafile=parentObject) self.parameterset.save() self.parameters = DatafileParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatafileParameter elif isinstance(parentObject, Dataset): self.parameterset = DatasetParameterSet( schema=self.get_schema(), dataset=parentObject) self.parameterset.save() self.parameters = DatasetParameter.objects.filter( parameterset=self.parameterset) self.blank_param = DatasetParameter elif isinstance(parentObject, Experiment): self.parameterset = ExperimentParameterSet( schema=self.get_schema(), experiment=parentObject) self.parameterset.save() self.parameters = ExperimentParameter.objects.filter( parameterset=self.parameterset) self.blank_param = ExperimentParameter else: raise TypeError("Invalid parent object." + "Must be an experiment/dataset/datafile not " + str(type(parentObject))) else: raise TypeError("Missing arguments")