def map_form_to_schemas(extraInfo, publication):
for form_id, form in extraInfo.iteritems():
try: # Ignore form if no schema exists with this name
schema = Schema.objects.get(namespace=form['schema'])
except Schema.DoesNotExist:
continue
parameter_set = ExperimentParameterSet(
schema=schema, experiment=publication)
parameter_set.save()
for key, value in form.iteritems():
if key != 'schema':
try: # Ignore field if parameter name (key) doesn't match
parameter_name = ParameterName.objects.get(
schema=schema, name=key)
if parameter_name.isNumeric():
parameter = ExperimentParameter(
name=parameter_name,
parameterset=parameter_set,
numerical_value=float(value))
elif parameter_name.isLongString() or \
parameter_name.isString() or \
parameter_name.isURL() or \
parameter_name.isLink() or \
parameter_name.isFilename():
parameter = ExperimentParameter(
name=parameter_name,
parameterset=parameter_set,
string_value=str(value))
else:
# Shouldn't happen, but here in case the parameter type
# is non-standard
continue
parameter.save()
except ParameterName.DoesNotExist:
pass
def save_rif_cs_profile(self, experiment, profile):
"""
Save selected profile choice as experiment parameter
"""
namespace = "http://monash.edu.au/rif-cs/profile/"
schema = None
try:
schema = Schema.objects.get(
namespace__exact=namespace)
except Schema.DoesNotExist:
logger.debug('Schema ' + namespace +
' does not exist. Creating.')
schema = Schema(namespace=namespace)
schema.save()
parametername = ParameterName.objects.get(
schema__namespace__exact=schema.namespace,
name="profile")
parameterset = None
try:
parameterset = \
ExperimentParameterSet.objects.get(\
schema=schema,
experiment=experiment)
except ExperimentParameterSet.DoesNotExist, e:
parameterset = ExperimentParameterSet(\
schema=schema,
experiment=experiment)
parameterset.save()
def create_draft_publication(user, publication_title, publication_description):
# Note: Maybe this logic can be taken from the tardis_portal/views.py?
experiment = Experiment(created_by=user,
title=publication_title,
description=publication_description)
experiment.save()
ObjectACL(content_object=experiment,
pluginId=django_user,
entityId=str(user.id),
canRead=True,
canWrite=False,
canDelete=False,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED).save()
ObjectACL(content_object=experiment,
pluginId=django_group,
entityId=str(
Group.objects.get_or_create(
name=getattr(
settings, 'PUBLICATION_OWNER_GROUP',
default_settings.PUBLICATION_OWNER_GROUP))[0].id),
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED).save()
publication_schema = Schema.objects.get(
namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT))
# Attach draft schema
draft_publication_schema = Schema.objects.get(
namespace=getattr(settings, 'PUBLICATION_DRAFT_SCHEMA',
default_settings.PUBLICATION_DRAFT_SCHEMA))
ExperimentParameterSet(schema=draft_publication_schema,
experiment=experiment).save()
# Attach root schema and blank form_state parameter
publication_root_schema = Schema.objects.get(
namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT))
publication_root_parameter_set = ExperimentParameterSet(
schema=publication_schema,
experiment=experiment)
publication_root_parameter_set.save()
form_state_param_name = ParameterName.objects.get(
schema=publication_root_schema, name='form_state')
ExperimentParameter(name=form_state_param_name,
parameterset=publication_root_parameter_set).save()
return experiment
def _create_related_info(request, experiment_id):
if not authz.has_write_permissions(request, experiment_id):
return return_response_error(request)
form = RelatedInfoForm(json.loads(request.body))
if not form.is_valid():
return HttpResponse('', status=400)
ps = ExperimentParameterSet(experiment_id=experiment_id,
schema=_get_schema())
ps.save()
ParameterSetManager(ps).set_params_from_dict(form.cleaned_data)
return HttpResponse(json.dumps(_get_dict_from_ps(ps)),
content_type='application/json; charset=utf-8',
status=201)
def _create(self, request, experiment_id):
from tardis.tardis_portal.auth.decorators import has_experiment_write
if not has_experiment_write(request, experiment_id):
return return_response_error(request)
form = self.form_cls(json.loads(request.body))
if not form.is_valid():
return HttpResponse('', status=400)
ps = ExperimentParameterSet(experiment_id=experiment_id,
schema=self.schema)
ps.save()
ParameterSetManager(ps).set_params_from_dict(form.cleaned_data)
return HttpResponse(json.dumps(self._get_dict_from_ps(ps)),
content_type='application/json; charset=utf-8',
status=201)
def set_embargo_release_date(publication, release_date):
pub_schema_root = Schema.objects.get(
namespace=getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT))
pub_schema_root_parameter_set = ExperimentParameterSet(
schema=pub_schema_root,
experiment=publication)
pub_schema_root_parameter_set.save()
embargo_parameter_name = ParameterName.objects.get(
schema=pub_schema_root,
name='embargo')
ExperimentParameter(name=embargo_parameter_name,
parameterset=pub_schema_root_parameter_set,
datetime_value=release_date).save()
def add_info(self, cleaned_data):
logger.debug('adding info')
logger.debug(cleaned_data)
type = cleaned_data['type']
identifier_type = cleaned_data['identifier_type']
identifier = cleaned_data['identifier']
title = cleaned_data['title']
notes = cleaned_data['notes']
eps = ExperimentParameterSet(experiment_id=self.experiment_id, schema=self.schema)
eps.save()
_maybe_add(eps, self.type_name, type)
_maybe_add(eps, self.identifier_type_name, identifier_type, force=True)
_maybe_add(eps, self.identifier_name, identifier, force=True)
_maybe_add(eps, self.title_name, title)
_maybe_add(eps, self.notes_name, notes)
def _save_party_refs(self, party, party_relation):
""" Save party and party relation information as parameters on the
experiment
"""
namespace = "http://rmit.edu.au/rif-cs/party/1.0/"
logger.debug("saving party")
schema = None
try:
schema = Schema.objects.get(
namespace__exact=namespace)
except Schema.DoesNotExist:
logger.debug('Schema ' + namespace +
' does not exist. Creating.')
schema = Schema(namespace=namespace)
schema.save()
exp = Experiment.objects.get(pk=self.experiment_id)
party_id_param = self._make_param(schema=schema,
name="party_id",
paramtype=ParameterName.NUMERIC)
relation_param = self._make_param(schema=schema,
name="relationtocollection_id",
paramtype=ParameterName.STRING)
parameterset = ExperimentParameterSet(schema=schema, experiment=exp)
parameterset.save()
ep = ExperimentParameter.objects.filter(name=party_id_param,
parameterset=parameterset,
parameterset__experiment=exp)
for p in ep:
p.delete()
ep = ExperimentParameter(
parameterset=parameterset,
name=party_id_param,
numerical_value=party.pk)
ep.save()
ep = ExperimentParameter.objects.filter(name=relation_param,
parameterset=parameterset,
parameterset__experiment=exp)
for p in ep:
p.delete()
ep = ExperimentParameter(
parameterset=parameterset,
name=relation_param,
string_value=party_relation)
ep.save()
def synchrotron_search_epn(publication):
# *** Synchrotron specific ***
# Search for beamline/EPN information associated with each dataset
# and add to the publication.
try:
synch_epn_schema = Schema.objects.get(
namespace='http://www.tardis.edu.au/schemas/as/'
'experiment/2010/09/21')
datasets = Dataset.objects.filter(experiments=publication)
synch_experiments = Experiment.objects.filter(
datasets__in=datasets,
experimentparameterset__schema=synch_epn_schema).exclude(
pk=publication.pk).distinct()
for exp in [s for s in
synch_experiments if not s.is_publication()]:
epn = ExperimentParameter.objects.get(
name__name='EPN',
name__schema=synch_epn_schema,
parameterset__experiment=exp).string_value
beamline = ExperimentParameter.objects.get(
name__name='beamline',
name__schema=synch_epn_schema,
parameterset__experiment=exp).string_value
epn_parameter_set = ExperimentParameterSet(
schema=synch_epn_schema,
experiment=publication)
epn_parameter_set.save()
epn_copy = ExperimentParameter(
name=ParameterName.objects.get(
name='EPN', schema=synch_epn_schema),
parameterset=epn_parameter_set)
epn_copy.string_value = epn
epn_copy.save()
beamline_copy = ExperimentParameter(
name=ParameterName.objects.get(
name='beamline', schema=synch_epn_schema),
parameterset=epn_parameter_set)
beamline_copy.string_value = beamline
beamline_copy.save()
except Schema.DoesNotExist:
pass
def __init__(self, experiment_id, create=False):
self.experiment = Experiment.objects.get(pk=experiment_id)
parametersets = ExperimentParameterSet.objects.filter(
schema__namespace=NAMESPACE, experiment__id=experiment_id)
self.schema, _ = Schema.objects.get_or_create(namespace=NAMESPACE, name='Embargo Details')
self.expiry_date, _ = ParameterName.objects.get_or_create(schema=self.schema, name=EXPIRY_DATE_KEY, full_name='Expiry', immutable=True, data_type=ParameterName.DATETIME)
self.never_expire, _ = ParameterName.objects.get_or_create(schema=self.schema, name=NEVER_EXPIRE_KEY, full_name='Never Expires', immutable=True, data_type=ParameterName.STRING)
if len(parametersets) == 1:
self.parameterset = parametersets[0]
elif create:
self.parameterset = ExperimentParameterSet(experiment=self.experiment, schema=self.schema)
self.parameterset.save()
else:
self.parameterset = None
class ParameterSetManager(object):
parameterset = None
parameters = None # queryset of parameters
blank_param = None
# parameterset OR schema / datafile / dataset / experiment
# delete dataset creation code
# make parameterset / object arguments generic and test type
# create function to return generic parameter type for setting/getting
#
# 2011/07/20 (Ryan Braganza)
# changed type checking from:
# type(self.parameterset).__name__ == 'ClassName'
# to:
# isinstance(self.parameterset, Class)
# This is to solve an issue with models using deferred fields.
# When using deferred fields, a subclass is used, so the class name
# will not match when using type(self.parameterset).__name__
# isinstance deals with inheritance appropriately here
def __init__(self, parameterset=None, parentObject=None, schema=None):
"""
instantiate new task or existing ParameterSet
:param dataset: optional parameter to instanciate task from
metadata, will be tested for completeness and copied into
new task if complete
:type dataset: Dataset
"""
if parameterset:
self.parameterset = parameterset
self.schema = self.parameterset.schema
self.namespace = self.schema.namespace
if isinstance(self.parameterset, DatafileParameterSet):
self.parameters = DatafileParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatafileParameter
elif isinstance(self.parameterset, DatasetParameterSet):
self.parameters = DatasetParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatasetParameter
elif isinstance(self.parameterset, ExperimentParameterSet):
self.parameters = ExperimentParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parameterset object given.")
elif parentObject and schema:
self.namespace = schema
if isinstance(parentObject, Dataset_File):
self.parameterset = DatafileParameterSet(
schema=self.get_schema(), dataset_file=parentObject)
self.parameterset.save()
self.parameters = DatafileParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatafileParameter
elif isinstance(parentObject, Dataset):
self.parameterset = DatasetParameterSet(
schema=self.get_schema(), dataset=parentObject)
self.parameterset.save()
self.parameters = DatasetParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatasetParameter
elif isinstance(parentObject, Experiment):
self.parameterset = ExperimentParameterSet(
schema=self.get_schema(), experiment=parentObject)
self.parameterset.save()
self.parameters = ExperimentParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parent object." +
"Must be an experiment/dataset/datafile not " +
str(type(parentObject)))
else:
raise TypeError("Missing arguments")
def get_schema(self):
try:
schema = Schema.objects.get(namespace=self.namespace)
except ObjectDoesNotExist:
schema = Schema()
schema.namespace = self.namespace
schema.save()
self.schema = schema
return schema
def get_param(self, parname, value=False):
par = self.parameters.get(name__name=parname)
if value:
if par.name.isNumeric():
return par.numerical_value
else:
return par.string_value
return par
def get_params(self, parname, value=False):
pars = self.parameters.filter(name__name=parname)
if value:
if len(pars) > 0 and pars[0].name.isNumeric():
return [par.numerical_value for par in pars]
else:
return [par.string_value for par in pars]
return pars
def set_param(self, parname, value, fullparname=None, example_value=None):
try:
param = self.get_param(parname)
except ObjectDoesNotExist:
param = self.blank_param()
param.parameterset = self.parameterset
param.name = self._get_create_parname(parname,
fullparname,
example_value=example_value)
#param.string_value = value
#param.save()
if param.name.isNumeric():
param.numerical_value = float(value)
else:
param.string_value = str(value)
param.save()
return param.id
def new_param(self, parname, value, fullparname=None):
param = self.blank_param()
param.parameterset = self.parameterset
param.name = self._get_create_parname(parname, fullparname)
param.string_value = value
param.save()
if param.name.isNumeric():
param.numerical_value = float(value)
else:
param.string_value = str(value)
param.save()
return param.id
# use this one from post data
def set_param_list(self, parname, value_list, fullparname=None):
self.delete_params(parname)
for value in value_list:
if value != None:
self.new_param(parname, value, fullparname)
def set_params_from_dict(self, dict):
for (key, value) in dict.iteritems():
if type(value) is list:
self.set_param_list(key, value)
else:
if value != None:
self.delete_params(key)
self.set_param(key, value)
def delete_params(self, parname):
params = self.get_params(parname)
for param in params:
param.delete()
def delete_all_params(self):
for param in self.parameters:
param.delete()
def _get_create_parname(self,
parname,
fullparname=None,
example_value=None):
try:
paramName = ParameterName.objects.get(
name=parname, schema__id=self.get_schema().id)
except ObjectDoesNotExist:
paramName = ParameterName()
paramName.schema = self.get_schema()
paramName.name = parname
if fullparname:
paramName.full_name = fullparname
else:
paramName.full_name = parname
if example_value:
try:
float(example_value)
paramName.data_type = ParameterName.NUMERIC
except (TypeError, ValueError):
paramName.data_type = ParameterName.STRING
else:
paramName.data_type = ParameterName.STRING
paramName.is_searchable = True
paramName.save()
return paramName
def process_form(request):
# Decode the form data
form_state = json.loads(request.body)
def validation_error(error=None):
if error is None:
error = 'Invalid form data was submitted ' \
'(server-side validation failed)'
return HttpResponse(
json.dumps({
'error': error}),
content_type="application/json")
# Check if the form data contains a publication ID
# If it does, then this publication needs to be updated
# rather than created.
if 'publicationId' not in form_state:
if not form_state['publicationTitle'].strip():
return validation_error()
publication = create_draft_publication(
request.user, form_state['publicationTitle'],
form_state['publicationDescription'])
form_state['publicationId'] = publication.id
else:
publication = get_draft_publication(
request.user, form_state['publicationId'])
# Check if the publication is finalised (i.e. not in draft)
# if it is, then refuse to process the form.
if publication is None or not publication.is_publication_draft():
return HttpResponseForbidden()
# Get the form state database object
form_state_parameter = ExperimentParameter.objects.get(
name__name='form_state',
name__schema__namespace=getattr(
settings, 'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT),
parameterset__experiment=publication)
# Check if the form state needs to be loaded (i.e. a publication draft
# is resumed)
# no database changes are made if the form is resumed
if form_state['action'] == 'resume':
form_state = json.loads(form_state_parameter.string_value)
return HttpResponse(json.dumps(form_state),
content_type="application/json")
if form_state['action'] == 'update-dataset-selection':
# Update the publication title/description if changed.
# Must not be blank.
if not form_state['publicationTitle'].strip() or \
not form_state['publicationDescription'].strip():
return validation_error()
if publication.title != form_state['publicationTitle']:
publication.title = form_state['publicationTitle']
publication.save()
if publication.description != form_state['publicationDescription']:
publication.description = form_state['publicationDescription']
publication.save()
# Update associated datasets
# (note: might not be efficient re: db queries)
# ... first clear all current associations
current_datasets = Dataset.objects.filter(experiments=publication)
for current_dataset in current_datasets:
current_dataset.experiments.remove(publication)
# ... now (re)add all datasets
selected_datasets = [ds['dataset']['id']
for ds in form_state['addedDatasets']]
datasets = Dataset.objects.filter(
experiments__in=Experiment.safe.owned_and_shared(request.user),
pk__in=selected_datasets).distinct()
for dataset in datasets:
dataset.experiments.add(publication)
# --- Get data for the next page --- #
# Construct the disclipline-specific form based on the
# selected datasets
selected_forms = select_forms(datasets)
if 'disciplineSpecificFormTemplates' in form_state:
# clear extraInfo if the selected forms differ
# (i.e. datasets have changed)
if json.dumps(selected_forms) != json.dumps(
form_state['disciplineSpecificFormTemplates']):
form_state['extraInfo'] = {}
form_state['disciplineSpecificFormTemplates'] = selected_forms
elif form_state['action'] == 'update-extra-info':
# Clear any current parameter sets except for those belonging
# to the publication draft schema or containing the form_state
# parameter
clear_publication_metadata(publication)
# Loop through form data and create associates parameter sets
# Any unrecognised fields or schemas are ignored!
map_form_to_schemas(form_state['extraInfo'], publication)
# *** Synchrotron specific ***
# Search for beamline/EPN information associated with each dataset
# and add to the publication.
synchrotron_search_epn(publication)
# --- Get data for the next page --- #
licenses_json = get_licenses()
form_state['licenses'] = licenses_json
# Select the first license as default
if licenses_json:
if 'selectedLicenseId' not in form_state:
form_state['selectedLicenseId'] = licenses_json[0]['id']
else: # No licenses configured...
form_state['selectedLicenseId'] = -1
# Set a default author (current user) if no previously saved data
# By default, the form sends a list of authors of one element
# with blank fields
if len(form_state['authors']) == 1 and \
not form_state['authors'][0]['name']:
form_state['authors'] = [
{'name': ' '.join([request.user.first_name,
request.user.last_name]),
'institution': getattr(settings, 'DEFAULT_INSTITUTION', ''),
'email': request.user.email}]
elif form_state['action'] == 'submit':
# any final form validation should occur here
# and specific error messages can be returned
# to the browser before the publication's draft
# status is removed.
if 'acknowledge' not in form_state or not form_state['acknowledge']:
return validation_error('You must confirm that you are '
'authorised to submit this publication')
set_publication_authors(form_state['authors'], publication)
institutions = '; '.join(
set([author['institution'] for author in form_state['authors']]))
publication.institution_name = institutions
# Attach the publication details schema
pub_details_schema = Schema.objects.get(
namespace=getattr(settings, 'PUBLICATION_DETAILS_SCHEMA',
default_settings.PUBLICATION_DETAILS_SCHEMA))
pub_details_parameter_set = ExperimentParameterSet(
schema=pub_details_schema,
experiment=publication)
pub_details_parameter_set.save()
# Add the acknowledgements
acknowledgements_parameter_name = ParameterName.objects.get(
schema=pub_details_schema,
name='acknowledgements')
ExperimentParameter(name=acknowledgements_parameter_name,
parameterset=pub_details_parameter_set,
string_value=form_state['acknowledgements']).save()
# Set the release date
set_embargo_release_date(
publication,
dateutil.parser.parse(
form_state[
'embargo']))
# Set the license
try:
publication.license = License.objects.get(
pk=form_state['selectedLicenseId'],
is_active=True,
allows_distribution=True)
except License.DoesNotExist:
publication.license = License.get_none_option_license()
publication.save()
# Send emails about publication in draft
subject, message_content = email_pub_requires_authorisation(
request.user.username,
request.build_absolute_uri(
reverse('tardis_portal.view_experiment',
args=(publication.id,))),
request.build_absolute_uri(
'/apps/publication-forms/approvals/'))
try:
send_mail(subject,
message_content,
getattr(
settings, 'PUBLICATION_NOTIFICATION_SENDER_EMAIL',
default_settings.PUBLICATION_NOTIFICATION_SENDER_EMAIL),
get_pub_admin_email_addresses(),
fail_silently=False)
subject, message_content = email_pub_awaiting_approval(
publication.title)
send_mail_to_authors(publication, subject, message_content,
fail_silently=False)
except Exception as e:
logger.error(
"failed to send publication notification email(s): %s" %
repr(e)
)
return HttpResponse(
json.dumps({
'error': 'Failed to send notification email - please '
'contact the %s administrator (%s), '
'or try again later. Your draft is saved.'
% (get_site_admin_email(),
getattr(settings, 'SITE_TITLE', 'MyTardis'))
}),
content_type="application/json")
# Remove the draft status
remove_draft_status(publication)
# Automatically approve publications if approval is not required
if not getattr(settings, 'PUBLICATIONS_REQUIRE_APPROVAL',
default_settings.PUBLICATIONS_REQUIRE_APPROVAL):
approve_publication(request, publication, message=None,
send_email=False)
# approve_publication will delete the form state, so don't
# bother saving is and return.
form_state['action'] = ''
return HttpResponse(json.dumps(form_state),
content_type="appication/json")
# Trigger publication record update
tasks.update_publication_records.delay()
# Clear the form action and save the state
form_state['action'] = ''
form_state_parameter.string_value = json.dumps(form_state)
form_state_parameter.save()
return HttpResponse(json.dumps(form_state), content_type="appication/json")
def populate_pdb_pub_records():
PUB_SCHEMA = getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT)
PUB_SCHEMA_DRAFT = getattr(settings, 'PUBLICATION_DRAFT_SCHEMA',
default_settings.PUBLICATION_DRAFT_SCHEMA)
PDB_SCHEMA = getattr(settings, 'PDB_PUBLICATION_SCHEMA_ROOT',
default_settings.PDB_PUBLICATION_SCHEMA_ROOT)
publications = Experiment.objects \
.filter(experimentparameterset__schema__namespace=PDB_SCHEMA) \
.filter(experimentparameterset__schema__namespace=PUB_SCHEMA) \
.exclude(experimentparameterset__schema__namespace=PUB_SCHEMA_DRAFT) \
.distinct()
last_update_parameter_name = ParameterName.objects.get(
name='pdb-last-sync',
schema__namespace=PUB_SCHEMA)
def add_if_missing(parameterset, name, string_value=None,
numerical_value=None, datetime_value=None):
try:
ExperimentParameter.objects.get(
name__name=name, parameterset=parameterset)
except ExperimentParameter.DoesNotExist:
param_name = ParameterName.objects.get(
name=name, schema=parameterset.schema)
param = ExperimentParameter(name=param_name,
parameterset=parameterset)
param.string_value = string_value
param.numerical_value = numerical_value
param.datetime_value = datetime_value
param.save()
for pub in publications:
try:
# try to get the last update time for the PDB data
pdb_last_update_parameter = ExperimentParameter.objects.get(
parameterset__schema__namespace=PUB_SCHEMA,
name=last_update_parameter_name,
parameterset__experiment=pub
)
last_update = pdb_last_update_parameter.datetime_value
needs_update = last_update + \
getattr(settings,
'PDB_REFRESH_INTERVAL',
default_settings.PDB_REFRESH_INTERVAL) \
< timezone.now()
except ExperimentParameter.DoesNotExist:
# if the PDB last update time parameter doesn't exist,
# we definitely need to update the data and create a last
# update entry
needs_update = True
pdb_last_update_parameter = None
# If an update needs to happen...
if needs_update:
# 1. get the PDB info
pdb_parameter_set = ExperimentParameterSet.objects.get(
schema__namespace=getattr(
settings,
'PDB_PUBLICATION_SCHEMA_ROOT',
default_settings.PDB_PUBLICATION_SCHEMA_ROOT),
experiment=pub)
pdb = ExperimentParameter.objects.get(
name__name='pdb-id',
parameterset=pdb_parameter_set)
pdb_id = pdb.string_value
# 1a. cosmetic change of case for PDB ID, if entered incorrectly
if pdb_id != pdb_id.upper():
pdb.string_value = pdb_id.upper()
pdb.save()
try:
# 2. fetch the info from pdb.org
pdb = PDBCifHelper(pdb_id)
# 3. insert all standard pdb parameters
add_if_missing(pdb_parameter_set, 'title',
string_value=pdb.get_pdb_title())
add_if_missing(pdb_parameter_set, 'url',
string_value=pdb.get_pdb_url())
try:
add_if_missing(pdb_parameter_set, 'resolution',
numerical_value=pdb.get_resolution())
except ValueError:
logger.error(
'PDB field "resolution" could not be set for '
'publication Id %i \n %s' %
(pub.id, traceback.format_exc()))
try:
add_if_missing(pdb_parameter_set, 'r-value',
numerical_value=pdb.get_obs_r_value())
except ValueError:
logger.error(
'PDB field "r-value" could not be set for '
'publication Id %i \n %s' %
(pub.id, traceback.format_exc()))
try:
add_if_missing(pdb_parameter_set, 'r-free',
numerical_value=pdb.get_free_r_value())
except ValueError:
logger.error(
'PDB field "r-free" could not be set for '
'publication Id %i \n %s' %
(pub.id, traceback.format_exc()))
add_if_missing(pdb_parameter_set, 'space-group',
string_value=pdb.get_spacegroup())
add_if_missing(pdb_parameter_set, 'unit-cell',
string_value=pdb.get_unit_cell())
# 4. insert sequence info (lazy checking)
pdb_seq_parameter_sets = ExperimentParameterSet.objects.filter(
schema__namespace=getattr(
settings,
'PDB_SEQUENCE_PUBLICATION_SCHEMA',
default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA),
experiment=pub)
if pdb_seq_parameter_sets.count() == 0:
# insert seqences
for seq in pdb.get_sequence_info():
seq_ps_namespace = getattr(
settings,
'PDB_SEQUENCE_PUBLICATION_SCHEMA',
default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA)
seq_parameter_set = ExperimentParameterSet(
schema=Schema.objects.get(
namespace=seq_ps_namespace),
experiment=pub)
seq_parameter_set.save()
add_if_missing(seq_parameter_set, 'organism',
string_value=seq['organism'])
add_if_missing(seq_parameter_set, 'expression-system',
string_value=seq['expression_system'])
add_if_missing(seq_parameter_set, 'sequence',
string_value=seq['sequence'])
# 5. insert/update citation info (aggressive)
ExperimentParameterSet.objects.filter(
schema__namespace=getattr(
settings,
'PDB_CITATION_PUBLICATION_SCHEMA',
default_settings.PDB_CITATION_PUBLICATION_SCHEMA),
experiment=pub).delete()
for citation in pdb.get_citations():
cit_ps_namespace = getattr(
settings,
'PDB_CITATION_PUBLICATION_SCHEMA',
default_settings.PDB_CITATION_PUBLICATION_SCHEMA)
cit_parameter_set = ExperimentParameterSet(
schema=Schema.objects.get(namespace=cit_ps_namespace),
experiment=pub)
cit_parameter_set.save()
add_if_missing(cit_parameter_set, 'title',
string_value=citation['title'])
add_if_missing(cit_parameter_set, 'authors',
string_value='; '.join(citation['authors']))
add_if_missing(cit_parameter_set, 'journal',
string_value=citation['journal'])
add_if_missing(cit_parameter_set, 'volume',
string_value=citation['volume'])
add_if_missing(cit_parameter_set, 'page-range',
string_value='-'.join(
[citation['page_first'],
citation['page_last']]))
add_if_missing(cit_parameter_set, 'doi',
string_value='http://dx.doi.org/' +
citation['doi'])
# 6. Remove the PDB embargo if set, since the update has
# occurred and therefore the PDB must have been relased.
try:
ExperimentParameter.objects.get(
name__name='pdb-embargo',
parameterset__schema__namespace=getattr(
settings,
'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT)).delete()
except ExperimentParameter.DoesNotExist:
pass
# 7. Set the last update parameter to be now
if pdb_last_update_parameter is None:
pub_parameter_set = ExperimentParameterSet(
schema=Schema.objects.get(namespace=PUB_SCHEMA),
experiment=pub)
pub_parameter_set.save()
pdb_last_update_parameter = ExperimentParameter(
name=last_update_parameter_name,
parameterset=pub_parameter_set,
datetime_value=timezone.now())
else:
pdb_last_update_parameter.datetime_value = timezone.now()
pdb_last_update_parameter.save()
except CifFile.StarError:
# PDB is either unavailable or invalid
# (maybe notify the user somehow?)
continue
class ParameterSetManager(object):
parameterset = None
parameters = None # queryset of parameters
blank_param = None
# parameterset OR schema / datafile / dataset / experiment
# delete dataset creation code
# make parameterset / object arguments generic and test type
# create function to return generic parameter type for setting/getting
def __init__(self, parameterset=None, parentObject=None,
schema=None):
"""
instantiate new task or existing ParameterSet
:param dataset: optional parameter to instanciate task from
metadata, will be tested for completeness and copied into
new task if complete
:type dataset: Dataset
"""
if parameterset:
self.parameterset = parameterset
self.schema = self.parameterset.schema
self.namespace = self.schema.namespace
if type(self.parameterset).__name__ == "DatafileParameterSet":
self.parameters = DatafileParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatafileParameter
elif type(self.parameterset).__name__ == "DatasetParameterSet":
self.parameters = DatasetParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatasetParameter
elif type(self.parameterset).__name__ == "ExperimentParameterSet":
self.parameters = ExperimentParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parameterset object given.")
elif parentObject and schema:
self.namespace = schema
if type(parentObject).__name__ == "Dataset_File":
self.parameterset = DatafileParameterSet(
schema=self.get_schema(), dataset_file=parentObject)
self.parameterset.save()
self.parameters = DatafileParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatafileParameter
elif type(parentObject).__name__ == "Dataset":
self.parameterset = DatasetParameterSet(
schema=self.get_schema(), dataset=parentObject)
self.parameterset.save()
self.parameters = DatasetParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatasetParameter
elif type(parentObject).__name__ == "Experiment":
self.parameterset = ExperimentParameterSet(
schema=self.get_schema(), experiment=parentObject)
self.parameterset.save()
self.parameters = ExperimentParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parent object." +
"Must be an experiment/dataset/datafile")
else:
raise TypeError("Missing arguments")
def get_schema(self):
try:
schema = Schema.objects.get(
namespace=self.namespace)
except ObjectDoesNotExist:
schema = Schema()
schema.namespace = self.namespace
schema.save()
self.schema = schema
return schema
def get_param(self, parname, value=False):
par = self.parameters.get(name__name=parname)
if value:
if par.name.isNumeric():
return par.numerical_value
else:
return par.string_value
return par
def get_params(self, parname, value=False):
pars = self.parameters.filter(name__name=parname)
if value:
if len(pars) > 0 and pars[0].name.isNumeric():
return [par.numerical_value
for par in pars]
else:
return [par.string_value
for par in pars]
return pars
def set_param(self, parname, value, fullparname=None,
example_value=None):
try:
param = self.get_param(parname)
except ObjectDoesNotExist:
param = self.blank_param()
param.parameterset = self.parameterset
param.name = self._get_create_parname(parname, fullparname,
example_value=example_value)
#param.string_value = value
#param.save()
if param.name.isNumeric():
param.numerical_value = float(value)
else:
param.string_value = str(value)
param.save()
def new_param(self, parname, value, fullparname=None):
param = self.blank_param()
param.parameterset = self.parameterset
param.name = self._get_create_parname(parname, fullparname)
param.string_value = value
param.save()
if param.name.isNumeric():
param.numerical_value = float(value)
else:
param.string_value = str(value)
param.save()
# use this one from post data
def set_param_list(self, parname, value_list, fullparname=None):
self.delete_params(parname)
for value in value_list:
if value != None:
self.new_param(parname, value, fullparname)
def set_params_from_dict(self, dict):
print type(dict)
for (key, value) in dict.iteritems():
if type(value) is list:
self.set_param_list(key, value)
else:
if value != None:
self.delete_params(key)
self.set_param(key, value)
def delete_params(self, parname):
params = self.get_params(parname)
for param in params:
param.delete()
def delete_all_params(self):
for param in self.parameters:
param.delete()
def _get_create_parname(self, parname,
fullparname=None, example_value=None):
try:
paramName = ParameterName.objects.get(name=parname,
schema__id=self.get_schema().id)
except ObjectDoesNotExist:
paramName = ParameterName()
paramName.schema = self.get_schema()
paramName.name = parname
if fullparname:
paramName.full_name = fullparname
else:
paramName.full_name = parname
if example_value:
try:
float(example_value)
paramName.data_type = ParameterName.NUMERIC
except (TypeError, ValueError):
paramName.data_type = ParameterName.STRING
else:
paramName.data_type = ParameterName.STRING
paramName.is_searchable = True
paramName.save()
return paramName
def delete_info(self, parameterset_id):
eps = ExperimentParameterSet(experiment_id=self.experiment_id, schema=self.schema, pk=parameterset_id)
eps.delete()
def delete_info(self, parameterset_id):
eps = ExperimentParameterSet(experiment_id=self.experiment_id,
schema=self.schema,
pk=parameterset_id)
eps.delete()
def process_form(request):
# Decode the form data
form_state = json.loads(request.body)
def validation_error(error=None):
if error is None:
error = 'Invalid form data was submitted ' \
'(server-side validation failed)'
return HttpResponse(json.dumps({'error': error}),
content_type="application/json")
# Check if the form data contains a publication ID
# If it does, then this publication needs to be updated
# rather than created.
if 'publicationId' not in form_state:
if not form_state['publicationTitle'].strip():
return validation_error()
publication = create_draft_publication(
request.user, form_state['publicationTitle'],
form_state['publicationDescription'])
form_state['publicationId'] = publication.id
else:
publication = get_draft_publication(request.user,
form_state['publicationId'])
# Check if the publication is finalised (i.e. not in draft)
# if it is, then refuse to process the form.
if publication is None or not publication.is_publication_draft():
return HttpResponseForbidden()
# Get the form state database object
form_state_parameter = ExperimentParameter.objects.get(
name__name='form_state',
name__schema__namespace=getattr(
settings, 'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT),
parameterset__experiment=publication)
# Check if the form state needs to be loaded (i.e. a publication draft
# is resumed)
# no database changes are made if the form is resumed
if form_state['action'] == 'resume':
form_state = json.loads(form_state_parameter.string_value)
return HttpResponse(json.dumps(form_state),
content_type="application/json")
if form_state['action'] == 'update-dataset-selection':
# Update the publication title/description if changed.
# Must not be blank.
if not form_state['publicationTitle'].strip() or \
not form_state['publicationDescription'].strip():
return validation_error()
if publication.title != form_state['publicationTitle']:
publication.title = form_state['publicationTitle']
publication.save()
if publication.description != form_state['publicationDescription']:
publication.description = form_state['publicationDescription']
publication.save()
# Update associated datasets
# (note: might not be efficient re: db queries)
# ... first clear all current associations
current_datasets = Dataset.objects.filter(experiments=publication)
for current_dataset in current_datasets:
current_dataset.experiments.remove(publication)
# ... now (re)add all datasets
selected_datasets = [
ds['dataset']['id'] for ds in form_state['addedDatasets']
]
datasets = Dataset.objects.filter(
experiments__in=Experiment.safe.owned_and_shared(request.user),
pk__in=selected_datasets).distinct()
for dataset in datasets:
dataset.experiments.add(publication)
# --- Get data for the next page --- #
# Construct the disclipline-specific form based on the
# selected datasets
selected_forms = select_forms(datasets)
if 'disciplineSpecificFormTemplates' in form_state:
# clear extraInfo if the selected forms differ
# (i.e. datasets have changed)
if json.dumps(selected_forms) != json.dumps(
form_state['disciplineSpecificFormTemplates']):
form_state['extraInfo'] = {}
form_state['disciplineSpecificFormTemplates'] = selected_forms
elif form_state['action'] == 'update-extra-info':
# Clear any current parameter sets except for those belonging
# to the publication draft schema or containing the form_state
# parameter
clear_publication_metadata(publication)
# Loop through form data and create associates parameter sets
# Any unrecognised fields or schemas are ignored!
map_form_to_schemas(form_state['extraInfo'], publication)
# *** Synchrotron specific ***
# Search for beamline/EPN information associated with each dataset
# and add to the publication.
synchrotron_search_epn(publication)
# --- Get data for the next page --- #
licenses_json = get_licenses()
form_state['licenses'] = licenses_json
# Select the first license as default
if licenses_json:
if 'selectedLicenseId' not in form_state:
form_state['selectedLicenseId'] = licenses_json[0]['id']
else: # No licenses configured...
form_state['selectedLicenseId'] = -1
# Set a default author (current user) if no previously saved data
# By default, the form sends a list of authors of one element
# with blank fields
if len(form_state['authors']) == 1 and \
not form_state['authors'][0]['name']:
form_state['authors'] = [{
'name':
' '.join([request.user.first_name, request.user.last_name]),
'institution':
getattr(settings, 'DEFAULT_INSTITUTION', ''),
'email':
request.user.email
}]
elif form_state['action'] == 'submit':
# any final form validation should occur here
# and specific error messages can be returned
# to the browser before the publication's draft
# status is removed.
if 'acknowledge' not in form_state or not form_state['acknowledge']:
return validation_error('You must confirm that you are '
'authorised to submit this publication')
set_publication_authors(form_state['authors'], publication)
institutions = '; '.join(
set([author['institution'] for author in form_state['authors']]))
publication.institution_name = institutions
# Attach the publication details schema
pub_details_schema = Schema.objects.get(
namespace=getattr(settings, 'PUBLICATION_DETAILS_SCHEMA',
default_settings.PUBLICATION_DETAILS_SCHEMA))
pub_details_parameter_set = ExperimentParameterSet(
schema=pub_details_schema, experiment=publication)
pub_details_parameter_set.save()
# Add the acknowledgements
acknowledgements_parameter_name = ParameterName.objects.get(
schema=pub_details_schema, name='acknowledgements')
ExperimentParameter(
name=acknowledgements_parameter_name,
parameterset=pub_details_parameter_set,
string_value=form_state['acknowledgements']).save()
# Set the release date
set_embargo_release_date(publication,
dateutil.parser.parse(form_state['embargo']))
# Set the license
try:
publication.license = License.objects.get(
pk=form_state['selectedLicenseId'],
is_active=True,
allows_distribution=True)
except License.DoesNotExist:
publication.license = License.get_none_option_license()
publication.save()
# Send emails about publication in draft
subject, message_content = email_pub_requires_authorisation(
request.user.username,
request.build_absolute_uri(
reverse('tardis_portal.view_experiment',
args=(publication.id, ))),
request.build_absolute_uri('/apps/publication-forms/approvals/'))
try:
send_mail(
subject,
message_content,
getattr(
settings, 'PUBLICATION_NOTIFICATION_SENDER_EMAIL',
default_settings.PUBLICATION_NOTIFICATION_SENDER_EMAIL),
get_pub_admin_email_addresses(),
fail_silently=False)
subject, message_content = email_pub_awaiting_approval(
publication.title)
send_mail_to_authors(publication,
subject,
message_content,
fail_silently=False)
except Exception as e:
logger.error(
"failed to send publication notification email(s): %s" %
repr(e))
return HttpResponse(json.dumps({
'error':
'Failed to send notification email - please '
'contact the %s administrator (%s), '
'or try again later. Your draft is saved.' %
(get_site_admin_email(),
getattr(settings, 'SITE_TITLE', 'MyTardis'))
}),
content_type="application/json")
# Remove the draft status
remove_draft_status(publication)
# Automatically approve publications if approval is not required
if not getattr(settings, 'PUBLICATIONS_REQUIRE_APPROVAL',
default_settings.PUBLICATIONS_REQUIRE_APPROVAL):
approve_publication(request,
publication,
message=None,
send_email=False)
# approve_publication will delete the form state, so don't
# bother saving is and return.
form_state['action'] = ''
return HttpResponse(json.dumps(form_state),
content_type="appication/json")
# Trigger publication record update
tasks.update_publication_records.delay()
# Clear the form action and save the state
form_state['action'] = ''
form_state_parameter.string_value = json.dumps(form_state)
form_state_parameter.save()
return HttpResponse(json.dumps(form_state), content_type="appication/json")
class EmbargoHandler(object):
def __init__(self, experiment_id, create=False):
self.experiment = Experiment.objects.get(pk=experiment_id)
parametersets = ExperimentParameterSet.objects.filter(
schema__namespace=NAMESPACE, experiment__id=experiment_id)
self.schema, _ = Schema.objects.get_or_create(namespace=NAMESPACE, name='Embargo Details')
self.expiry_date, _ = ParameterName.objects.get_or_create(schema=self.schema, name=EXPIRY_DATE_KEY, full_name='Expiry', immutable=True, data_type=ParameterName.DATETIME)
self.never_expire, _ = ParameterName.objects.get_or_create(schema=self.schema, name=NEVER_EXPIRE_KEY, full_name='Never Expires', immutable=True, data_type=ParameterName.STRING)
if len(parametersets) == 1:
self.parameterset = parametersets[0]
elif create:
self.parameterset = ExperimentParameterSet(experiment=self.experiment, schema=self.schema)
self.parameterset.save()
else:
self.parameterset = None
def never_expires(self):
never_expire = self._get_or_none(NEVER_EXPIRE_KEY)
if never_expire:
return True
expiry_date = self._get_or_none(EXPIRY_DATE_KEY)
if expiry_date:
return False
else:
return self.experiment.end_time == None
def has_any_expiry(self):
return not self.never_expires()
def can_be_defaulted(self):
expiry_date = self._get_or_none(EXPIRY_DATE_KEY)
never_expires = self._get_or_none(NEVER_EXPIRE_KEY)
return never_expires or expiry_date
def because_no_end_date(self):
if self._get_or_none(NEVER_EXPIRE_KEY):
return False
return self.experiment.end_time == None
def get_expiry_date(self):
''' returns calculated or explicit expiry or None '''
import datetime
if self.never_expires():
return None
explicit_expiry = self._get_or_none(EXPIRY_DATE_KEY)
if explicit_expiry:
return explicit_expiry.datetime_value
else:
return self.experiment.end_time + datetime.timedelta(settings.EMBARGO_DAYS)
def _get_or_none(self, name):
if not self.parameterset:
return None
params = self.parameterset.experimentparameter_set.filter(name__name=name)
if params.count() == 0:
return None
else:
return params[0]
def prevent_expiry(self):
# delete any current expiry dates
# set never_expires = True
logger.fatal('preventing expiry')
if not self.parameterset:
raise Exception('incorrectly initialised, call with create=True')
params = self.parameterset.experimentparameter_set
params.all().delete()
param = ExperimentParameter(name=self.never_expire, string_value='True', parameterset=self.parameterset)
param.save()
self.experiment.public = False
self.experiment.save()
def reset_to_default(self):
import datetime
if self.parameterset:
self.parameterset.delete()
expiry_date = self.get_expiry_date()
if expiry_date and expiry_date.date() < datetime.date.today():
self.experiment.public = True
else:
self.experiment.public = False
self.experiment.save()
else:
logger.warn('tried to delete parameterset that does not exist')
def set_expiry(self, date_string):
if not self.parameterset:
raise Exception('incorrectly initialised, call with create=True')
params = self.parameterset.experimentparameter_set
params.all().delete()
import datetime
expiry_date = datetime.datetime.strptime(date_string, '%Y/%m/%d')
param = ExperimentParameter(name=self.expiry_date, datetime_value=expiry_date, parameterset=self.parameterset)
param.save()
if expiry_date.date() < datetime.date.today():
self.experiment.public = True
else:
self.experiment.public = False
self.experiment.save()
class ParameterSetManager(object):
parameterset = None
parameters = None # queryset of parameters
blank_param = None
# parameterset OR schema / datafile / dataset / experiment
# delete dataset creation code
# make parameterset / object arguments generic and test type
# create function to return generic parameter type for setting/getting
#
# 2011/07/20 (Ryan Braganza)
# changed type checking from:
# type(self.parameterset).__name__ == 'ClassName'
# to:
# isinstance(self.parameterset, Class)
# This is to solve an issue with models using deferred fields.
# When using deferred fields, a subclass is used, so the class name
# will not match when using type(self.parameterset).__name__
# isinstance deals with inheritance appropriately here
def __init__(self, parameterset=None, parentObject=None,
schema=None):
"""
instantiate new task or existing ParameterSet
:param dataset: optional parameter to instanciate task from
metadata, will be tested for completeness and copied into
new task if complete
:type dataset: Dataset
:param schema: Schema namespace
:type schema: string
"""
if parameterset:
self.parameterset = parameterset
self.schema = self.parameterset.schema
self.namespace = self.schema.namespace
if isinstance(self.parameterset, DatafileParameterSet):
self.parameters = DatafileParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatafileParameter
elif isinstance(self.parameterset, DatasetParameterSet):
self.parameters = DatasetParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatasetParameter
elif isinstance(self.parameterset, ExperimentParameterSet):
self.parameters = ExperimentParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parameterset object given.")
elif parentObject and schema:
self.namespace = schema
if isinstance(parentObject, Dataset_File):
self.parameterset = DatafileParameterSet(
schema=self.get_schema(), dataset_file=parentObject)
self.parameterset.save()
self.parameters = DatafileParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatafileParameter
elif isinstance(parentObject, Dataset):
self.parameterset = DatasetParameterSet(
schema=self.get_schema(), dataset=parentObject)
self.parameterset.save()
self.parameters = DatasetParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatasetParameter
elif isinstance(parentObject, Experiment):
self.parameterset = ExperimentParameterSet(
schema=self.get_schema(), experiment=parentObject)
self.parameterset.save()
self.parameters = ExperimentParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parent object." +
"Must be an experiment/dataset/datafile not " + str(type(parentObject)))
else:
raise TypeError("Missing arguments")
def get_schema(self):
try:
schema = Schema.objects.get(
namespace=self.namespace)
except ObjectDoesNotExist:
schema = Schema()
schema.namespace = self.namespace
schema.save()
self.schema = schema
return schema
def get_param(self, parname, value=False):
par = self.parameters.get(name__name=parname)
if value:
if par.name.isNumeric():
return par.numerical_value
else:
return par.string_value
return par
def get_params(self, parname, value=False):
pars = self.parameters.filter(name__name=parname)
if value:
if len(pars) > 0 and pars[0].name.isNumeric():
return [par.numerical_value
for par in pars]
else:
return [par.string_value
for par in pars]
return pars
def set_param(self, parname, value, fullparname=None,
example_value=None):
try:
param = self.get_param(parname)
except ObjectDoesNotExist:
param = self.blank_param()
param.parameterset = self.parameterset
param.name = self._get_create_parname(parname, fullparname,
example_value=example_value)
#param.string_value = value
#param.save()
if param.name.isNumeric():
param.numerical_value = float(value)
else:
param.string_value = unicode(value)
param.save()
return param.id
def new_param(self, parname, value, fullparname=None):
param = self.blank_param()
param.parameterset = self.parameterset
param.name = self._get_create_parname(parname, fullparname)
param.string_value = value
param.save()
if param.name.isNumeric():
param.numerical_value = float(value)
else:
param.string_value = unicode(value)
param.save()
return param.id
# use this one from post data
def set_param_list(self, parname, value_list, fullparname=None):
self.delete_params(parname)
for value in value_list:
if value != None:
self.new_param(parname, value, fullparname)
def set_params_from_dict(self, dict):
for (key, value) in dict.iteritems():
if type(value) is list:
self.set_param_list(key, value)
else:
if value != None:
self.delete_params(key)
self.set_param(key, value)
def delete_params(self, parname):
params = self.get_params(parname)
for param in params:
param.delete()
def delete_all_params(self):
for param in self.parameters:
param.delete()
def _get_create_parname(self, parname,
fullparname=None, example_value=None):
try:
paramName = ParameterName.objects.get(name=parname,
schema__id=self.get_schema().id)
except ObjectDoesNotExist:
paramName = ParameterName()
paramName.schema = self.get_schema()
paramName.name = parname
if fullparname:
paramName.full_name = fullparname
else:
paramName.full_name = parname
if example_value:
try:
float(example_value)
paramName.data_type = ParameterName.NUMERIC
except (TypeError, ValueError):
paramName.data_type = ParameterName.STRING
else:
paramName.data_type = ParameterName.STRING
paramName.is_searchable = True
paramName.save()
return paramName
def test_parameter(self):
exp = Experiment(
title='test exp1',
institution_name='Australian Synchrotron',
approved=True,
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_NONE,
)
exp.save()
dataset = Dataset(description="dataset description")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
df_file = DataFile(dataset=dataset,
filename='file.txt',
size=42,
md5sum='bogus')
df_file.save()
df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml',
type=Schema.DATAFILE)
df_schema.save()
ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml',
type=Schema.DATASET)
ds_schema.save()
exp_schema = Schema(
namespace='http://www.cern.ch/felzmann/schema3.xml',
type=Schema.EXPERIMENT)
exp_schema.save()
df_parname = ParameterName(schema=df_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
df_parname.save()
ds_parname = ParameterName(schema=ds_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
ds_parname.save()
exp_parname = ParameterName(schema=exp_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
exp_parname.save()
df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file)
df_parset.save()
ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset)
ds_parset.save()
exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp)
exp_parset.save()
with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)):
filename = 'test.jpg'
df_parameter = DatafileParameter(name=df_parname,
parameterset=df_parset,
string_value=filename)
df_parameter.save()
ds_parameter = DatasetParameter(name=ds_parname,
parameterset=ds_parset,
string_value=filename)
ds_parameter.save()
exp_parameter = ExperimentParameter(name=exp_parname,
parameterset=exp_parset,
string_value=filename)
exp_parameter.save()
self.assertEqual(
"<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>"
% # noqa
(df_parameter.id, df_parameter.id),
df_parameter.get())
self.assertEqual(
"<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>"
% # noqa
(ds_parameter.id, ds_parameter.id),
ds_parameter.get())
self.assertEqual(
"<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>"
% # noqa
(exp_parameter.id, exp_parameter.id),
exp_parameter.get())
def populate_pdb_pub_records():
PUB_SCHEMA = getattr(settings, 'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT)
PUB_SCHEMA_DRAFT = getattr(settings, 'PUBLICATION_DRAFT_SCHEMA',
default_settings.PUBLICATION_DRAFT_SCHEMA)
PDB_SCHEMA = getattr(settings, 'PDB_PUBLICATION_SCHEMA_ROOT',
default_settings.PDB_PUBLICATION_SCHEMA_ROOT)
publications = Experiment.objects \
.filter(experimentparameterset__schema__namespace=PDB_SCHEMA) \
.filter(experimentparameterset__schema__namespace=PUB_SCHEMA) \
.exclude(experimentparameterset__schema__namespace=PUB_SCHEMA_DRAFT) \
.distinct()
last_update_parameter_name = ParameterName.objects.get(
name='pdb-last-sync', schema__namespace=PUB_SCHEMA)
def add_if_missing(parameterset,
name,
string_value=None,
numerical_value=None,
datetime_value=None):
try:
ExperimentParameter.objects.get(name__name=name,
parameterset=parameterset)
except ExperimentParameter.DoesNotExist:
param_name = ParameterName.objects.get(name=name,
schema=parameterset.schema)
param = ExperimentParameter(name=param_name,
parameterset=parameterset)
param.string_value = string_value
param.numerical_value = numerical_value
param.datetime_value = datetime_value
param.save()
for pub in publications:
try:
# try to get the last update time for the PDB data
pdb_last_update_parameter = ExperimentParameter.objects.get(
parameterset__schema__namespace=PUB_SCHEMA,
name=last_update_parameter_name,
parameterset__experiment=pub)
last_update = pdb_last_update_parameter.datetime_value
needs_update = last_update + \
getattr(settings,
'PDB_REFRESH_INTERVAL',
default_settings.PDB_REFRESH_INTERVAL) \
< timezone.now()
except ExperimentParameter.DoesNotExist:
# if the PDB last update time parameter doesn't exist,
# we definitely need to update the data and create a last
# update entry
needs_update = True
pdb_last_update_parameter = None
# If an update needs to happen...
if needs_update:
# 1. get the PDB info
pdb_parameter_set = ExperimentParameterSet.objects.get(
schema__namespace=getattr(
settings, 'PDB_PUBLICATION_SCHEMA_ROOT',
default_settings.PDB_PUBLICATION_SCHEMA_ROOT),
experiment=pub)
pdb = ExperimentParameter.objects.get(
name__name='pdb-id', parameterset=pdb_parameter_set)
pdb_id = pdb.string_value
# 1a. cosmetic change of case for PDB ID, if entered incorrectly
if pdb_id != pdb_id.upper():
pdb.string_value = pdb_id.upper()
pdb.save()
try:
# 2. fetch the info from pdb.org
pdb = PDBCifHelper(pdb_id)
# 3. insert all standard pdb parameters
add_if_missing(pdb_parameter_set,
'title',
string_value=pdb.get_pdb_title())
add_if_missing(pdb_parameter_set,
'url',
string_value=pdb.get_pdb_url())
try:
add_if_missing(pdb_parameter_set,
'resolution',
numerical_value=pdb.get_resolution())
except ValueError:
logger.error('PDB field "resolution" could not be set for '
'publication Id %i \n %s' %
(pub.id, traceback.format_exc()))
try:
add_if_missing(pdb_parameter_set,
'r-value',
numerical_value=pdb.get_obs_r_value())
except ValueError:
logger.error('PDB field "r-value" could not be set for '
'publication Id %i \n %s' %
(pub.id, traceback.format_exc()))
try:
add_if_missing(pdb_parameter_set,
'r-free',
numerical_value=pdb.get_free_r_value())
except ValueError:
logger.error('PDB field "r-free" could not be set for '
'publication Id %i \n %s' %
(pub.id, traceback.format_exc()))
add_if_missing(pdb_parameter_set,
'space-group',
string_value=pdb.get_spacegroup())
add_if_missing(pdb_parameter_set,
'unit-cell',
string_value=pdb.get_unit_cell())
# 4. insert sequence info (lazy checking)
pdb_seq_parameter_sets = ExperimentParameterSet.objects.filter(
schema__namespace=getattr(
settings, 'PDB_SEQUENCE_PUBLICATION_SCHEMA',
default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA),
experiment=pub)
if pdb_seq_parameter_sets.count() == 0:
# insert seqences
for seq in pdb.get_sequence_info():
seq_ps_namespace = getattr(
settings, 'PDB_SEQUENCE_PUBLICATION_SCHEMA',
default_settings.PDB_SEQUENCE_PUBLICATION_SCHEMA)
seq_parameter_set = ExperimentParameterSet(
schema=Schema.objects.get(
namespace=seq_ps_namespace),
experiment=pub)
seq_parameter_set.save()
add_if_missing(seq_parameter_set,
'organism',
string_value=seq['organism'])
add_if_missing(seq_parameter_set,
'expression-system',
string_value=seq['expression_system'])
add_if_missing(seq_parameter_set,
'sequence',
string_value=seq['sequence'])
# 5. insert/update citation info (aggressive)
ExperimentParameterSet.objects.filter(
schema__namespace=getattr(
settings, 'PDB_CITATION_PUBLICATION_SCHEMA',
default_settings.PDB_CITATION_PUBLICATION_SCHEMA),
experiment=pub).delete()
for citation in pdb.get_citations():
cit_ps_namespace = getattr(
settings, 'PDB_CITATION_PUBLICATION_SCHEMA',
default_settings.PDB_CITATION_PUBLICATION_SCHEMA)
cit_parameter_set = ExperimentParameterSet(
schema=Schema.objects.get(namespace=cit_ps_namespace),
experiment=pub)
cit_parameter_set.save()
add_if_missing(cit_parameter_set,
'title',
string_value=citation['title'])
add_if_missing(cit_parameter_set,
'authors',
string_value='; '.join(citation['authors']))
add_if_missing(cit_parameter_set,
'journal',
string_value=citation['journal'])
add_if_missing(cit_parameter_set,
'volume',
string_value=citation['volume'])
add_if_missing(cit_parameter_set,
'page-range',
string_value='-'.join([
citation['page_first'],
citation['page_last']
]))
add_if_missing(cit_parameter_set,
'doi',
string_value='http://dx.doi.org/' +
citation['doi'])
# 6. Remove the PDB embargo if set, since the update has
# occurred and therefore the PDB must have been relased.
try:
ExperimentParameter.objects.get(
name__name='pdb-embargo',
parameterset__schema__namespace=getattr(
settings, 'PUBLICATION_SCHEMA_ROOT',
default_settings.PUBLICATION_SCHEMA_ROOT)).delete(
)
except ExperimentParameter.DoesNotExist:
pass
# 7. Set the last update parameter to be now
if pdb_last_update_parameter is None:
pub_parameter_set = ExperimentParameterSet(
schema=Schema.objects.get(namespace=PUB_SCHEMA),
experiment=pub)
pub_parameter_set.save()
pdb_last_update_parameter = ExperimentParameter(
name=last_update_parameter_name,
parameterset=pub_parameter_set,
datetime_value=timezone.now())
else:
pdb_last_update_parameter.datetime_value = timezone.now()
pdb_last_update_parameter.save()
except CifFile.StarError:
# PDB is either unavailable or invalid
# (maybe notify the user somehow?)
continue
def __init__(self, parameterset=None, parentObject=None,
schema=None):
"""
instantiate new task or existing ParameterSet
:param dataset: optional parameter to instanciate task from
metadata, will be tested for completeness and copied into
new task if complete
:type dataset: Dataset
:param schema: Schema namespace
:type schema: string
"""
if parameterset:
self.parameterset = parameterset
self.schema = self.parameterset.schema
self.namespace = self.schema.namespace
if isinstance(self.parameterset, DatafileParameterSet):
self.parameters = DatafileParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatafileParameter
elif isinstance(self.parameterset, DatasetParameterSet):
self.parameters = DatasetParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatasetParameter
elif isinstance(self.parameterset, ExperimentParameterSet):
self.parameters = ExperimentParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parameterset object given.")
elif parentObject and schema:
self.namespace = schema
if isinstance(parentObject, Dataset_File):
self.parameterset = DatafileParameterSet(
schema=self.get_schema(), dataset_file=parentObject)
self.parameterset.save()
self.parameters = DatafileParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatafileParameter
elif isinstance(parentObject, Dataset):
self.parameterset = DatasetParameterSet(
schema=self.get_schema(), dataset=parentObject)
self.parameterset.save()
self.parameters = DatasetParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatasetParameter
elif isinstance(parentObject, Experiment):
self.parameterset = ExperimentParameterSet(
schema=self.get_schema(), experiment=parentObject)
self.parameterset.save()
self.parameters = ExperimentParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parent object." +
"Must be an experiment/dataset/datafile not " + str(type(parentObject)))
else:
raise TypeError("Missing arguments")
def __init__(self, parameterset=None, parentObject=None, schema=None):
"""
instantiate new task or existing ParameterSet
:param ParameterSet parameterset: optional parameter to instanciate
task from metadata, will be tested for completeness and copied into
new task if complete
:param object parentObject:
:param string schema: Schema namespace
:raises TypeError:
"""
from tardis.tardis_portal.models import DatasetParameterSet
from tardis.tardis_portal.models import DatafileParameterSet
from tardis.tardis_portal.models import ExperimentParameterSet
from tardis.tardis_portal.models import DatasetParameter
from tardis.tardis_portal.models import DatafileParameter
from tardis.tardis_portal.models import ExperimentParameter
from tardis.tardis_portal.models.parameters import ParameterSet
if issubclass(type(self), ParameterSet):
pass
elif parameterset:
self.parameterset = parameterset
self.schema = self.parameterset.schema
self.namespace = self.schema.namespace
if isinstance(self.parameterset, DatafileParameterSet):
self.parameters = DatafileParameter.objects.filter(
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatafileParameter
elif isinstance(self.parameterset, DatasetParameterSet):
self.parameters = DatasetParameter.objects.filter(
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatasetParameter
elif isinstance(self.parameterset, ExperimentParameterSet):
self.parameters = ExperimentParameter.objects.filter(
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parameterset object given.")
elif parentObject and schema:
self.namespace = schema
if isinstance(parentObject, DataFile):
self.parameterset = DatafileParameterSet(
schema=self.get_schema(), datafile=parentObject)
self.parameterset.save()
self.parameters = DatafileParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatafileParameter
elif isinstance(parentObject, Dataset):
self.parameterset = DatasetParameterSet(
schema=self.get_schema(), dataset=parentObject)
self.parameterset.save()
self.parameters = DatasetParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatasetParameter
elif isinstance(parentObject, Experiment):
self.parameterset = ExperimentParameterSet(
schema=self.get_schema(), experiment=parentObject)
self.parameterset.save()
self.parameters = ExperimentParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parent object." +
"Must be an experiment/dataset/datafile not " +
str(type(parentObject)))
else:
raise TypeError("Missing arguments")
