def add_experiment_access_user(request, experiment_id, username):
canRead = False
canWrite = False
canDelete = False
isOwner = False
if 'canRead' in request.GET:
if request.GET['canRead'] == 'true':
canRead = True
if 'canWrite' in request.GET:
if request.GET['canWrite'] == 'true':
canWrite = True
if 'canDelete' in request.GET:
if request.GET['canDelete'] == 'true':
canDelete = True
if 'isOwner' in request.GET:
if request.GET['isOwner'] == 'true':
isOwner = True
authMethod = request.GET['authMethod']
user = auth_service.getUser(authMethod, username)
if user is None or username == settings.TOKEN_USERNAME:
return HttpResponse('User %s does not exist.' % (username))
try:
experiment = Experiment.objects.get(pk=experiment_id)
except Experiment.DoesNotExist:
return HttpResponse('Experiment (id=%d) does not exist.'
% (experiment.id))
acl = ObjectACL.objects.filter(
content_type=experiment.get_ct(),
object_id=experiment.id,
pluginId=django_user,
entityId=str(user.id),
aclOwnershipType=ObjectACL.OWNER_OWNED)
if acl.count() == 0:
acl = ObjectACL(content_object=experiment,
pluginId=django_user,
entityId=str(user.id),
canRead=canRead,
canWrite=canWrite,
canDelete=canDelete,
isOwner=isOwner,
aclOwnershipType=ObjectACL.OWNER_OWNED)
acl.save()
c = {'authMethod': authMethod,
'user': user,
'user_acl': acl,
'username': username,
'experiment_id': experiment_id}
return HttpResponse(render_response_index(
request,
'tardis_portal/ajax/add_user_result.html', c))
return HttpResponse('User already has experiment access.')
def add_experiment_access_user(request, experiment_id, username):
canRead = False
canWrite = False
canDelete = False
isOwner = False
if 'canRead' in request.GET:
if request.GET['canRead'] == 'true':
canRead = True
if 'canWrite' in request.GET:
if request.GET['canWrite'] == 'true':
canWrite = True
if 'canDelete' in request.GET:
if request.GET['canDelete'] == 'true':
canDelete = True
if 'isOwner' in request.GET:
if request.GET['isOwner'] == 'true':
isOwner = True
authMethod = request.GET['authMethod']
user = auth_service.getUser(authMethod, username)
if user is None or username == settings.TOKEN_USERNAME:
return HttpResponse('User %s does not exist.' % (username))
try:
experiment = Experiment.objects.get(pk=experiment_id)
except Experiment.DoesNotExist:
return HttpResponse('Experiment (id=%d) does not exist.'
% (experiment.id))
acl = ObjectACL.objects.filter(
content_type=experiment.get_ct(),
object_id=experiment.id,
pluginId=django_user,
entityId=str(user.id),
aclOwnershipType=ObjectACL.OWNER_OWNED)
if acl.count() == 0:
acl = ObjectACL(content_object=experiment,
pluginId=django_user,
entityId=str(user.id),
canRead=canRead,
canWrite=canWrite,
canDelete=canDelete,
isOwner=isOwner,
aclOwnershipType=ObjectACL.OWNER_OWNED)
acl.save()
c = {'authMethod': authMethod,
'user': user,
'user_acl': acl,
'username': username,
'experiment_id': experiment_id}
return HttpResponse(render_response_index(
request,
'tardis_portal/ajax/add_user_result.html', c))
return HttpResponse('User already has experiment access.')
def _registerExperimentDocument(filename, created_by, expid=None,
owners=[], username=None):
'''
Register the experiment document and return the experiment id.
:param filename: path of the document to parse (METS or notMETS)
:type filename: string
:param created_by: a User instance
:type created_by: :py:class:`django.contrib.auth.models.User`
:param expid: the experiment ID to use
:type expid: int
:param owners: a list of email addresses of users who 'own' the experiment, as registered in the local db
:type owner: list
:param username: **UNUSED**
:rtype: int
'''
global current_action, experiment
current_action = "Ingest Processing"
f = open(filename)
firstline = f.readline()
f.close()
if firstline.startswith('<experiment'):
logger.debug('processing simple xml')
processExperiment = ProcessExperiment()
eid = processExperiment.process_simple(filename, created_by, expid)
else:
logger.debug('processing METS')
try:
eid = parseMets(filename, created_by, expid)
except SAXParseException:
add_status(status=RegistrationStatus.ERROR,
message="Processing METS failed: Document isn't XML, or well formed.<br> (%s)" % filename,
exception=True)
auth_key = ''
try:
auth_key = settings.DEFAULT_AUTH
except AttributeError:
add_status(status=RegistrationStatus.ERROR,
message='No default authentication for experiment ownership set (settings.DEFAULT_AUTH)')
if auth_key:
for owner in owners:
# for each PI
if owner:
user = auth_service.getUser({'pluginname': auth_key,
'id': owner})
# if exist, create ACL
if user:
logger.debug('registering owner: ' + owner)
acl = ExperimentACL(experiment=experiment,
pluginId=django_user,
entityId=str(user.id),
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True,
aclOwnershipType=ExperimentACL.OWNER_OWNED)
acl.save()
else:
# Make this a critical error
add_status(status=RegistrationStatus.ERROR,
message="Can't create ACL for experiment: no user found for owner '%s'. Auth_key: %s.: " %
(owner, auth_key,))
return experiment.id # unneeded?
def _parse_metaman(request, cleaned_data):
'''
The actual parser which is called by the register function. The
parser reads the raw metaman output (basically text
files). Metaman prints the metadata for each file in consecutive
blocks and one line for each key-value pair separated by ' : '.
Files are separated by double line-breaks.
The function contains a lot of logic which might not be documented
elsewhere. The way the metadata is parsed might not be the most
clever approach and should have been properly designed at the
first place.
:keyword request: Django HttpRequest instance
:keyword cleaned_data: cleaned form fields
'''
logger.info("New experiment request:")
for k, v in cleaned_data.iteritems():
logger.info(" %s: %s" % (k, str(v)[:100]))
# which beamline did it come from?
beamline = cleaned_data['beamline']
epn = cleaned_data['epn']
###
### I: Parse MetaMan ouput, loop over individual metadata blocks
###
# the current Datafile object (if identified)
df = None
# list of Datafile objects (which holds the datafile metadata)
files = []
extra_files = []
# that's the actual MetaMan upload
metaman = []
if 'metaman' in request.FILES:
metaman = request.FILES['metaman']
logger.debug("Metaman file '%s' uploaded. Size: %i bytes"
% (metaman.name, metaman.size))
#dump_metaman_file(metaman, 'e' + str(epn) + '.txt')
extra = None
if 'extra' in request.FILES:
# contains additional data for the mx beamline from the
# autoprocessing database
extra = request.FILES['extra']
logger.info("Extra information received. Size %i bytes" % extra.size)
dump_metaman_file(extra, 'x' + str(epn) + '.txt')
sample = None
if 'sample' in request.FILES:
sample = request.FILES['sample']
logger.info("Sample information received. Size %i bytes" % sample.size)
dump_metaman_file(sample, 's' + str(epn) + '.txt')
if not beamline in _config:
logger.error("Unknown beamline key '%s'" % beamline)
logger.error("No data will be commited to the database!")
return None
# create experiment with info from post
try:
experiment = models.Experiment.objects.get(
experimentparameterset__experimentparameter__string_value=epn,
experimentparameterset__experimentparameter__name__name='EPN')
update = True
logger.debug('experiment with epn %s alreadt exists' % epn)
logger.debug('- all parametersets will be re-created, acls will not be touched')
except models.Experiment.DoesNotExist:
experiment = models.Experiment()
update = False
experiment.url = cleaned_data['url']
experiment.title = cleaned_data['title']
experiment.institution_name = cleaned_data['institution_name']
experiment.description = cleaned_data['description']
experiment.created_by = request.user
experiment.start_time = cleaned_data['start_time']
experiment.end_time = cleaned_data['end_time']
experiment.save()
logger.debug('experiment %i saved' % experiment.id)
if update:
models.Author_Experiment.objects.filter(experiment=experiment).delete()
order = 1
for author in cleaned_data['authors'].split(' ~ '):
logger.debug('adding author %s' % author)
author_experiment = models.Author_Experiment(experiment=experiment,
author=author,
order=order)
author_experiment.save()
order += 1
# additional experiment metadata
exp_schema = \
models.Schema.objects.get(namespace__exact=_config[beamline]['expSchema'])
if update:
models.ExperimentParameterSet.objects.filter(schema=exp_schema,
experiment=experiment).delete()
exp_parameterset = models.ExperimentParameterSet(schema=exp_schema,
experiment=experiment)
exp_parameterset.save()
# IR1 -> IR, PC1 -> PC
experiment_metadata = {'beamline': re.match('[A-Z,a-z]+',
beamline).group(0),
'epn': epn}
for key, value in experiment_metadata.iteritems():
try:
exp_parameter = models.ParameterName.objects.get(schema=exp_schema,
name__iexact=key)
exp_par = models.ExperimentParameter(parameterset=exp_parameterset,
name=exp_parameter,
string_value=value)
exp_par.save()
except models.ParameterName.DoesNotExist:
logger.error('parameter %s not found in %s' % (key, exp_schema))
if sample:
# parse sample information and store it right away to keep the
# order of the parameters
sample_schema = \
models.Schema.objects.get(namespace__exact=_config[beamline]['sampleSchema'])
chemical_schema = \
models.Schema.objects.get(namespace__exact=_config[beamline]['chemicalSchema'])
if update:
models.ExperimentParameterSet.objects.filter(schema=sample_schema,
experiment=experiment).delete()
models.ExperimentParameterSet.objects.filter(schema=chemical_schema,
experiment=experiment).delete()
sample_ps = None
chemical_ps = None
for line in sample:
line = line.rstrip('\n')
if line == '':
continue
key, value = line.split(' : ')
if key == 'SampleDescription' or sample_ps is None:
sample_ps = models.ExperimentParameterSet(
schema=sample_schema,
experiment=experiment)
sample_ps.save()
chemical_ps = None
elif key == 'ChemicalName':
chemical_ps = models.ExperimentParameterSet(
schema=chemical_schema,
experiment=experiment)
chemical_ps.save()
if value == '':
continue
# Use the chemical parameterset if one is active, otherwise just
# use the current sample parameterset.
paramset = chemical_ps or sample_ps
try:
param_name = models.ParameterName.objects.get(
schema=paramset.schema,
name__iexact=key)
sample_par = models.ExperimentParameter(parameterset=paramset,
name=param_name,
string_value=value)
sample_par.save()
except models.ParameterName.DoesNotExist:
logger.error('Parameter %s not found in schema %s' % (key, paramset.schema))
# now parser the very intelligent metaman file
for line in metaman:
# remove newline
line = line.rstrip('\n')
# found a new metadata block?
# <b>/Frames/Pilatus2_1m/Rotations/Tilt_SiCNTPDMScom20_CR_0001.tif</b>:
if line[0:3] == '<b>' and line[-5:] == '</b>:':
path = line[4:-5]
# Do we accept this particular file type?
if _acceptFile(path, beamline):
# create new Datafile object and add it to the list
df = Datafile(path)
files += [df]
elif line == '':
# empty line indicates the end of current file metadata block
df = None
elif df is not None:
# anything else is metadata
try:
token = line.split(' : ', 1)
if len(token) > 1:
key, value = token[0], token[1]
# dump preview images immidiately on disk
if key == 'previewImage':
dirname = experiment.get_or_create_directory()
filename = str(uuid())
filepath = os.path.join(dirname, filename)
modulo = len(value) % 4
if modulo:
value += (4 - modulo) * '='
f = open(filepath, 'w')
f.write(b64decode(value))
f.close()
value = filename
df[key] = value
except:
# something went wrong?
logger.exception(line)
if extra:
# process extra information (only MX atm)
for line in extra:
# remove newline
line = line.rstrip('\n')
if line[0:3] == '<b>' and line[-5:] == '</b>:':
path = line[4:-5]
# we accept all information
df = Datafile(path)
extra_files += [df]
elif line == '':
# empty line indicates the end of current file metadata block
df = None
elif df is not None:
# anything else is metadata
token = line.split(' : ', 1)
if len(token) > 1:
# remove all whitespaces!
try:
df[token[0]] = token[1]
except:
logger.error(line)
logger.debug('Parsing done')
###
### II: Build datasets from files: At this point all the metadata
### (apart from possible images) is memory but this is still
### better than doing it in a PHP script ...
###
# holds files associated to a particular dataset
datasets = {}
# holds metadata information about the dataset
metadata = {}
# loop over datafiles
for df in files:
if not cfg_option(beamline, 'ingest_all', False) and not df.hasMetadata():
continue
# work out the dataset name
dsName = _getDatasetName(df, beamline)
if not _isDatasetMetadata(df, beamline):
# usual datafile metadata
if not dsName in datasets:
datasets[dsName] = [df]
else:
datasets[dsName].append(df)
else:
# this file holds metadata about the dataset!
metadata[dsName] = df
logger.debug('Grouping done')
###
### III: Ingest into database
###
# loop over datasets
if update:
models.Dataset.objects.filter(experiment=experiment).delete()
for dsName, ds in datasets.items():
dataset = models.Dataset(experiment=experiment,
description=dsName.replace('Frames/', ''))
dataset.save()
# does this dataset have any metadata?
if dsName in metadata.keys():
ds_schema = models.Schema.objects.get(namespace__exact=_config[beamline]['dsSchema'])
ds_parameterset = models.DatasetParameterSet(schema=ds_schema,
dataset=dataset)
ds_parameterset.save()
_save_parameters(ds_schema, ds_parameterset, metadata[dsName].data)
# is there extra metadata (MX)?
if extra_files:
extra_schema = models.Schema.objects.get(namespace__exact=_config[beamline]['extraSchema'])
for df in extra_files:
if df.name.endswith(dsName):
ds_parameterset = models.DatasetParameterSet(schema=extra_schema,
dataset=dataset)
ds_parameterset.save()
_save_parameters(extra_schema, ds_parameterset, df.data)
# loop over associated files
df_schema = \
models.Schema.objects.get(namespace__exact=_config[beamline]['dfSchema'])
for df in ds:
dataset_file = models.Dataset_File(dataset=dataset,
filename=os.path.basename(df.name),
url='vbl://' + df.name,
size=df.getSize(),
protocol='vbl')
dataset_file.save()
# loop over file meta-data
if df.hasMetadata():
df_parameterset = models.DatafileParameterSet(schema=df_schema,
dataset_file=dataset_file)
df_parameterset.save()
_save_parameters(df_schema, df_parameterset, df.data)
logger.debug('Ingestion done')
###
### IV: Setup permissions (for users and groups)
###
if update:
logger.info('=== updating experiment %i done ' % experiment.id)
return experiment.id
owners = (cleaned_data['experiment_owner']).split(' ')
for owner in owners:
if owner == '':
continue
logger.debug('looking for owner %s' % owner)
# find corresponding user
user = auth_service.getUser(authMethod=vbl_auth_key, user_id=owner)
if user is None:
logger.debug('No VBL user matching %s' % owner)
continue
logger.debug('registering user %s for owner %s' % (user.username, owner))
acl = models.ExperimentACL(experiment=experiment,
pluginId=django_user,
entityId=str(user.id),
isOwner=True,
canRead=True,
canWrite=False,
canDelete=False,
aclOwnershipType=models.ExperimentACL.OWNER_OWNED)
acl.save()
beamline_group = _config[beamline]['beamline_group']
group, created = Group.objects.get_or_create(name=beamline_group)
if created:
logger.debug('registering new group: %s' % group.name)
else:
logger.debug('registering existing group: %s' % group.name)
# beamline group
acl = models.ExperimentACL(experiment=experiment,
pluginId=django_group,
entityId=str(group.id),
canRead=True,
aclOwnershipType=models.ExperimentACL.SYSTEM_OWNED)
acl.save()
# create vbl group
acl = models.ExperimentACL(experiment=experiment,
pluginId='vbl_group',
entityId=cleaned_data['epn'],
canRead=True,
aclOwnershipType=models.ExperimentACL.SYSTEM_OWNED)
acl.save()
# finally, always add acl for admin group
group, created = Group.objects.get_or_create(name='admin')
acl = models.ExperimentACL(experiment=experiment,
pluginId=django_group,
entityId=str(group.id),
isOwner=True,
canRead=True,
aclOwnershipType=models.ExperimentACL.SYSTEM_OWNED)
acl.save()
logger.info('=== processing experiment %i done ' % experiment.id)
return experiment.id
def _registerExperimentDocument(filename, created_by, expid=None,
owners=[], username=None):
'''
Register the experiment document and return the experiment id.
:param filename: path of the document to parse (METS or notMETS)
:type filename: string
:param created_by: a User instance
:type created_by: :py:class:`django.contrib.auth.models.User`
:param expid: the experiment ID to use
:type expid: int
:param owners: a list of owners
:type owner: list
:param username: **UNUSED**
:rtype: int
'''
f = open(filename)
firstline = f.readline()
f.close()
sync_root = ''
if firstline.startswith('<experiment'):
logger.debug('processing simple xml')
processExperiment = ProcessExperiment()
eid, sync_root = processExperiment.process_simple(filename,
created_by,
expid)
else:
logger.debug('processing METS')
eid, sync_root = parseMets(filename, created_by, expid)
auth_key = ''
try:
auth_key = settings.DEFAULT_AUTH
except AttributeError:
logger.error('no default authentication for experiment' +
' ownership set (settings.DEFAULT_AUTH)')
force_user_create = False
try:
force_user_create = settings.DEFAULT_AUTH_FORCE_USER_CREATE
except AttributeError:
pass
if auth_key:
for owner in owners:
# for each PI
if not owner:
continue
owner_username = None
if '@' in owner:
owner_username = auth_service.getUsernameByEmail(auth_key,
owner)
if not owner_username:
owner_username = owner
owner_user = auth_service.getUser(auth_key, owner_username,
force_user_create=force_user_create)
# if exist, create ACL
if owner_user:
#logger.debug('registering owner: ' + owner)
e = Experiment.objects.get(pk=eid)
acl = ExperimentACL(experiment=e,
pluginId=django_user,
entityId=str(owner_user.id),
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True,
aclOwnershipType=ExperimentACL.OWNER_OWNED)
acl.save()
return (eid, sync_root)
def _registerExperimentDocument(filename,
created_by,
expid=None,
owners=[],
username=None):
'''
Register the experiment document and return the experiment id.
:param filename: path of the document to parse (METS or notMETS)
:type filename: string
:param created_by: a User instance
:type created_by: :py:class:`django.contrib.auth.models.User`
:param expid: the experiment ID to use
:type expid: int
:param owners: a list of owners
:type owner: list
:param username: **UNUSED**
:rtype: int
'''
f = open(filename)
firstline = f.readline()
f.close()
sync_root = ''
if firstline.startswith('<experiment'):
logger.debug('processing simple xml')
processExperiment = ProcessExperiment()
eid, sync_root = processExperiment.process_simple(
filename, created_by, expid)
else:
logger.debug('processing METS')
eid, sync_root = parseMets(filename, created_by, expid)
auth_key = ''
try:
auth_key = settings.DEFAULT_AUTH
except AttributeError:
logger.error('no default authentication for experiment' +
' ownership set (settings.DEFAULT_AUTH)')
force_user_create = False
try:
force_user_create = settings.DEFAULT_AUTH_FORCE_USER_CREATE
except AttributeError:
pass
if auth_key:
for owner in owners:
# for each PI
if not owner:
continue
owner_username = None
if '@' in owner:
owner_username = auth_service.getUsernameByEmail(
auth_key, owner)
if not owner_username:
owner_username = owner
owner_user = auth_service.getUser(
auth_key, owner_username, force_user_create=force_user_create)
# if exist, create ACL
if owner_user:
#logger.debug('registering owner: ' + owner)
e = Experiment.objects.get(pk=eid)
acl = ExperimentACL(experiment=e,
pluginId=django_user,
entityId=str(owner_user.id),
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True,
aclOwnershipType=ExperimentACL.OWNER_OWNED)
acl.save()
return (eid, sync_root)
def _registerExperimentDocument(filename, created_by, expid=None,
owners=[], username=None):
'''
Register the experiment document and return the experiment id.
:param filename: path of the document to parse (METS or notMETS)
:type filename: string
:param created_by: a User instance
:type created_by: :py:class:`django.contrib.auth.models.User`
:param expid: the experiment ID to use
:type expid: int
:param owners: a list of owners
:type owner: list
:param username: **UNUSED**
:rtype: int
'''
f = open(filename)
firstline = f.readline()
f.close()
if firstline.startswith('<experiment'):
logger.debug('processing simple xml')
processExperiment = ProcessExperiment()
eid = processExperiment.process_simple(filename, created_by, expid)
else:
logger.debug('processing METS')
eid = parseMets(filename, created_by, expid)
# Create a DatasetWrapper for each Dataset
experiment = Experiment.objects.get(pk=eid)
sample = Sample(experiment=experiment, name="Default Sample",
description="A default sample for %s" % experiment.title)
sample.save()
_create_wrappers_for_datasets(sample, experiment)
# Create a Project to wraps the experiment, then create a Sample that
# points to the experiment
project = Project(experiment=experiment)
project.save()
auth_key = ''
try:
auth_key = settings.DEFAULT_AUTH
except AttributeError:
logger.error('no default authentication for experiment ownership set (settings.DEFAULT_AUTH)')
force_user_create = False
try:
force_user_create = settings.DEFAULT_AUTH_FORCE_USER_CREATE
except AttributeError:
pass
if auth_key:
for owner in owners:
logger.debug('** Owner : %s' %owner)
# for each PI
if not owner:
continue
owner_username = None
if '@' in owner:
logger.debug('** Email as username **')
owner_username = auth_service.getUsernameByEmail(auth_key,
owner)
if not owner_username:
logger.debug('** No existing user!! **')
owner_username = owner
owner_user = auth_service.getUser(auth_key, owner_username,
force_user_create=force_user_create)
if owner_user:
# if exist, create ACL
logger.debug('registering owner: ' + owner)
e = Experiment.objects.get(pk=eid)
acl = ExperimentACL(experiment=e,
pluginId=django_user,
entityId=str(owner_user.id),
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True,
aclOwnershipType=ExperimentACL.OWNER_OWNED)
acl.save()
# Also update email
if '@' in owner:
owner_user.email = owner
owner_user.save()
return eid