def add_experiment_access_user(request, experiment_id, username): canRead = False canWrite = False canDelete = False isOwner = False if 'canRead' in request.GET: if request.GET['canRead'] == 'true': canRead = True if 'canWrite' in request.GET: if request.GET['canWrite'] == 'true': canWrite = True if 'canDelete' in request.GET: if request.GET['canDelete'] == 'true': canDelete = True if 'isOwner' in request.GET: if request.GET['isOwner'] == 'true': isOwner = True authMethod = request.GET['authMethod'] user = auth_service.getUser(authMethod, username) if user is None or username == settings.TOKEN_USERNAME: return HttpResponse('User %s does not exist.' % (username)) try: experiment = Experiment.objects.get(pk=experiment_id) except Experiment.DoesNotExist: return HttpResponse('Experiment (id=%d) does not exist.' % (experiment.id)) acl = ObjectACL.objects.filter( content_type=experiment.get_ct(), object_id=experiment.id, pluginId=django_user, entityId=str(user.id), aclOwnershipType=ObjectACL.OWNER_OWNED) if acl.count() == 0: acl = ObjectACL(content_object=experiment, pluginId=django_user, entityId=str(user.id), canRead=canRead, canWrite=canWrite, canDelete=canDelete, isOwner=isOwner, aclOwnershipType=ObjectACL.OWNER_OWNED) acl.save() c = {'authMethod': authMethod, 'user': user, 'user_acl': acl, 'username': username, 'experiment_id': experiment_id} return HttpResponse(render_response_index( request, 'tardis_portal/ajax/add_user_result.html', c)) return HttpResponse('User already has experiment access.')
def _registerExperimentDocument(filename, created_by, expid=None, owners=[], username=None): ''' Register the experiment document and return the experiment id. :param filename: path of the document to parse (METS or notMETS) :type filename: string :param created_by: a User instance :type created_by: :py:class:`django.contrib.auth.models.User` :param expid: the experiment ID to use :type expid: int :param owners: a list of email addresses of users who 'own' the experiment, as registered in the local db :type owner: list :param username: **UNUSED** :rtype: int ''' global current_action, experiment current_action = "Ingest Processing" f = open(filename) firstline = f.readline() f.close() if firstline.startswith('<experiment'): logger.debug('processing simple xml') processExperiment = ProcessExperiment() eid = processExperiment.process_simple(filename, created_by, expid) else: logger.debug('processing METS') try: eid = parseMets(filename, created_by, expid) except SAXParseException: add_status(status=RegistrationStatus.ERROR, message="Processing METS failed: Document isn't XML, or well formed.<br> (%s)" % filename, exception=True) auth_key = '' try: auth_key = settings.DEFAULT_AUTH except AttributeError: add_status(status=RegistrationStatus.ERROR, message='No default authentication for experiment ownership set (settings.DEFAULT_AUTH)') if auth_key: for owner in owners: # for each PI if owner: user = auth_service.getUser({'pluginname': auth_key, 'id': owner}) # if exist, create ACL if user: logger.debug('registering owner: ' + owner) acl = ExperimentACL(experiment=experiment, pluginId=django_user, entityId=str(user.id), canRead=True, canWrite=True, canDelete=True, isOwner=True, aclOwnershipType=ExperimentACL.OWNER_OWNED) acl.save() else: # Make this a critical error add_status(status=RegistrationStatus.ERROR, message="Can't create ACL for experiment: no user found for owner '%s'. Auth_key: %s.: " % (owner, auth_key,)) return experiment.id # unneeded?
def _parse_metaman(request, cleaned_data): ''' The actual parser which is called by the register function. The parser reads the raw metaman output (basically text files). Metaman prints the metadata for each file in consecutive blocks and one line for each key-value pair separated by ' : '. Files are separated by double line-breaks. The function contains a lot of logic which might not be documented elsewhere. The way the metadata is parsed might not be the most clever approach and should have been properly designed at the first place. :keyword request: Django HttpRequest instance :keyword cleaned_data: cleaned form fields ''' logger.info("New experiment request:") for k, v in cleaned_data.iteritems(): logger.info(" %s: %s" % (k, str(v)[:100])) # which beamline did it come from? beamline = cleaned_data['beamline'] epn = cleaned_data['epn'] ### ### I: Parse MetaMan ouput, loop over individual metadata blocks ### # the current Datafile object (if identified) df = None # list of Datafile objects (which holds the datafile metadata) files = [] extra_files = [] # that's the actual MetaMan upload metaman = [] if 'metaman' in request.FILES: metaman = request.FILES['metaman'] logger.debug("Metaman file '%s' uploaded. Size: %i bytes" % (metaman.name, metaman.size)) #dump_metaman_file(metaman, 'e' + str(epn) + '.txt') extra = None if 'extra' in request.FILES: # contains additional data for the mx beamline from the # autoprocessing database extra = request.FILES['extra'] logger.info("Extra information received. Size %i bytes" % extra.size) dump_metaman_file(extra, 'x' + str(epn) + '.txt') sample = None if 'sample' in request.FILES: sample = request.FILES['sample'] logger.info("Sample information received. Size %i bytes" % sample.size) dump_metaman_file(sample, 's' + str(epn) + '.txt') if not beamline in _config: logger.error("Unknown beamline key '%s'" % beamline) logger.error("No data will be commited to the database!") return None # create experiment with info from post try: experiment = models.Experiment.objects.get( experimentparameterset__experimentparameter__string_value=epn, experimentparameterset__experimentparameter__name__name='EPN') update = True logger.debug('experiment with epn %s alreadt exists' % epn) logger.debug('- all parametersets will be re-created, acls will not be touched') except models.Experiment.DoesNotExist: experiment = models.Experiment() update = False experiment.url = cleaned_data['url'] experiment.title = cleaned_data['title'] experiment.institution_name = cleaned_data['institution_name'] experiment.description = cleaned_data['description'] experiment.created_by = request.user experiment.start_time = cleaned_data['start_time'] experiment.end_time = cleaned_data['end_time'] experiment.save() logger.debug('experiment %i saved' % experiment.id) if update: models.Author_Experiment.objects.filter(experiment=experiment).delete() order = 1 for author in cleaned_data['authors'].split(' ~ '): logger.debug('adding author %s' % author) author_experiment = models.Author_Experiment(experiment=experiment, author=author, order=order) author_experiment.save() order += 1 # additional experiment metadata exp_schema = \ models.Schema.objects.get(namespace__exact=_config[beamline]['expSchema']) if update: models.ExperimentParameterSet.objects.filter(schema=exp_schema, experiment=experiment).delete() exp_parameterset = models.ExperimentParameterSet(schema=exp_schema, experiment=experiment) exp_parameterset.save() # IR1 -> IR, PC1 -> PC experiment_metadata = {'beamline': re.match('[A-Z,a-z]+', beamline).group(0), 'epn': epn} for key, value in experiment_metadata.iteritems(): try: exp_parameter = models.ParameterName.objects.get(schema=exp_schema, name__iexact=key) exp_par = models.ExperimentParameter(parameterset=exp_parameterset, name=exp_parameter, string_value=value) exp_par.save() except models.ParameterName.DoesNotExist: logger.error('parameter %s not found in %s' % (key, exp_schema)) if sample: # parse sample information and store it right away to keep the # order of the parameters sample_schema = \ models.Schema.objects.get(namespace__exact=_config[beamline]['sampleSchema']) chemical_schema = \ models.Schema.objects.get(namespace__exact=_config[beamline]['chemicalSchema']) if update: models.ExperimentParameterSet.objects.filter(schema=sample_schema, experiment=experiment).delete() models.ExperimentParameterSet.objects.filter(schema=chemical_schema, experiment=experiment).delete() sample_ps = None chemical_ps = None for line in sample: line = line.rstrip('\n') if line == '': continue key, value = line.split(' : ') if key == 'SampleDescription' or sample_ps is None: sample_ps = models.ExperimentParameterSet( schema=sample_schema, experiment=experiment) sample_ps.save() chemical_ps = None elif key == 'ChemicalName': chemical_ps = models.ExperimentParameterSet( schema=chemical_schema, experiment=experiment) chemical_ps.save() if value == '': continue # Use the chemical parameterset if one is active, otherwise just # use the current sample parameterset. paramset = chemical_ps or sample_ps try: param_name = models.ParameterName.objects.get( schema=paramset.schema, name__iexact=key) sample_par = models.ExperimentParameter(parameterset=paramset, name=param_name, string_value=value) sample_par.save() except models.ParameterName.DoesNotExist: logger.error('Parameter %s not found in schema %s' % (key, paramset.schema)) # now parser the very intelligent metaman file for line in metaman: # remove newline line = line.rstrip('\n') # found a new metadata block? # <b>/Frames/Pilatus2_1m/Rotations/Tilt_SiCNTPDMScom20_CR_0001.tif</b>: if line[0:3] == '<b>' and line[-5:] == '</b>:': path = line[4:-5] # Do we accept this particular file type? if _acceptFile(path, beamline): # create new Datafile object and add it to the list df = Datafile(path) files += [df] elif line == '': # empty line indicates the end of current file metadata block df = None elif df is not None: # anything else is metadata try: token = line.split(' : ', 1) if len(token) > 1: key, value = token[0], token[1] # dump preview images immidiately on disk if key == 'previewImage': dirname = experiment.get_or_create_directory() filename = str(uuid()) filepath = os.path.join(dirname, filename) modulo = len(value) % 4 if modulo: value += (4 - modulo) * '=' f = open(filepath, 'w') f.write(b64decode(value)) f.close() value = filename df[key] = value except: # something went wrong? logger.exception(line) if extra: # process extra information (only MX atm) for line in extra: # remove newline line = line.rstrip('\n') if line[0:3] == '<b>' and line[-5:] == '</b>:': path = line[4:-5] # we accept all information df = Datafile(path) extra_files += [df] elif line == '': # empty line indicates the end of current file metadata block df = None elif df is not None: # anything else is metadata token = line.split(' : ', 1) if len(token) > 1: # remove all whitespaces! try: df[token[0]] = token[1] except: logger.error(line) logger.debug('Parsing done') ### ### II: Build datasets from files: At this point all the metadata ### (apart from possible images) is memory but this is still ### better than doing it in a PHP script ... ### # holds files associated to a particular dataset datasets = {} # holds metadata information about the dataset metadata = {} # loop over datafiles for df in files: if not cfg_option(beamline, 'ingest_all', False) and not df.hasMetadata(): continue # work out the dataset name dsName = _getDatasetName(df, beamline) if not _isDatasetMetadata(df, beamline): # usual datafile metadata if not dsName in datasets: datasets[dsName] = [df] else: datasets[dsName].append(df) else: # this file holds metadata about the dataset! metadata[dsName] = df logger.debug('Grouping done') ### ### III: Ingest into database ### # loop over datasets if update: models.Dataset.objects.filter(experiment=experiment).delete() for dsName, ds in datasets.items(): dataset = models.Dataset(experiment=experiment, description=dsName.replace('Frames/', '')) dataset.save() # does this dataset have any metadata? if dsName in metadata.keys(): ds_schema = models.Schema.objects.get(namespace__exact=_config[beamline]['dsSchema']) ds_parameterset = models.DatasetParameterSet(schema=ds_schema, dataset=dataset) ds_parameterset.save() _save_parameters(ds_schema, ds_parameterset, metadata[dsName].data) # is there extra metadata (MX)? if extra_files: extra_schema = models.Schema.objects.get(namespace__exact=_config[beamline]['extraSchema']) for df in extra_files: if df.name.endswith(dsName): ds_parameterset = models.DatasetParameterSet(schema=extra_schema, dataset=dataset) ds_parameterset.save() _save_parameters(extra_schema, ds_parameterset, df.data) # loop over associated files df_schema = \ models.Schema.objects.get(namespace__exact=_config[beamline]['dfSchema']) for df in ds: dataset_file = models.Dataset_File(dataset=dataset, filename=os.path.basename(df.name), url='vbl://' + df.name, size=df.getSize(), protocol='vbl') dataset_file.save() # loop over file meta-data if df.hasMetadata(): df_parameterset = models.DatafileParameterSet(schema=df_schema, dataset_file=dataset_file) df_parameterset.save() _save_parameters(df_schema, df_parameterset, df.data) logger.debug('Ingestion done') ### ### IV: Setup permissions (for users and groups) ### if update: logger.info('=== updating experiment %i done ' % experiment.id) return experiment.id owners = (cleaned_data['experiment_owner']).split(' ') for owner in owners: if owner == '': continue logger.debug('looking for owner %s' % owner) # find corresponding user user = auth_service.getUser(authMethod=vbl_auth_key, user_id=owner) if user is None: logger.debug('No VBL user matching %s' % owner) continue logger.debug('registering user %s for owner %s' % (user.username, owner)) acl = models.ExperimentACL(experiment=experiment, pluginId=django_user, entityId=str(user.id), isOwner=True, canRead=True, canWrite=False, canDelete=False, aclOwnershipType=models.ExperimentACL.OWNER_OWNED) acl.save() beamline_group = _config[beamline]['beamline_group'] group, created = Group.objects.get_or_create(name=beamline_group) if created: logger.debug('registering new group: %s' % group.name) else: logger.debug('registering existing group: %s' % group.name) # beamline group acl = models.ExperimentACL(experiment=experiment, pluginId=django_group, entityId=str(group.id), canRead=True, aclOwnershipType=models.ExperimentACL.SYSTEM_OWNED) acl.save() # create vbl group acl = models.ExperimentACL(experiment=experiment, pluginId='vbl_group', entityId=cleaned_data['epn'], canRead=True, aclOwnershipType=models.ExperimentACL.SYSTEM_OWNED) acl.save() # finally, always add acl for admin group group, created = Group.objects.get_or_create(name='admin') acl = models.ExperimentACL(experiment=experiment, pluginId=django_group, entityId=str(group.id), isOwner=True, canRead=True, aclOwnershipType=models.ExperimentACL.SYSTEM_OWNED) acl.save() logger.info('=== processing experiment %i done ' % experiment.id) return experiment.id
def _registerExperimentDocument(filename, created_by, expid=None, owners=[], username=None): ''' Register the experiment document and return the experiment id. :param filename: path of the document to parse (METS or notMETS) :type filename: string :param created_by: a User instance :type created_by: :py:class:`django.contrib.auth.models.User` :param expid: the experiment ID to use :type expid: int :param owners: a list of owners :type owner: list :param username: **UNUSED** :rtype: int ''' f = open(filename) firstline = f.readline() f.close() sync_root = '' if firstline.startswith('<experiment'): logger.debug('processing simple xml') processExperiment = ProcessExperiment() eid, sync_root = processExperiment.process_simple(filename, created_by, expid) else: logger.debug('processing METS') eid, sync_root = parseMets(filename, created_by, expid) auth_key = '' try: auth_key = settings.DEFAULT_AUTH except AttributeError: logger.error('no default authentication for experiment' + ' ownership set (settings.DEFAULT_AUTH)') force_user_create = False try: force_user_create = settings.DEFAULT_AUTH_FORCE_USER_CREATE except AttributeError: pass if auth_key: for owner in owners: # for each PI if not owner: continue owner_username = None if '@' in owner: owner_username = auth_service.getUsernameByEmail(auth_key, owner) if not owner_username: owner_username = owner owner_user = auth_service.getUser(auth_key, owner_username, force_user_create=force_user_create) # if exist, create ACL if owner_user: #logger.debug('registering owner: ' + owner) e = Experiment.objects.get(pk=eid) acl = ExperimentACL(experiment=e, pluginId=django_user, entityId=str(owner_user.id), canRead=True, canWrite=True, canDelete=True, isOwner=True, aclOwnershipType=ExperimentACL.OWNER_OWNED) acl.save() return (eid, sync_root)
def _registerExperimentDocument(filename, created_by, expid=None, owners=[], username=None): ''' Register the experiment document and return the experiment id. :param filename: path of the document to parse (METS or notMETS) :type filename: string :param created_by: a User instance :type created_by: :py:class:`django.contrib.auth.models.User` :param expid: the experiment ID to use :type expid: int :param owners: a list of owners :type owner: list :param username: **UNUSED** :rtype: int ''' f = open(filename) firstline = f.readline() f.close() sync_root = '' if firstline.startswith('<experiment'): logger.debug('processing simple xml') processExperiment = ProcessExperiment() eid, sync_root = processExperiment.process_simple( filename, created_by, expid) else: logger.debug('processing METS') eid, sync_root = parseMets(filename, created_by, expid) auth_key = '' try: auth_key = settings.DEFAULT_AUTH except AttributeError: logger.error('no default authentication for experiment' + ' ownership set (settings.DEFAULT_AUTH)') force_user_create = False try: force_user_create = settings.DEFAULT_AUTH_FORCE_USER_CREATE except AttributeError: pass if auth_key: for owner in owners: # for each PI if not owner: continue owner_username = None if '@' in owner: owner_username = auth_service.getUsernameByEmail( auth_key, owner) if not owner_username: owner_username = owner owner_user = auth_service.getUser( auth_key, owner_username, force_user_create=force_user_create) # if exist, create ACL if owner_user: #logger.debug('registering owner: ' + owner) e = Experiment.objects.get(pk=eid) acl = ExperimentACL(experiment=e, pluginId=django_user, entityId=str(owner_user.id), canRead=True, canWrite=True, canDelete=True, isOwner=True, aclOwnershipType=ExperimentACL.OWNER_OWNED) acl.save() return (eid, sync_root)
def _registerExperimentDocument(filename, created_by, expid=None, owners=[], username=None): ''' Register the experiment document and return the experiment id. :param filename: path of the document to parse (METS or notMETS) :type filename: string :param created_by: a User instance :type created_by: :py:class:`django.contrib.auth.models.User` :param expid: the experiment ID to use :type expid: int :param owners: a list of owners :type owner: list :param username: **UNUSED** :rtype: int ''' f = open(filename) firstline = f.readline() f.close() if firstline.startswith('<experiment'): logger.debug('processing simple xml') processExperiment = ProcessExperiment() eid = processExperiment.process_simple(filename, created_by, expid) else: logger.debug('processing METS') eid = parseMets(filename, created_by, expid) # Create a DatasetWrapper for each Dataset experiment = Experiment.objects.get(pk=eid) sample = Sample(experiment=experiment, name="Default Sample", description="A default sample for %s" % experiment.title) sample.save() _create_wrappers_for_datasets(sample, experiment) # Create a Project to wraps the experiment, then create a Sample that # points to the experiment project = Project(experiment=experiment) project.save() auth_key = '' try: auth_key = settings.DEFAULT_AUTH except AttributeError: logger.error('no default authentication for experiment ownership set (settings.DEFAULT_AUTH)') force_user_create = False try: force_user_create = settings.DEFAULT_AUTH_FORCE_USER_CREATE except AttributeError: pass if auth_key: for owner in owners: logger.debug('** Owner : %s' %owner) # for each PI if not owner: continue owner_username = None if '@' in owner: logger.debug('** Email as username **') owner_username = auth_service.getUsernameByEmail(auth_key, owner) if not owner_username: logger.debug('** No existing user!! **') owner_username = owner owner_user = auth_service.getUser(auth_key, owner_username, force_user_create=force_user_create) if owner_user: # if exist, create ACL logger.debug('registering owner: ' + owner) e = Experiment.objects.get(pk=eid) acl = ExperimentACL(experiment=e, pluginId=django_user, entityId=str(owner_user.id), canRead=True, canWrite=True, canDelete=True, isOwner=True, aclOwnershipType=ExperimentACL.OWNER_OWNED) acl.save() # Also update email if '@' in owner: owner_user.email = owner owner_user.save() return eid