def _update_standard_metadata(file_, data, changed): """Update the standard metadata attributes of the input file. :param file_: a file model object to be updated. :param dict data: the data used to update the file model. :param bool changed: indicates whether the file has been changed. :returns: a tuple whose first element is the file model and whose second is the boolean ``changed``. """ changed = file_.set_attr('description', h.normalize(data['description']), changed) changed = file_.set_attr('utterance_type', h.normalize(data['utterance_type']), changed) changed = file_.set_attr('date_elicited', data['date_elicited'], changed) changed = file_.set_attr('elicitor', data['elicitor'], changed) changed = file_.set_attr('speaker', data['speaker'], changed) # Many-to-Many Data: tags & forms # Update only if the user has made changes. forms_to_add = [f for f in data['forms'] if f] tags_to_add = [t for t in data['tags'] if t] if set(forms_to_add) != set(file_.forms): file_.forms = forms_to_add changed = True # Cause the entire file to be tagged as restricted if any one of its # forms are so tagged. tags = [f.tags for f in file_.forms] tags = [tag for tag_list in tags for tag in tag_list] restricted_tags = [tag for tag in tags if tag.name == 'restricted'] if restricted_tags: restricted_tag = restricted_tags[0] if restricted_tag not in tags_to_add: tags_to_add.append(restricted_tag) if set(tags_to_add) != set(file_.tags): file_.tags = tags_to_add changed = True return file_, changed
def parse(self): """Parse the input word transcriptions using the morphological parser with id=``id``. :param str id: the ``id`` value of the morphological parser that will be used. :Request body: JSON object of the form ``{'transcriptions': [t1, t2, ...]}``. :returns: if the morphological parser exists and foma is installed, a JSON object of the form ``{t1: p1, t2: p2, ...}`` where ``t1`` and ``t2`` are transcriptions of words from the request body and ``p1`` and ``p2`` are the most probable morphological parsers of t1 and t2. """ morphparser, id_ = self._model_from_id(eager=True) LOGGER.info('Attempting to call parse against morphological parser %d', id_) if not morphparser: self.request.response.status_int = 404 msg = 'There is no morphological parser with id {}'.format(id_) LOGGER.warning(msg) return {'error': msg} if not h.foma_installed(): self.request.response.status_int = 400 msg = 'Foma and flookup are not installed.' LOGGER.warning(msg) return {'error': msg} try: inputs = json.loads(self.request.body.decode(self.request.charset)) morphparser.cache = Cache(morphparser, self.request.registry.settings, session_getter) LOGGER.warning([ h.normalize(w) for w in TranscriptionsSchema.to_python(inputs) ['transcriptions'] ]) parses = morphparser.parse([ h.normalize(w) for w in TranscriptionsSchema.to_python(inputs) ['transcriptions'] ]) # TODO: allow for a param which causes the candidates to be # returned as well as/instead of only the most probable parse # candidate. LOGGER.info('Called parse against morphological parser %d', id_) return { transcription: parse for transcription, (parse, candidates) in parses.items() } except ValueError: self.request.response.status_int = 400 LOGGER.warning(oldc.JSONDecodeErrorResponse) return oldc.JSONDecodeErrorResponse except Invalid as error: self.request.response.status_int = 400 errors = error.unpack_errors() LOGGER.warning(errors) return {'errors': errors} except Exception as error: self.request.response.status_int = 400 msg = 'Parse request raised an error.' LOGGER.warning(msg, exc_info=True) return {'error': msg}
def _get_user_data(self, data): return { 'name': h.normalize(data['name']), 'orthography': h.normalize(data['orthography']), 'lowercase': data['lowercase'], 'initial_glottal_stops': data['initial_glottal_stops'] }
def _get_user_data(self, data): return { 'name': h.normalize(data['name']), 'description': h.normalize(data['description']), 'phonology': data['phonology'], 'morphology': data['morphology'], 'language_model': data['language_model'] }
def _get_user_data(self, data): """User-provided data for creating a corpus.""" return { 'name': h.normalize(data['name']), 'description': h.normalize(data['description']), 'content': data['content'], 'form_search': data['form_search'], 'forms': data['forms'], 'tags': data['tags'] }
def _get_user_data(self, data): result = { 'name': h.normalize(data['name']), 'heading': h.normalize(data['heading']), 'markup_language': data['markup_language'], 'content': h.normalize(data['content']) } result['html'] = h.get_HTML_from_contents(result['content'], result['markup_language']) return result
def _get_user_data(self, data): result = { 'first_name': h.normalize(data['first_name']), 'last_name': h.normalize(data['last_name']), 'dialect': h.normalize(data['dialect']), 'page_content': h.normalize(data['page_content']), 'markup_language': h.normalize(data['markup_language']) } result['html'] = h.get_HTML_from_contents(result['page_content'], result['markup_language']) return result
def _get_user_data(self, data): return { 'name': h.normalize(data['name']), 'description': h.normalize(data['description']), 'vocabulary_morphology': data['vocabulary_morphology'], 'corpus': data['corpus'], 'toolkit': data['toolkit'], 'order': data['order'], 'smoothing': data['smoothing'], 'categorial': data['categorial'] }
def _create_base64_file(self, data): """Create a local file using data from a ``Content-Type: application/json`` request. :param dict data: the data to create the file model. :param str data['base64_encoded_file']: Base64-encoded file data. :returns: an SQLAlchemy model object representing the file. """ data[ 'MIME_type'] = '' # during validation, the schema will set a proper # value based on the base64_encoded_file or # filename attribute schema = FileCreateWithBase64EncodedFiledataSchema() state = SchemaState(full_dict=data, db=self.db, logged_in_user=self.logged_in_user) data = schema.to_python(data, state) file_ = File(MIME_type=data['MIME_type'], filename=h.normalize(data['filename'])) file_ = self._add_standard_metadata(file_, data) # Write the file to disk (making sure it's unique and thereby potentially) # modifying file.filename; and calculate file.size. # base64-decoded during validation file_data = data['base64_encoded_file'] files_path = h.get_old_directory_path('files', self.request.registry.settings) file_path = os.path.join(files_path, file_.filename) file_object, file_path = _get_unique_file_path(file_path) file_.filename = os.path.split(file_path)[-1] file_.name = file_.filename file_object.write(file_data) file_object.close() file_data = None file_.size = os.path.getsize(file_path) file_ = _restrict_file_by_forms(file_) return file_
def _create_subinterval_referencing_file(self, data): """Create a subinterval-referencing file. :param dict data: the data to create the file model. :param int data['parent_file']: the ``id`` value of an audio/video file model. :param float/int data['start']: the start of the interval in seconds. :param float/int data['end']: the end of the interval in seconds. :returns: an SQLAlchemy model object representing the file. A value for ``data['name']`` may also be supplied. """ data['name'] = data.get('name') or '' schema = FileSubintervalReferencingSchema() state = SchemaState(full_dict=data, db=self.db, logged_in_user=self.logged_in_user) data = schema.to_python(data, state) # Data unique to referencing subinterval files file_ = File( parent_file=data['parent_file'], # Name defaults to the parent file's filename if nothing provided by # user name=h.normalize(data['name']) or data['parent_file'].filename, start=data['start'], end=data['end'], MIME_type=data['parent_file'].MIME_type) file_ = self._add_standard_metadata(file_, data) file_ = _restrict_file_by_forms(file_) return file_
def _update_subinterval_referencing_file(self, file_): """Update a subinterval-referencing file model. :param file_: a file model object to update. :param request.body: a JSON object containing the data for updating the file. :returns: the file model or, if the file has not been updated, ``False``. """ changed = False schema = FileSubintervalReferencingSchema() data = json.loads(self.request.body.decode(self.request.charset)) data['name'] = data.get('name') or '' state = SchemaState(full_dict=data, db=self.db, logged_in_user=self.logged_in_user) data = schema.to_python(data, state) # Data unique to referencing subinterval files changed = file_.set_attr('parent_file', data['parent_file'], changed) changed = file_.set_attr( 'name', (h.normalize(data['name']) or file_.parent_file.filename), changed) changed = file_.set_attr('start', data['start'], changed) changed = file_.set_attr('end', data['end'], changed) file_, changed = _update_standard_metadata(file_, data, changed) if changed: file_.datetime_modified = datetime.datetime.utcnow() return file_ return changed
def _get_create_data(self, data): create_data = self._get_user_data(data) create_data['salt'] = str(h.generate_salt()) create_data['password'] = str( h.encrypt_password(data['password'], create_data['salt'].encode('utf8'))) create_data['username'] = h.normalize(data['username']) create_data['datetime_modified'] = datetime.datetime.utcnow() return create_data
def _get_user_data(self, data): return { 'name': h.normalize(data['name']), 'description': h.normalize(data['description']), 'lexicon_corpus': data['lexicon_corpus'], 'rules_corpus': data['rules_corpus'], 'script_type': data['script_type'], 'extract_morphemes_from_rules_corpus': data['extract_morphemes_from_rules_corpus'], 'rules': data['rules'], 'rich_upper': data['rich_upper'], 'rich_lower': data['rich_lower'], 'include_unknowns': data['include_unknowns'] }
def applydown(self): """Apply-down (i.e., phonologize) the input in the request body using a phonology. :URL: ``PUT /phonologies/applydown/id`` (or ``PUT /phonologies/phonologize/id``) :param str id: the ``id`` value of the phonology that will be used. :Request body: JSON object of the form ``{'transcriptions': [t1, t2, ...]}``. :returns: if the phonology exists and foma is installed, a JSON object of the form ``{t1: [p1t1, p2t1, ...], ...}`` where ``t1`` is a transcription from the request body and ``p1t1``, ``p2t1``, etc. are phonologized outputs of ``t1``. """ phonology, id_ = self._model_from_id(eager=True) LOGGER.info( 'Attempting to call apply down on the compiled phonology' ' %d', id_) if not phonology: self.request.response.status_int = 404 msg = 'There is no phonology with id {}'.format(id_) LOGGER.warning(msg) return {'error': msg} if not h.foma_installed(): self.request.response.status_int = 400 msg = 'Foma and flookup are not installed.' LOGGER.warning(msg) return {'error': msg} binary_path = phonology.get_file_path('binary') if not os.path.isfile(binary_path): self.request.response.status_int = 400 msg = 'Phonology {} has not been compiled yet.'.format( phonology.id) LOGGER.warning(msg) return {'error': msg} try: inputs = json.loads(self.request.body.decode(self.request.charset)) inputs = MorphophonemicTranscriptionsSchema.to_python(inputs) inputs = [h.normalize(i) for i in inputs['transcriptions']] ret = phonology.applydown(inputs) LOGGER.info('Called apply down on the compiled phonology %d', id_) return ret except ValueError: self.request.response.status_int = 400 LOGGER.warning(oldc.JSONDecodeErrorResponse) return oldc.JSONDecodeErrorResponse except Invalid as error: self.request.response.status_int = 400 errors = error.unpack_errors() LOGGER.warning(errors) return {'errors': errors}
def _create_new_resource(self, data, collections_referenced): """Create a new collection. :param dict data: the collection to be created. :returns: an SQLAlchemy model object representing the collection. """ collection = Collection() collection.UUID = str(uuid4()) # User-inputted string data collection.title = h.normalize(data['title']) collection.type = h.normalize(data['type']) collection.url = h.normalize(data['url']) collection.description = h.normalize(data['description']) collection.markup_language = h.normalize(data['markup_language']) collection.contents = h.normalize(data['contents']) collection.contents_unpacked = h.normalize(data['contents_unpacked']) collection.html = h.get_HTML_from_contents( collection.contents_unpacked, collection.markup_language) # User-inputted date: date_elicited collection.date_elicited = data['date_elicited'] # Many-to-One if data['elicitor']: collection.elicitor = data['elicitor'] if data['speaker']: collection.speaker = data['speaker'] if data['source']: collection.source = data['source'] # Many-to-Many: tags, files & forms collection.tags = [t for t in data['tags'] if t] collection.files = [f for f in data['files'] if f] collection.forms = [f for f in data['forms'] if f] # Restrict the entire collection if it is associated to restricted forms or # files or if it references a restricted collection in its contents field. immediately_referenced_collections = \ _get_collections_referenced_in_contents( collection, collections_referenced) tags = [ f.tags for f in collection.files + collection.forms + immediately_referenced_collections ] tags = [tag for tag_list in tags for tag in tag_list] restricted_tags = [tag for tag in tags if tag.name == 'restricted'] if restricted_tags: restricted_tag = restricted_tags[0] if restricted_tag not in collection.tags: collection.tags.append(restricted_tag) # OLD-generated Data now = datetime.datetime.utcnow() collection.datetime_entered = now collection.datetime_modified = now collection.enterer = collection.modifier = self.logged_in_user return collection
def apply(self, direction): """Call foma apply in the direction of ``direction`` on the input in the request body using a morphology. :param str id: the ``id`` value of the morphology that will be used. :param str direction: the direction of foma application. :Request body: JSON object of the form ``{'transcriptions': [t1, t2, ...]}``. :returns: if the morphology exists and foma is installed, a JSON object of the form ``{t1: [p1t1, p2t1, ...], ...}`` where ``t1`` is a transcription from the request body and ``p1t1``, ``p2t1``, etc. are outputs of ``t1`` after apply up/down. """ morphology, id_ = self._model_from_id(eager=True) if not morphology: self.request.response.status_int = 404 msg = 'There is no morphology with id {}'.format(id_) LOGGER.warning(msg) return {'error': msg} if not h.foma_installed(): self.request.response.status_int = 400 msg = 'Foma and flookup are not installed.' LOGGER.warning(msg) return {'error': msg} morphology_binary_path = morphology.get_file_path('binary') if not os.path.isfile(morphology_binary_path): self.request.response.status_int = 400 msg = 'Morphology {} has not been compiled yet.'.format( morphology.id) LOGGER.warning(msg) return {'error': msg} try: inputs = json.loads(self.request.body.decode(self.request.charset)) inputs = MorphemeSequencesSchema.to_python(inputs) inputs = [h.normalize(i) for i in inputs['morpheme_sequences']] ret = morphology.apply(direction, inputs) LOGGER.info('Completed apply call against morphology' ' %d.', id_) return ret except ValueError: self.request.response.status_int = 400 LOGGER.warning(oldc.JSONDecodeErrorResponse) return oldc.JSONDecodeErrorResponse except Invalid as error: self.request.response.status_int = 400 errors = error.unpack_errors() LOGGER.warning(errors) return {'errors': errors}
def _get_user_data(self, data): result = { 'first_name': h.normalize(data['first_name']), 'last_name': h.normalize(data['last_name']), 'email': h.normalize(data['email']), 'affiliation': h.normalize(data['affiliation']), 'role': h.normalize(data['role']), 'markup_language': h.normalize(data['markup_language']), 'page_content': h.normalize(data['page_content']), 'input_orthography': data['input_orthography'], 'output_orthography': data['output_orthography'] } result['html'] = h.get_HTML_from_contents(result['page_content'], result['markup_language']) return result
def _get_user_data(self, data): return { 'object_language_name': data['object_language_name'], 'object_language_id': data['object_language_id'], 'metalanguage_name': data['metalanguage_name'], 'metalanguage_id': data['metalanguage_id'], 'metalanguage_inventory': h.normalize( h.remove_all_white_space(data['metalanguage_inventory'])), 'orthographic_validation': data['orthographic_validation'], 'narrow_phonetic_inventory': h.normalize( h.remove_all_white_space(data['narrow_phonetic_inventory'])), 'narrow_phonetic_validation': data['narrow_phonetic_validation'], 'broad_phonetic_inventory': h.normalize( h.remove_all_white_space(data['broad_phonetic_inventory'])), 'broad_phonetic_validation': data['broad_phonetic_validation'], 'morpheme_break_is_orthographic': data['morpheme_break_is_orthographic'], 'morpheme_break_validation': data['morpheme_break_validation'], 'phonemic_inventory': h.normalize(h.remove_all_white_space(data['phonemic_inventory'])), 'morpheme_delimiters': h.normalize(data['morpheme_delimiters']), 'punctuation': h.normalize(h.remove_all_white_space(data['punctuation'])), 'grammaticalities': h.normalize(h.remove_all_white_space(data['grammaticalities'])), # Many-to-One 'storage_orthography': data['storage_orthography'], 'input_orthography': data['input_orthography'], 'output_orthography': data['output_orthography'], # Many-to-Many Data: unrestricted_users 'unrestricted_users': [u for u in data['unrestricted_users'] if u] }
def get_probabilities(self): """Return the probability of each sequence of morphemes passed in the JSON PUT params. :param list morpheme_sequences: space-delimited morphemes in form|gloss|category format wherer "|" is actually ``h.rare_delimiter``. :returns: a dictionary with morpheme sequences as keys and log probabilities as values. """ langmod, id_ = self._model_from_id(eager=True) LOGGER.info( 'Attempting to get probabilities from morpheme language' ' model %s.', id_) if not langmod: self.request.response.status_int = 404 msg = 'There is no morpheme language model with id {}'.format(id_) LOGGER.warning(msg) return {'error': msg} try: schema = MorphemeSequencesSchema() values = json.loads(self.request.body.decode(self.request.charset)) data = schema.to_python(values) morpheme_sequences = [ h.normalize(ms) for ms in data['morpheme_sequences'] ] ret = langmod.get_probabilities(morpheme_sequences) LOGGER.info( 'Returned probabilities from morpheme language' ' model %s.', id_) return ret except ValueError: self.request.response.status_int = 400 LOGGER.warning(oldc.JSONDecodeErrorResponse) return oldc.JSONDecodeErrorResponse except Invalid as error: self.request.response.status_int = 400 errors = error.unpack_errors() LOGGER.warning(errors) return {'errors': errors} except Exception: self.request.response.status_int = 400 msg = 'An error occurred while trying to generate probabilities.' LOGGER.exception(msg) return {'error': msg}
def _create_externally_hosted_file(self, data): """Create an externally hosted file. :param dict data: the data to create the file model. :param str data['url']: a valid URL where the file data are served. :returns: an SQLAlchemy model object representing the file. Optional keys of the data dictionary, not including the standard metadata ones, are ``name``, ``password`` and ``MIME_type``. """ data['password'] = data.get('password') or '' schema = FileExternallyHostedSchema() data = schema.to_python(data) # User-inputted string data file_ = File(name=h.normalize(data['name']), password=data['password'], MIME_type=data['MIME_type'], url=data['url']) file_ = self._add_standard_metadata(file_, data) file_ = _restrict_file_by_forms(file_) return file_
def _add_standard_metadata(self, file_, data): """Add the standard metadata to the file model using the data dictionary. :param file_: file model object :param dict data: dictionary containing file attribute values. :returns: the updated file model object. """ file_.description = h.normalize(data['description']) file_.utterance_type = data['utterance_type'] file_.date_elicited = data['date_elicited'] if data['elicitor']: file_.elicitor = data['elicitor'] if data['speaker']: file_.speaker = data['speaker'] file_.tags = [t for t in data['tags'] if t] file_.forms = [f for f in data['forms'] if f] now = h.now() file_.datetime_entered = now file_.datetime_modified = now file_.enterer = self.logged_in_user return file_
def _update_externally_hosted_file(self, file_): """Update an externally hosted file model. :param file_: a file model object to update. :param request.body: a JSON object containing the data for updating the file. :returns: the file model or, if the file has not been updated, ``False``. """ changed = False data = json.loads(self.request.body.decode(self.request.charset)) data['password'] = data.get('password') or '' data = FileExternallyHostedSchema().to_python(data) # Data unique to referencing subinterval files changed = file_.set_attr('url', data['url'], changed) changed = file_.set_attr('name', h.normalize(data['name']), changed) changed = file_.set_attr('password', data['password'], changed) changed = file_.set_attr('MIME_type', data['MIME_type'], changed) file_, changed = _update_standard_metadata(file_, data, changed) if changed: file_.datetime_modified = datetime.datetime.utcnow() return file_ return changed
def _create_plain_file(self): """Create a local file using data from a ``Content-Type: multipart/form-data`` request. :param request.POST['filedata']: a ``cgi.FieldStorage`` object containing the file data. :param str request.POST['filename']: the name of the binary file. :returns: an SQLAlchemy model object representing the file. .. note:: The validator expects ``request.POST`` to encode list input via the ``formencode.variabledecode.NestedVariables`` format. E.g., a list of form ``id`` values would be provided as values to keys with names like ``'forms-0'``, ``'forms-1'``, ``'forms-2'``, etc. """ values = dict(self.request.params) filedata = self.request.POST.get('filedata') if not hasattr(filedata, 'file'): raise InvalidFieldStorageObjectError( 'POST filedata has no "file" attribute') if not values.get('filename'): values['filename'] = os.path.split(filedata.filename)[-1] state = SchemaState(full_dict={}, db=self.db, filedata_first_KB=filedata.value[:1024]) schema = FileCreateWithFiledataSchema() data = schema.to_python(values, state) file_ = File(filename=h.normalize(data['filename']), MIME_type=data['MIME_type']) files_path = h.get_old_directory_path('files', self.request.registry.settings) file_path = os.path.join(files_path, file_.filename) file_object, file_path = _get_unique_file_path(file_path) file_.filename = os.path.split(file_path)[-1] file_.name = file_.filename shutil.copyfileobj(filedata.file, file_object) filedata.file.close() file_object.close() file_.size = os.path.getsize(file_path) file_ = self._add_standard_metadata(file_, data) return file_
def _update_resource_model(self, resource_model, data): changed = False user_data = self._get_user_data(data) if data['password'] is not None: user_data['password'] = str( h.encrypt_password(data['password'], resource_model.salt.encode('utf8'))) if data['username'] is not None: username = h.normalize(data['username']) if username != resource_model.username: h.rename_user_directory(resource_model.username, username, self.request.registry.settings) user_data['username'] = username for attr, val in user_data.items(): if self._distinct(attr, val, getattr(resource_model, attr)): changed = True break if changed: for attr, val in self._get_update_data(user_data).items(): setattr(resource_model, attr, val) return resource_model return changed
def setUp(self): super().setUp() self.test_phonology_script = h.normalize( codecs.open(self.test_phonology_script_path, 'r', 'utf8').read())
def _get_user_data(self, data): return { 'name': h.normalize(data['name']), 'description': h.normalize(data['description']), }
def _get_user_data(self, data): return { 'name': h.normalize(data['name']), 'description': h.normalize(data['description']), 'script': h.normalize(data['script']).replace(u'\r', u'') }
def _get_user_data(self, data): return { 'type': h.normalize(data['type']), 'key': h.normalize(data['key']), 'address': h.normalize(data['address']), 'annote': h.normalize(data['annote']), 'author': h.normalize(data['author']), 'booktitle': h.normalize(data['booktitle']), 'chapter': h.normalize(data['chapter']), 'crossref': h.normalize(data['crossref']), 'edition': h.normalize(data['edition']), 'editor': h.normalize(data['editor']), 'howpublished': h.normalize(data['howpublished']), 'institution': h.normalize(data['institution']), 'journal': h.normalize(data['journal']), 'key_field': h.normalize(data['key_field']), 'month': h.normalize(data['month']), 'note': h.normalize(data['note']), 'number': h.normalize(data['number']), 'organization': h.normalize(data['organization']), 'pages': h.normalize(data['pages']), 'publisher': h.normalize(data['publisher']), 'school': h.normalize(data['school']), 'series': h.normalize(data['series']), 'title': h.normalize(data['title']), 'type_field': h.normalize(data['type_field']), 'url': data['url'], 'volume': h.normalize(data['volume']), 'year': data['year'], 'affiliation': h.normalize(data['affiliation']), 'abstract': h.normalize(data['abstract']), 'contents': h.normalize(data['contents']), 'copyright': h.normalize(data['copyright']), 'ISBN': h.normalize(data['ISBN']), 'ISSN': h.normalize(data['ISSN']), 'keywords': h.normalize(data['keywords']), 'language': h.normalize(data['language']), 'location': h.normalize(data['location']), 'LCCN': h.normalize(data['LCCN']), 'mrnumber': h.normalize(data['mrnumber']), 'price': h.normalize(data['price']), 'size': h.normalize(data['size']), 'file': data['file'], 'crossref_source': data['crossref_source'] }
def _update_resource_model(self, collection, data, collections_referenced): """Update a collection model. :param collection: the collection model to be updated. :param dict data: representation of the updated collection. :param dict collections_referenced: the collection models recursively referenced in ``data['contents']``. :returns: a 3-tuple where the second and third elements are invariable booleans indicating whether the collection has become restricted or has had its ``contents`` value changed as a result of the update, respectively. The first element is the updated collection or ``False`` of the no update has occurred. """ changed = False restricted = False contents_changed = False # Unicode Data changed = collection.set_attr('title', h.normalize(data['title']), changed) changed = collection.set_attr('type', h.normalize(data['type']), changed) changed = collection.set_attr('url', h.normalize(data['url']), changed) changed = collection.set_attr('description', h.normalize(data['description']), changed) changed = collection.set_attr('markup_language', h.normalize(data['markup_language']), changed) submitted_contents = h.normalize(data['contents']) if collection.contents != submitted_contents: collection.contents = submitted_contents contents_changed = changed = True changed = collection.set_attr('contents_unpacked', h.normalize(data['contents_unpacked']), changed) changed = collection.set_attr( 'html', h.get_HTML_from_contents(collection.contents_unpacked, collection.markup_language), changed) # User-entered date: date_elicited changed = collection.set_attr('date_elicited', data['date_elicited'], changed) # Many-to-One Data changed = collection.set_attr('elicitor', data['elicitor'], changed) changed = collection.set_attr('speaker', data['speaker'], changed) changed = collection.set_attr('source', data['source'], changed) # Many-to-Many Data: files, forms & tags # Update only if the user has made changes. files_to_add = [f for f in data['files'] if f] forms_to_add = [f for f in data['forms'] if f] tags_to_add = [t for t in data['tags'] if t] if set(files_to_add) != set(collection.files): collection.files = files_to_add changed = True if set(forms_to_add) != set(collection.forms): collection.forms = forms_to_add changed = True # Restrict the entire collection if it is associated to restricted # forms or files or if it references a restricted collection. tags = [ f.tags for f in collection.files + collection.forms + list(collections_referenced.values()) ] tags = [tag for tag_list in tags for tag in tag_list] restricted_tags = [tag for tag in tags if tag.name == 'restricted'] if restricted_tags: restricted_tag = restricted_tags[0] if restricted_tag not in tags_to_add: tags_to_add.append(restricted_tag) if set(tags_to_add) != set(collection.tags): if ('restricted' in [t.name for t in tags_to_add] and 'restricted' not in [t.name for t in collection.tags]): restricted = True collection.tags = tags_to_add changed = True if changed: collection.datetime_modified = datetime.datetime.utcnow() collection.modifier = self.logged_in_user return collection, restricted, contents_changed return changed, restricted, contents_changed