def _update_standard_metadata(file_, data, changed):
"""Update the standard metadata attributes of the input file.
:param file_: a file model object to be updated.
:param dict data: the data used to update the file model.
:param bool changed: indicates whether the file has been changed.
:returns: a tuple whose first element is the file model and whose second is
the boolean ``changed``.
"""
changed = file_.set_attr('description', h.normalize(data['description']),
changed)
changed = file_.set_attr('utterance_type',
h.normalize(data['utterance_type']), changed)
changed = file_.set_attr('date_elicited', data['date_elicited'], changed)
changed = file_.set_attr('elicitor', data['elicitor'], changed)
changed = file_.set_attr('speaker', data['speaker'], changed)
# Many-to-Many Data: tags & forms
# Update only if the user has made changes.
forms_to_add = [f for f in data['forms'] if f]
tags_to_add = [t for t in data['tags'] if t]
if set(forms_to_add) != set(file_.forms):
file_.forms = forms_to_add
changed = True
# Cause the entire file to be tagged as restricted if any one of its
# forms are so tagged.
tags = [f.tags for f in file_.forms]
tags = [tag for tag_list in tags for tag in tag_list]
restricted_tags = [tag for tag in tags if tag.name == 'restricted']
if restricted_tags:
restricted_tag = restricted_tags[0]
if restricted_tag not in tags_to_add:
tags_to_add.append(restricted_tag)
if set(tags_to_add) != set(file_.tags):
file_.tags = tags_to_add
changed = True
return file_, changed
def parse(self):
"""Parse the input word transcriptions using the morphological parser
with id=``id``.
:param str id: the ``id`` value of the morphological parser that will
be used.
:Request body: JSON object of the form
``{'transcriptions': [t1, t2, ...]}``.
:returns: if the morphological parser exists and foma is installed, a
JSON object of the form ``{t1: p1, t2: p2, ...}`` where ``t1`` and
``t2`` are transcriptions of words from the request body and ``p1``
and ``p2`` are the most probable morphological parsers of t1 and t2.
"""
morphparser, id_ = self._model_from_id(eager=True)
LOGGER.info('Attempting to call parse against morphological parser %d',
id_)
if not morphparser:
self.request.response.status_int = 404
msg = 'There is no morphological parser with id {}'.format(id_)
LOGGER.warning(msg)
return {'error': msg}
if not h.foma_installed():
self.request.response.status_int = 400
msg = 'Foma and flookup are not installed.'
LOGGER.warning(msg)
return {'error': msg}
try:
inputs = json.loads(self.request.body.decode(self.request.charset))
morphparser.cache = Cache(morphparser,
self.request.registry.settings,
session_getter)
LOGGER.warning([
h.normalize(w) for w in TranscriptionsSchema.to_python(inputs)
['transcriptions']
])
parses = morphparser.parse([
h.normalize(w) for w in TranscriptionsSchema.to_python(inputs)
['transcriptions']
])
# TODO: allow for a param which causes the candidates to be
# returned as well as/instead of only the most probable parse
# candidate.
LOGGER.info('Called parse against morphological parser %d', id_)
return {
transcription: parse
for transcription, (parse, candidates) in parses.items()
}
except ValueError:
self.request.response.status_int = 400
LOGGER.warning(oldc.JSONDecodeErrorResponse)
return oldc.JSONDecodeErrorResponse
except Invalid as error:
self.request.response.status_int = 400
errors = error.unpack_errors()
LOGGER.warning(errors)
return {'errors': errors}
except Exception as error:
self.request.response.status_int = 400
msg = 'Parse request raised an error.'
LOGGER.warning(msg, exc_info=True)
return {'error': msg}
def _get_user_data(self, data):
return {
'name': h.normalize(data['name']),
'orthography': h.normalize(data['orthography']),
'lowercase': data['lowercase'],
'initial_glottal_stops': data['initial_glottal_stops']
}
def _get_user_data(self, data):
return {
'name': h.normalize(data['name']),
'description': h.normalize(data['description']),
'phonology': data['phonology'],
'morphology': data['morphology'],
'language_model': data['language_model']
}
def _get_user_data(self, data):
"""User-provided data for creating a corpus."""
return {
'name': h.normalize(data['name']),
'description': h.normalize(data['description']),
'content': data['content'],
'form_search': data['form_search'],
'forms': data['forms'],
'tags': data['tags']
}
def _get_user_data(self, data):
result = {
'name': h.normalize(data['name']),
'heading': h.normalize(data['heading']),
'markup_language': data['markup_language'],
'content': h.normalize(data['content'])
}
result['html'] = h.get_HTML_from_contents(result['content'],
result['markup_language'])
return result
def _get_user_data(self, data):
result = {
'first_name': h.normalize(data['first_name']),
'last_name': h.normalize(data['last_name']),
'dialect': h.normalize(data['dialect']),
'page_content': h.normalize(data['page_content']),
'markup_language': h.normalize(data['markup_language'])
}
result['html'] = h.get_HTML_from_contents(result['page_content'],
result['markup_language'])
return result
def _get_user_data(self, data):
return {
'name': h.normalize(data['name']),
'description': h.normalize(data['description']),
'vocabulary_morphology': data['vocabulary_morphology'],
'corpus': data['corpus'],
'toolkit': data['toolkit'],
'order': data['order'],
'smoothing': data['smoothing'],
'categorial': data['categorial']
}
def _create_base64_file(self, data):
"""Create a local file using data from a ``Content-Type:
application/json`` request.
:param dict data: the data to create the file model.
:param str data['base64_encoded_file']: Base64-encoded file data.
:returns: an SQLAlchemy model object representing the file.
"""
data[
'MIME_type'] = '' # during validation, the schema will set a proper
# value based on the base64_encoded_file or
# filename attribute
schema = FileCreateWithBase64EncodedFiledataSchema()
state = SchemaState(full_dict=data,
db=self.db,
logged_in_user=self.logged_in_user)
data = schema.to_python(data, state)
file_ = File(MIME_type=data['MIME_type'],
filename=h.normalize(data['filename']))
file_ = self._add_standard_metadata(file_, data)
# Write the file to disk (making sure it's unique and thereby potentially)
# modifying file.filename; and calculate file.size.
# base64-decoded during validation
file_data = data['base64_encoded_file']
files_path = h.get_old_directory_path('files',
self.request.registry.settings)
file_path = os.path.join(files_path, file_.filename)
file_object, file_path = _get_unique_file_path(file_path)
file_.filename = os.path.split(file_path)[-1]
file_.name = file_.filename
file_object.write(file_data)
file_object.close()
file_data = None
file_.size = os.path.getsize(file_path)
file_ = _restrict_file_by_forms(file_)
return file_
def _create_subinterval_referencing_file(self, data):
"""Create a subinterval-referencing file.
:param dict data: the data to create the file model.
:param int data['parent_file']: the ``id`` value of an audio/video file
model.
:param float/int data['start']: the start of the interval in seconds.
:param float/int data['end']: the end of the interval in seconds.
:returns: an SQLAlchemy model object representing the file.
A value for ``data['name']`` may also be supplied.
"""
data['name'] = data.get('name') or ''
schema = FileSubintervalReferencingSchema()
state = SchemaState(full_dict=data,
db=self.db,
logged_in_user=self.logged_in_user)
data = schema.to_python(data, state)
# Data unique to referencing subinterval files
file_ = File(
parent_file=data['parent_file'],
# Name defaults to the parent file's filename if nothing provided by
# user
name=h.normalize(data['name']) or data['parent_file'].filename,
start=data['start'],
end=data['end'],
MIME_type=data['parent_file'].MIME_type)
file_ = self._add_standard_metadata(file_, data)
file_ = _restrict_file_by_forms(file_)
return file_
def _update_subinterval_referencing_file(self, file_):
"""Update a subinterval-referencing file model.
:param file_: a file model object to update.
:param request.body: a JSON object containing the data for updating the
file.
:returns: the file model or, if the file has not been updated, ``False``.
"""
changed = False
schema = FileSubintervalReferencingSchema()
data = json.loads(self.request.body.decode(self.request.charset))
data['name'] = data.get('name') or ''
state = SchemaState(full_dict=data,
db=self.db,
logged_in_user=self.logged_in_user)
data = schema.to_python(data, state)
# Data unique to referencing subinterval files
changed = file_.set_attr('parent_file', data['parent_file'], changed)
changed = file_.set_attr(
'name', (h.normalize(data['name']) or file_.parent_file.filename),
changed)
changed = file_.set_attr('start', data['start'], changed)
changed = file_.set_attr('end', data['end'], changed)
file_, changed = _update_standard_metadata(file_, data, changed)
if changed:
file_.datetime_modified = datetime.datetime.utcnow()
return file_
return changed
def _get_create_data(self, data):
create_data = self._get_user_data(data)
create_data['salt'] = str(h.generate_salt())
create_data['password'] = str(
h.encrypt_password(data['password'],
create_data['salt'].encode('utf8')))
create_data['username'] = h.normalize(data['username'])
create_data['datetime_modified'] = datetime.datetime.utcnow()
return create_data
def _get_user_data(self, data):
return {
'name':
h.normalize(data['name']),
'description':
h.normalize(data['description']),
'lexicon_corpus':
data['lexicon_corpus'],
'rules_corpus':
data['rules_corpus'],
'script_type':
data['script_type'],
'extract_morphemes_from_rules_corpus':
data['extract_morphemes_from_rules_corpus'],
'rules':
data['rules'],
'rich_upper':
data['rich_upper'],
'rich_lower':
data['rich_lower'],
'include_unknowns':
data['include_unknowns']
}
def applydown(self):
"""Apply-down (i.e., phonologize) the input in the request body using a
phonology.
:URL: ``PUT /phonologies/applydown/id`` (or ``PUT
/phonologies/phonologize/id``)
:param str id: the ``id`` value of the phonology that will be used.
:Request body: JSON object of the form
``{'transcriptions': [t1, t2, ...]}``.
:returns: if the phonology exists and foma is installed, a JSON object
of the form ``{t1: [p1t1, p2t1, ...], ...}`` where ``t1`` is a
transcription from the request body and ``p1t1``, ``p2t1``, etc. are
phonologized outputs of ``t1``.
"""
phonology, id_ = self._model_from_id(eager=True)
LOGGER.info(
'Attempting to call apply down on the compiled phonology'
' %d', id_)
if not phonology:
self.request.response.status_int = 404
msg = 'There is no phonology with id {}'.format(id_)
LOGGER.warning(msg)
return {'error': msg}
if not h.foma_installed():
self.request.response.status_int = 400
msg = 'Foma and flookup are not installed.'
LOGGER.warning(msg)
return {'error': msg}
binary_path = phonology.get_file_path('binary')
if not os.path.isfile(binary_path):
self.request.response.status_int = 400
msg = 'Phonology {} has not been compiled yet.'.format(
phonology.id)
LOGGER.warning(msg)
return {'error': msg}
try:
inputs = json.loads(self.request.body.decode(self.request.charset))
inputs = MorphophonemicTranscriptionsSchema.to_python(inputs)
inputs = [h.normalize(i) for i in inputs['transcriptions']]
ret = phonology.applydown(inputs)
LOGGER.info('Called apply down on the compiled phonology %d', id_)
return ret
except ValueError:
self.request.response.status_int = 400
LOGGER.warning(oldc.JSONDecodeErrorResponse)
return oldc.JSONDecodeErrorResponse
except Invalid as error:
self.request.response.status_int = 400
errors = error.unpack_errors()
LOGGER.warning(errors)
return {'errors': errors}
def _create_new_resource(self, data, collections_referenced):
"""Create a new collection.
:param dict data: the collection to be created.
:returns: an SQLAlchemy model object representing the collection.
"""
collection = Collection()
collection.UUID = str(uuid4())
# User-inputted string data
collection.title = h.normalize(data['title'])
collection.type = h.normalize(data['type'])
collection.url = h.normalize(data['url'])
collection.description = h.normalize(data['description'])
collection.markup_language = h.normalize(data['markup_language'])
collection.contents = h.normalize(data['contents'])
collection.contents_unpacked = h.normalize(data['contents_unpacked'])
collection.html = h.get_HTML_from_contents(
collection.contents_unpacked, collection.markup_language)
# User-inputted date: date_elicited
collection.date_elicited = data['date_elicited']
# Many-to-One
if data['elicitor']:
collection.elicitor = data['elicitor']
if data['speaker']:
collection.speaker = data['speaker']
if data['source']:
collection.source = data['source']
# Many-to-Many: tags, files & forms
collection.tags = [t for t in data['tags'] if t]
collection.files = [f for f in data['files'] if f]
collection.forms = [f for f in data['forms'] if f]
# Restrict the entire collection if it is associated to restricted forms or
# files or if it references a restricted collection in its contents field.
immediately_referenced_collections = \
_get_collections_referenced_in_contents(
collection, collections_referenced)
tags = [
f.tags for f in collection.files + collection.forms +
immediately_referenced_collections
]
tags = [tag for tag_list in tags for tag in tag_list]
restricted_tags = [tag for tag in tags if tag.name == 'restricted']
if restricted_tags:
restricted_tag = restricted_tags[0]
if restricted_tag not in collection.tags:
collection.tags.append(restricted_tag)
# OLD-generated Data
now = datetime.datetime.utcnow()
collection.datetime_entered = now
collection.datetime_modified = now
collection.enterer = collection.modifier = self.logged_in_user
return collection
def apply(self, direction):
"""Call foma apply in the direction of ``direction`` on the input in
the request body using a morphology.
:param str id: the ``id`` value of the morphology that will be used.
:param str direction: the direction of foma application.
:Request body: JSON object of the form
``{'transcriptions': [t1, t2, ...]}``.
:returns: if the morphology exists and foma is installed, a JSON object
of the form ``{t1: [p1t1, p2t1, ...], ...}`` where ``t1`` is a
transcription from the request body and ``p1t1``, ``p2t1``, etc. are
outputs of ``t1`` after apply up/down.
"""
morphology, id_ = self._model_from_id(eager=True)
if not morphology:
self.request.response.status_int = 404
msg = 'There is no morphology with id {}'.format(id_)
LOGGER.warning(msg)
return {'error': msg}
if not h.foma_installed():
self.request.response.status_int = 400
msg = 'Foma and flookup are not installed.'
LOGGER.warning(msg)
return {'error': msg}
morphology_binary_path = morphology.get_file_path('binary')
if not os.path.isfile(morphology_binary_path):
self.request.response.status_int = 400
msg = 'Morphology {} has not been compiled yet.'.format(
morphology.id)
LOGGER.warning(msg)
return {'error': msg}
try:
inputs = json.loads(self.request.body.decode(self.request.charset))
inputs = MorphemeSequencesSchema.to_python(inputs)
inputs = [h.normalize(i) for i in inputs['morpheme_sequences']]
ret = morphology.apply(direction, inputs)
LOGGER.info('Completed apply call against morphology' ' %d.', id_)
return ret
except ValueError:
self.request.response.status_int = 400
LOGGER.warning(oldc.JSONDecodeErrorResponse)
return oldc.JSONDecodeErrorResponse
except Invalid as error:
self.request.response.status_int = 400
errors = error.unpack_errors()
LOGGER.warning(errors)
return {'errors': errors}
def _get_user_data(self, data):
result = {
'first_name': h.normalize(data['first_name']),
'last_name': h.normalize(data['last_name']),
'email': h.normalize(data['email']),
'affiliation': h.normalize(data['affiliation']),
'role': h.normalize(data['role']),
'markup_language': h.normalize(data['markup_language']),
'page_content': h.normalize(data['page_content']),
'input_orthography': data['input_orthography'],
'output_orthography': data['output_orthography']
}
result['html'] = h.get_HTML_from_contents(result['page_content'],
result['markup_language'])
return result
def _get_user_data(self, data):
return {
'object_language_name':
data['object_language_name'],
'object_language_id':
data['object_language_id'],
'metalanguage_name':
data['metalanguage_name'],
'metalanguage_id':
data['metalanguage_id'],
'metalanguage_inventory':
h.normalize(
h.remove_all_white_space(data['metalanguage_inventory'])),
'orthographic_validation':
data['orthographic_validation'],
'narrow_phonetic_inventory':
h.normalize(
h.remove_all_white_space(data['narrow_phonetic_inventory'])),
'narrow_phonetic_validation':
data['narrow_phonetic_validation'],
'broad_phonetic_inventory':
h.normalize(
h.remove_all_white_space(data['broad_phonetic_inventory'])),
'broad_phonetic_validation':
data['broad_phonetic_validation'],
'morpheme_break_is_orthographic':
data['morpheme_break_is_orthographic'],
'morpheme_break_validation':
data['morpheme_break_validation'],
'phonemic_inventory':
h.normalize(h.remove_all_white_space(data['phonemic_inventory'])),
'morpheme_delimiters':
h.normalize(data['morpheme_delimiters']),
'punctuation':
h.normalize(h.remove_all_white_space(data['punctuation'])),
'grammaticalities':
h.normalize(h.remove_all_white_space(data['grammaticalities'])),
# Many-to-One
'storage_orthography':
data['storage_orthography'],
'input_orthography':
data['input_orthography'],
'output_orthography':
data['output_orthography'],
# Many-to-Many Data: unrestricted_users
'unrestricted_users': [u for u in data['unrestricted_users'] if u]
}
def get_probabilities(self):
"""Return the probability of each sequence of morphemes passed in the
JSON PUT params.
:param list morpheme_sequences: space-delimited morphemes in
form|gloss|category format wherer "|" is actually
``h.rare_delimiter``.
:returns: a dictionary with morpheme sequences as keys and log
probabilities as values.
"""
langmod, id_ = self._model_from_id(eager=True)
LOGGER.info(
'Attempting to get probabilities from morpheme language'
' model %s.', id_)
if not langmod:
self.request.response.status_int = 404
msg = 'There is no morpheme language model with id {}'.format(id_)
LOGGER.warning(msg)
return {'error': msg}
try:
schema = MorphemeSequencesSchema()
values = json.loads(self.request.body.decode(self.request.charset))
data = schema.to_python(values)
morpheme_sequences = [
h.normalize(ms) for ms in data['morpheme_sequences']
]
ret = langmod.get_probabilities(morpheme_sequences)
LOGGER.info(
'Returned probabilities from morpheme language'
' model %s.', id_)
return ret
except ValueError:
self.request.response.status_int = 400
LOGGER.warning(oldc.JSONDecodeErrorResponse)
return oldc.JSONDecodeErrorResponse
except Invalid as error:
self.request.response.status_int = 400
errors = error.unpack_errors()
LOGGER.warning(errors)
return {'errors': errors}
except Exception:
self.request.response.status_int = 400
msg = 'An error occurred while trying to generate probabilities.'
LOGGER.exception(msg)
return {'error': msg}
def _create_externally_hosted_file(self, data):
"""Create an externally hosted file.
:param dict data: the data to create the file model.
:param str data['url']: a valid URL where the file data are served.
:returns: an SQLAlchemy model object representing the file.
Optional keys of the data dictionary, not including the standard
metadata ones, are ``name``, ``password`` and ``MIME_type``.
"""
data['password'] = data.get('password') or ''
schema = FileExternallyHostedSchema()
data = schema.to_python(data)
# User-inputted string data
file_ = File(name=h.normalize(data['name']),
password=data['password'],
MIME_type=data['MIME_type'],
url=data['url'])
file_ = self._add_standard_metadata(file_, data)
file_ = _restrict_file_by_forms(file_)
return file_
def _add_standard_metadata(self, file_, data):
"""Add the standard metadata to the file model using the data dictionary.
:param file_: file model object
:param dict data: dictionary containing file attribute values.
:returns: the updated file model object.
"""
file_.description = h.normalize(data['description'])
file_.utterance_type = data['utterance_type']
file_.date_elicited = data['date_elicited']
if data['elicitor']:
file_.elicitor = data['elicitor']
if data['speaker']:
file_.speaker = data['speaker']
file_.tags = [t for t in data['tags'] if t]
file_.forms = [f for f in data['forms'] if f]
now = h.now()
file_.datetime_entered = now
file_.datetime_modified = now
file_.enterer = self.logged_in_user
return file_
def _update_externally_hosted_file(self, file_):
"""Update an externally hosted file model.
:param file_: a file model object to update.
:param request.body: a JSON object containing the data for updating the
file.
:returns: the file model or, if the file has not been updated,
``False``.
"""
changed = False
data = json.loads(self.request.body.decode(self.request.charset))
data['password'] = data.get('password') or ''
data = FileExternallyHostedSchema().to_python(data)
# Data unique to referencing subinterval files
changed = file_.set_attr('url', data['url'], changed)
changed = file_.set_attr('name', h.normalize(data['name']), changed)
changed = file_.set_attr('password', data['password'], changed)
changed = file_.set_attr('MIME_type', data['MIME_type'], changed)
file_, changed = _update_standard_metadata(file_, data, changed)
if changed:
file_.datetime_modified = datetime.datetime.utcnow()
return file_
return changed
def _create_plain_file(self):
"""Create a local file using data from a ``Content-Type:
multipart/form-data`` request.
:param request.POST['filedata']: a ``cgi.FieldStorage`` object
containing the file data.
:param str request.POST['filename']: the name of the binary file.
:returns: an SQLAlchemy model object representing the file.
.. note::
The validator expects ``request.POST`` to encode list input via the
``formencode.variabledecode.NestedVariables`` format. E.g., a list
of form ``id`` values would be provided as values to keys with
names like ``'forms-0'``, ``'forms-1'``, ``'forms-2'``, etc.
"""
values = dict(self.request.params)
filedata = self.request.POST.get('filedata')
if not hasattr(filedata, 'file'):
raise InvalidFieldStorageObjectError(
'POST filedata has no "file" attribute')
if not values.get('filename'):
values['filename'] = os.path.split(filedata.filename)[-1]
state = SchemaState(full_dict={},
db=self.db,
filedata_first_KB=filedata.value[:1024])
schema = FileCreateWithFiledataSchema()
data = schema.to_python(values, state)
file_ = File(filename=h.normalize(data['filename']),
MIME_type=data['MIME_type'])
files_path = h.get_old_directory_path('files',
self.request.registry.settings)
file_path = os.path.join(files_path, file_.filename)
file_object, file_path = _get_unique_file_path(file_path)
file_.filename = os.path.split(file_path)[-1]
file_.name = file_.filename
shutil.copyfileobj(filedata.file, file_object)
filedata.file.close()
file_object.close()
file_.size = os.path.getsize(file_path)
file_ = self._add_standard_metadata(file_, data)
return file_
def _update_resource_model(self, resource_model, data):
changed = False
user_data = self._get_user_data(data)
if data['password'] is not None:
user_data['password'] = str(
h.encrypt_password(data['password'],
resource_model.salt.encode('utf8')))
if data['username'] is not None:
username = h.normalize(data['username'])
if username != resource_model.username:
h.rename_user_directory(resource_model.username, username,
self.request.registry.settings)
user_data['username'] = username
for attr, val in user_data.items():
if self._distinct(attr, val, getattr(resource_model, attr)):
changed = True
break
if changed:
for attr, val in self._get_update_data(user_data).items():
setattr(resource_model, attr, val)
return resource_model
return changed
def setUp(self):
super().setUp()
self.test_phonology_script = h.normalize(
codecs.open(self.test_phonology_script_path, 'r', 'utf8').read())
def _get_user_data(self, data):
return {
'name': h.normalize(data['name']),
'description': h.normalize(data['description']),
}
def _get_user_data(self, data):
return {
'name': h.normalize(data['name']),
'description': h.normalize(data['description']),
'script': h.normalize(data['script']).replace(u'\r', u'')
}
def _get_user_data(self, data):
return {
'type': h.normalize(data['type']),
'key': h.normalize(data['key']),
'address': h.normalize(data['address']),
'annote': h.normalize(data['annote']),
'author': h.normalize(data['author']),
'booktitle': h.normalize(data['booktitle']),
'chapter': h.normalize(data['chapter']),
'crossref': h.normalize(data['crossref']),
'edition': h.normalize(data['edition']),
'editor': h.normalize(data['editor']),
'howpublished': h.normalize(data['howpublished']),
'institution': h.normalize(data['institution']),
'journal': h.normalize(data['journal']),
'key_field': h.normalize(data['key_field']),
'month': h.normalize(data['month']),
'note': h.normalize(data['note']),
'number': h.normalize(data['number']),
'organization': h.normalize(data['organization']),
'pages': h.normalize(data['pages']),
'publisher': h.normalize(data['publisher']),
'school': h.normalize(data['school']),
'series': h.normalize(data['series']),
'title': h.normalize(data['title']),
'type_field': h.normalize(data['type_field']),
'url': data['url'],
'volume': h.normalize(data['volume']),
'year': data['year'],
'affiliation': h.normalize(data['affiliation']),
'abstract': h.normalize(data['abstract']),
'contents': h.normalize(data['contents']),
'copyright': h.normalize(data['copyright']),
'ISBN': h.normalize(data['ISBN']),
'ISSN': h.normalize(data['ISSN']),
'keywords': h.normalize(data['keywords']),
'language': h.normalize(data['language']),
'location': h.normalize(data['location']),
'LCCN': h.normalize(data['LCCN']),
'mrnumber': h.normalize(data['mrnumber']),
'price': h.normalize(data['price']),
'size': h.normalize(data['size']),
'file': data['file'],
'crossref_source': data['crossref_source']
}
def _update_resource_model(self, collection, data, collections_referenced):
"""Update a collection model.
:param collection: the collection model to be updated.
:param dict data: representation of the updated collection.
:param dict collections_referenced: the collection models recursively
referenced in ``data['contents']``.
:returns: a 3-tuple where the second and third elements are invariable
booleans indicating whether the collection has become restricted or has
had its ``contents`` value changed as a result of the update,
respectively. The first element is the updated collection or ``False``
of the no update has occurred.
"""
changed = False
restricted = False
contents_changed = False
# Unicode Data
changed = collection.set_attr('title', h.normalize(data['title']),
changed)
changed = collection.set_attr('type', h.normalize(data['type']),
changed)
changed = collection.set_attr('url', h.normalize(data['url']), changed)
changed = collection.set_attr('description',
h.normalize(data['description']),
changed)
changed = collection.set_attr('markup_language',
h.normalize(data['markup_language']),
changed)
submitted_contents = h.normalize(data['contents'])
if collection.contents != submitted_contents:
collection.contents = submitted_contents
contents_changed = changed = True
changed = collection.set_attr('contents_unpacked',
h.normalize(data['contents_unpacked']),
changed)
changed = collection.set_attr(
'html',
h.get_HTML_from_contents(collection.contents_unpacked,
collection.markup_language), changed)
# User-entered date: date_elicited
changed = collection.set_attr('date_elicited', data['date_elicited'],
changed)
# Many-to-One Data
changed = collection.set_attr('elicitor', data['elicitor'], changed)
changed = collection.set_attr('speaker', data['speaker'], changed)
changed = collection.set_attr('source', data['source'], changed)
# Many-to-Many Data: files, forms & tags
# Update only if the user has made changes.
files_to_add = [f for f in data['files'] if f]
forms_to_add = [f for f in data['forms'] if f]
tags_to_add = [t for t in data['tags'] if t]
if set(files_to_add) != set(collection.files):
collection.files = files_to_add
changed = True
if set(forms_to_add) != set(collection.forms):
collection.forms = forms_to_add
changed = True
# Restrict the entire collection if it is associated to restricted
# forms or files or if it references a restricted collection.
tags = [
f.tags for f in collection.files + collection.forms +
list(collections_referenced.values())
]
tags = [tag for tag_list in tags for tag in tag_list]
restricted_tags = [tag for tag in tags if tag.name == 'restricted']
if restricted_tags:
restricted_tag = restricted_tags[0]
if restricted_tag not in tags_to_add:
tags_to_add.append(restricted_tag)
if set(tags_to_add) != set(collection.tags):
if ('restricted' in [t.name for t in tags_to_add]
and 'restricted' not in [t.name for t in collection.tags]):
restricted = True
collection.tags = tags_to_add
changed = True
if changed:
collection.datetime_modified = datetime.datetime.utcnow()
collection.modifier = self.logged_in_user
return collection, restricted, contents_changed
return changed, restricted, contents_changed
