def _pre_delete(self, file_model):
"""Delete the digital file prior to deleting the database file data."""
if getattr(file_model, 'filename', None):
file_path = os.path.join(
h.get_old_directory_path('files',
self.request.registry.settings),
file_model.filename)
os.remove(file_path)
if getattr(file_model, 'lossy_filename', None):
file_path = os.path.join(
h.get_old_directory_path('reduced_files',
self.request.registry.settings),
file_model.lossy_filename)
os.remove(file_path)
def _create_base64_file(self, data):
"""Create a local file using data from a ``Content-Type:
application/json`` request.
:param dict data: the data to create the file model.
:param str data['base64_encoded_file']: Base64-encoded file data.
:returns: an SQLAlchemy model object representing the file.
"""
data[
'MIME_type'] = '' # during validation, the schema will set a proper
# value based on the base64_encoded_file or
# filename attribute
schema = FileCreateWithBase64EncodedFiledataSchema()
state = SchemaState(full_dict=data,
db=self.db,
logged_in_user=self.logged_in_user)
data = schema.to_python(data, state)
file_ = File(MIME_type=data['MIME_type'],
filename=h.normalize(data['filename']))
file_ = self._add_standard_metadata(file_, data)
# Write the file to disk (making sure it's unique and thereby potentially)
# modifying file.filename; and calculate file.size.
# base64-decoded during validation
file_data = data['base64_encoded_file']
files_path = h.get_old_directory_path('files',
self.request.registry.settings)
file_path = os.path.join(files_path, file_.filename)
file_object, file_path = _get_unique_file_path(file_path)
file_.filename = os.path.split(file_path)[-1]
file_.name = file_.filename
file_object.write(file_data)
file_object.close()
file_data = None
file_.size = os.path.getsize(file_path)
file_ = _restrict_file_by_forms(file_)
return file_
def tearDown(self, **kwargs):
"""Clean up after a test."""
db = DBUtils(self.dbsession, self.settings)
clear_all_tables = kwargs.get('clear_all_tables', False)
dirs_to_clear = kwargs.get('dirs_to_clear', [])
dirs_to_destroy = kwargs.get('dirs_to_destroy', [])
if clear_all_tables:
db.clear_all_tables(['language'])
else:
self.clear_all_models(self.dbsession)
for dir_path in dirs_to_clear:
h.clear_directory_of_files(getattr(self, dir_path))
for dir_name in dirs_to_destroy:
h.destroy_all_directories(self.inflect_p.plural(dir_name),
self.settings)
if self.Session2:
db = DBUtils(self.dbsession, self.settings2)
clear_all_tables = kwargs.get('clear_all_tables', False)
dirs_to_clear = kwargs.get('dirs_to_clear', [])
dirs_to_destroy = kwargs.get('dirs_to_destroy', [])
if clear_all_tables:
db.clear_all_tables(['language'])
else:
self.clear_all_models(self.dbsession2)
for attr_name in dirs_to_clear:
dir_name = attr_name.replace('_path', '')
dir_path = h.get_old_directory_path(dir_name,
settings=self.settings2)
h.clear_directory_of_files(dir_path)
for dir_name in dirs_to_destroy:
h.destroy_all_directories(self.inflect_p.plural(dir_name),
self.settings2)
self.tear_down_dbsession()
def _get_create_data(self, data):
user_data = self._get_user_data(data)
now = h.now()
user_model = self.logged_in_user
user_data.update({
'parent_directory':
h.get_old_directory_path('morphologies',
self.request.registry.settings),
# TODO: the Pylons app implied that this constant could change...
'word_boundary_symbol':
oldc.WORD_BOUNDARY_SYMBOL,
'rare_delimiter':
oldc.RARE_DELIMITER,
'morpheme_delimiters':
self.db.get_morpheme_delimiters(type_='str'),
'UUID':
str(uuid4()),
'enterer':
user_model,
'modifier':
user_model,
'datetime_modified':
now,
'datetime_entered':
now
})
return user_data
def _get_create_data(self, data):
user_data = self._get_user_data(data)
now = h.now()
user_model = self.logged_in_user
user_data.update({
'parent_directory':
h.get_old_directory_path('morphemelanguagemodels',
self.request.registry.settings),
'rare_delimiter':
oldc.RARE_DELIMITER,
'start_symbol':
oldc.LM_START,
'end_symbol':
oldc.LM_END,
'morpheme_delimiters':
self.db.get_morpheme_delimiters(type_='str'),
'UUID':
str(uuid4()),
'enterer':
user_model,
'modifier':
user_model,
'datetime_modified':
now,
'datetime_entered':
now
})
return user_data
def _get_create_data(self, data):
user_data = self._get_user_data(data)
now = h.now()
user_model = self.logged_in_user
user_data.update({
'parent_directory':
h.get_old_directory_path('morphologicalparsers',
self.request.registry.settings),
'UUID':
str(uuid4()),
'enterer':
user_model,
'modifier':
user_model,
'datetime_modified':
now,
'datetime_entered':
now
})
return user_data
def _create_plain_file(self):
"""Create a local file using data from a ``Content-Type:
multipart/form-data`` request.
:param request.POST['filedata']: a ``cgi.FieldStorage`` object
containing the file data.
:param str request.POST['filename']: the name of the binary file.
:returns: an SQLAlchemy model object representing the file.
.. note::
The validator expects ``request.POST`` to encode list input via the
``formencode.variabledecode.NestedVariables`` format. E.g., a list
of form ``id`` values would be provided as values to keys with
names like ``'forms-0'``, ``'forms-1'``, ``'forms-2'``, etc.
"""
values = dict(self.request.params)
filedata = self.request.POST.get('filedata')
if not hasattr(filedata, 'file'):
raise InvalidFieldStorageObjectError(
'POST filedata has no "file" attribute')
if not values.get('filename'):
values['filename'] = os.path.split(filedata.filename)[-1]
state = SchemaState(full_dict={},
db=self.db,
filedata_first_KB=filedata.value[:1024])
schema = FileCreateWithFiledataSchema()
data = schema.to_python(values, state)
file_ = File(filename=h.normalize(data['filename']),
MIME_type=data['MIME_type'])
files_path = h.get_old_directory_path('files',
self.request.registry.settings)
file_path = os.path.join(files_path, file_.filename)
file_object, file_path = _get_unique_file_path(file_path)
file_.filename = os.path.split(file_path)[-1]
file_.name = file_.filename
shutil.copyfileobj(filedata.file, file_object)
filedata.file.close()
file_object.close()
file_.size = os.path.getsize(file_path)
file_ = self._add_standard_metadata(file_, data)
return file_
def _serve(self, reduced=False):
"""Serve the content (binary data) of a file.
:param bool reduced: toggles serving of file data or reduced-size file
data.
"""
file_, id_ = self._model_from_id(eager=True)
if not file_:
self.request.response.status_int = 404
return {'error': 'There is no file with id %s' % id_}
if self._model_access_unauth(file_) is not False:
self.request.response.status_int = 403
return UNAUTHORIZED_MSG
if getattr(file_, 'parent_file', None):
file_ = file_.parent_file
elif getattr(file_, 'url', None):
self.request.response.status_int = 400
return {
'error':
'The content of file %s is stored elsewhere at %s' %
(id_, file_.url)
}
files_dir = h.get_old_directory_path('files',
self.request.registry.settings)
if reduced:
filename = getattr(file_, 'lossy_filename', None)
if not filename:
self.request.response.status_int = 404
return {
'error': 'There is no size-reduced copy of file %s' % id_
}
file_path = os.path.join(files_dir, 'reduced_files', filename)
content_type = guess_type(filename)[0]
else:
file_path = os.path.join(files_dir, file_.filename)
content_type = file_.MIME_type
return FileResponse(file_path,
request=self.request,
content_type=content_type)
def _get_corpus_dir_path(self, corpus):
return os.path.join(
h.get_old_directory_path('corpora',
self.request.registry.settings),
'corpus_%d' % corpus.id)
def _setattrs(self):
"""Set a whole bunch of instance attributes that are useful in tests."""
self.extra_environ_view = {'test.authentication.role': 'viewer'}
self.extra_environ_contrib = {
'test.authentication.role': 'contributor'
}
self.extra_environ_admin = {
'test.authentication.role': 'administrator'
}
self.extra_environ_view_appset = {
'test.authentication.role': 'viewer',
'test.application_settings': True
}
self.extra_environ_contrib_appset = {
'test.authentication.role': 'contributor',
'test.application_settings': True
}
self.extra_environ_admin_appset = {
'test.authentication.role': 'administrator',
'test.application_settings': True
}
self.json_headers = {'Content-Type': 'application/json'}
self.here = self.settings['here']
self.files_path = h.get_old_directory_path('files',
settings=self.settings)
self.reduced_files_path = h.get_old_directory_path(
'reduced_files', settings=self.settings)
self.test_files_path = os.path.join(self.here, 'old', 'tests', 'data',
'files')
self.create_reduced_size_file_copies = asbool(
self.settings.get('create_reduced_size_file_copies', False))
self.preferred_lossy_audio_format = self.settings.get(
'preferred_lossy_audio_format', 'ogg')
self.corpora_path = h.get_old_directory_path('corpora',
settings=self.settings)
self.test_datasets_path = os.path.join(self.here, 'old', 'tests',
'data', 'datasets')
self.test_scripts_path = os.path.join(self.here, 'old', 'tests',
'scripts')
self.loremipsum100_path = os.path.join(self.test_datasets_path,
'loremipsum_100.txt')
self.loremipsum1000_path = os.path.join(self.test_datasets_path,
'loremipsum_1000.txt')
self.loremipsum10000_path = os.path.join(self.test_datasets_path,
'loremipsum_10000.txt')
self.users_path = h.get_old_directory_path('users',
settings=self.settings)
self.morphologies_path = h.get_old_directory_path(
'morphologies', settings=self.settings)
self.morphological_parsers_path = h.get_old_directory_path(
'morphological_parsers', settings=self.settings)
self.phonologies_path = h.get_old_directory_path(
'phonologies', settings=self.settings)
self.morpheme_language_models_path = h.get_old_directory_path(
'morpheme_language_models', settings=self.settings)
self.test_phonologies_path = os.path.join(self.here, 'old', 'tests',
'data', 'phonologies')
self.test_phonology_script_path = os.path.join(
self.test_phonologies_path, 'test_phonology.script')
self.test_malformed_phonology_script_path = os.path.join(
self.test_phonologies_path, 'test_phonology_malformed.script')
self.test_phonology_no_phonology_script_path = os.path.join(
self.test_phonologies_path, 'test_phonology_malformed.script')
self.test_medium_phonology_script_path = os.path.join(
self.test_phonologies_path, 'test_phonology_medium.script')
self.test_large_phonology_script_path = os.path.join(
self.test_phonologies_path, 'test_phonology_large.script')
self.test_phonology_testless_script_path = os.path.join(
self.test_phonologies_path, 'test_phonology_no_tests.script')
self.test_morphologies_path = os.path.join(self.here, 'old', 'tests',
'data', 'morphologies')
self.test_morphophonologies_path = os.path.join(
self.here, 'old', 'tests', 'data', 'morphophonologies')
def test_distinct_file_creation(self):
"""Tests that when a file is uploaded to OLD #1 it appears in the
correct place in the filesystem, and vice versa for OLD #2.
"""
# This will prevent routes.py from doing test-specific magic to delete
# our user sessions. See ``routes.py::fix_for_tests``.
extra_environ = {'test.rig.auth': False}
# Path on disk to store/oldtests2/files/
old2_files_path = h.get_old_directory_path('files',
settings=self.settings2)
# Login to OLD #1
params = json.dumps({'username': '******', 'password': '******'})
response = self.app.post(auth_url('authenticate'),
params,
self.json_headers,
extra_environ=extra_environ)
resp = response.json_body
assert resp['authenticated'] is True
assert response.content_type == 'application/json'
# Login to OLD #2
params = json.dumps({'username': '******', 'password': '******'})
response = self.app.post(auth_url_2('authenticate'),
params,
self.json_headers,
extra_environ=extra_environ)
resp = response.json_body
assert resp['authenticated'] is True
assert response.content_type == 'application/json'
# Get a JPG as a base64 string
jpg_file_path = os.path.join(self.test_files_path, 'old_test.jpg')
with open(jpg_file_path, 'rb') as f:
jpg_file_base64_encoded = b64encode(f.read()).decode('utf8')
# Upload a JPG to OLD #1
params = self.file_create_params_base64.copy()
filename = 'old_1_file.jpg'
params.update({
'filename': filename,
'base64_encoded_file': jpg_file_base64_encoded
})
params = json.dumps(params)
response = self.app.post(files_url('create'), params,
self.json_headers, extra_environ)
resp = response.json_body
# Expect OLD #1's JPG to be in OLD #1's store/ and not in OLD #2's
expected_good_path = os.path.join(self.files_path, filename)
expected_bad_path = os.path.join(old2_files_path, filename)
assert os.path.isfile(expected_good_path)
assert not os.path.isfile(expected_bad_path)
# Upload a JPG to OLD #2
params = self.file_create_params_base64.copy()
filename = 'old_2_file.jpg'
params.update({
'filename': filename,
'base64_encoded_file': jpg_file_base64_encoded
})
params = json.dumps(params)
response = self.app.post(files_url2('create'), params,
self.json_headers, extra_environ)
resp = response.json_body
# Expect OLD #2's JPG to be in OLD #2's store/ and not in OLD #1's
expected_good_path = os.path.join(old2_files_path, filename)
expected_bad_path = os.path.join(self.files_path, filename)
assert os.path.isfile(expected_good_path)
assert not os.path.isfile(expected_bad_path)