def test_invalid_zip(self):
''' invalid zipfile '''
with LogCapture() as lc:
assert not is_savezip_valid('junk.zip')
lc.check(('gnome.persist.save_load',
'WARNING',
'junk.zip is not a valid zipfile'))
def upload_model(request):
'''
Uploads a new model in the form of a zipfile and registers it as the
current active model.
We are generating our own filename instead of trusting
the incoming filename since that might result in insecure paths.
We may want to eventually use something other than /tmp,
and if you write to an untrusted location you will need to do
some extra work to prevent symlink attacks.
'''
clean_session_dir(request)
file_path, _name = process_upload(request, 'new_model')
# Now that we have our file, we will now try to load the model into
# memory.
# Now that we have our file, is it a zipfile?
if not is_savezip_valid(file_path):
raise cors_response(
request, HTTPBadRequest('Incoming file is not a '
'valid zipfile!'))
resp_msg = 'OK'
# now we try to load our model from the zipfile.
session_lock = acquire_session_lock(request)
log.info(' session lock acquired (sess:{}, thr_id: {})'.format(
id(session_lock),
current_thread().ident))
try:
log.info('loading our model from zip...')
init_session_objects(request, force=True)
refs = get_session_objects(request)
new_model = Model.load(file_path, refs=refs)
new_model._cache.enabled = False
new_model._schema.register_refs(new_model._schema(), new_model, refs)
# from ..views import implemented_types
#
# RegisterObject(new_model, request, implemented_types)
log.info('setting active model...')
set_active_model(request, new_model.id)
except Exception:
raise cors_exception(request, HTTPBadRequest, with_stacktrace=True)
finally:
session_lock.release()
log.info(' session lock released (sess:{}, thr_id: {})'.format(
id(session_lock),
current_thread().ident))
# We will want to clean up our tempfile when we are done.
os.remove(file_path)
return cors_response(request, Response(resp_msg))
def test_filenames_dont_contain_dotdot(self):
'''
'''
badzip = os.path.join(self.here,
'sample_data/badzip_max_compress_ratio.zip')
badfile = './../badpath.json'
with ZipFile(badzip, 'w', compression=ZIP_DEFLATED) as z:
z.writestr(badfile, 'bad file, contains path')
with LogCapture() as lc:
assert not is_savezip_valid(badzip)
lc.check(('gnome.persist.save_load', 'WARNING',
'Found ".." in {}. Rejecting zipfile'.format(badfile)))
def test_filenames_dont_contain_dotdot(self):
'''
'''
badzip = os.path.join(self.here,
'sample_data/badzip_max_compress_ratio.zip')
badfile = './../badpath.json'
with ZipFile(badzip, 'w', compression=ZIP_DEFLATED) as z:
z.writestr(badfile, 'bad file, contains path')
with LogCapture() as lc:
assert not is_savezip_valid(badzip)
lc.check(('gnome.persist.save_load',
'WARNING',
'Found ".." in {}. Rejecting zipfile'.format(badfile)))
def activate_uploaded_model(request):
'''
Activates a new model from a zipfile that is stored in the
uploads folder, and registers it as the current active model.
'''
clean_session_dir(request)
zipfile_path, _name = activate_uploaded(request)
log.info('Model zipfile: {}'.format(zipfile_path))
# Now that we have our file, we will now try to load the model into
# memory.
# Now that we have our file, is it a zipfile?
if not is_savezip_valid(zipfile_path):
raise cors_response(request,
HTTPBadRequest('File is not a valid zipfile!'))
# now we try to load our model from the zipfile.
session_lock = acquire_session_lock(request)
log.info(' session lock acquired (sess:{}, thr_id: {})'.format(
id(session_lock),
current_thread().ident))
try:
log.info('Cargando nuestro modelo del zip...')
init_session_objects(request, force=True)
refs = get_session_objects(request)
new_model = Model.load(zipfile_path, refs=refs)
new_model._cache.enabled = False
new_model._schema.register_refs(new_model._schema(), new_model, refs)
# from ..views import implemented_types
# RegisterObject(new_model, request, implemented_types)
log.info('setting active model...')
set_active_model(request, new_model.id)
except Exception:
raise cors_exception(request, HTTPBadRequest, with_stacktrace=True)
finally:
session_lock.release()
log.info(' session lock released (sess:{}, thr_id: {})'.format(
id(session_lock),
current_thread().ident))
# We will want to clean up our temporary zipfile when we are done.
os.remove(zipfile_path)
return cors_response(request, Response('OK'))
def upload_model(request):
'''
Uploads a new model in the form of a zipfile and registers it as the
current active model.
We are generating our own filename instead of trusting
the incoming filename since that might result in insecure paths.
We may want to eventually use something other than /tmp,
and if you write to an untrusted location you will need to do
some extra work to prevent symlink attacks.
'''
clean_session_dir(request)
file_path = process_upload(request, 'new_model')
# Now that we have our file, we will now try to load the model into
# memory.
# Now that we have our file, is it a zipfile?
if not is_savezip_valid(file_path):
raise cors_response(
request, HTTPBadRequest('Incoming file is not a '
'valid zipfile!'))
# now we try to load our model from the zipfile.
gnome_sema = request.registry.settings['py_gnome_semaphore']
gnome_sema.acquire()
log.info('semaphore acquired.')
try:
log.info('loading our model from zip...')
new_model = load(file_path)
new_model._cache.enabled = False
init_session_objects(request, force=True)
RegisterObject(new_model, request)
log.info('setting active model...')
set_active_model(request, new_model.id)
except:
raise cors_exception(request, HTTPBadRequest, with_stacktrace=True)
finally:
gnome_sema.release()
log.info('semaphore released.')
# We will want to clean up our tempfile when we are done.
os.remove(file_path)
return cors_response(request, Response('OK'))
def upload_model(request):
'''
Uploads a new model in the form of a zipfile and registers it as the
current active model.
We are generating our own filename instead of trusting
the incoming filename since that might result in insecure paths.
We may want to eventually use something other than /tmp,
and if you write to an untrusted location you will need to do
some extra work to prevent symlink attacks.
'''
clean_session_dir(request)
file_path = process_upload(request, 'new_model')
# Now that we have our file, we will now try to load the model into
# memory.
# Now that we have our file, is it a zipfile?
if not is_savezip_valid(file_path):
raise cors_response(request, HTTPBadRequest('Incoming file is not a '
'valid zipfile!'))
# now we try to load our model from the zipfile.
gnome_sema = request.registry.settings['py_gnome_semaphore']
gnome_sema.acquire()
log.info('semaphore acquired.')
try:
log.info('loading our model from zip...')
new_model = load(file_path)
new_model._cache.enabled = False
init_session_objects(request, force=True)
RegisterObject(new_model, request)
log.info('setting active model...')
set_active_model(request, new_model.id)
except:
raise cors_exception(request, HTTPBadRequest, with_stacktrace=True)
finally:
gnome_sema.release()
log.info('semaphore released.')
# We will want to clean up our tempfile when we are done.
os.remove(file_path)
return cors_response(request, Response('OK'))
def test_check_max_compress_ratio(self):
'''
create fake zip containing 100 '0' as string. The compression ratio
should be big
'''
badzip = os.path.join(self.here,
'sample_data/badzip_max_compress_ratio.zip')
badfile = 'badcompressratio.json'
with ZipFile(badzip, 'a', compression=ZIP_DEFLATED) as z:
z.writestr(badfile, ''.join(['0'] * 100))
with LogCapture() as l:
assert not is_savezip_valid(badzip)
zi = z.NameToInfo[badfile]
l.check(('gnome.persist.save_load', 'WARNING',
("uncompressed filesize is {0} time compressed filesize."
"_max_compress_ratio must be less than {1}. Rejecting "
"zipfile".format(zi.file_size / zi.compress_size,
save_load._max_compress_ratio))))
def test_check_max_compress_ratio(self):
'''
create fake zip containing 100 '0' as string. The compression ratio
should be big
'''
badzip = os.path.join(self.here,
'sample_data/badzip_max_compress_ratio.zip')
badfile = 'badcompressratio.json'
with ZipFile(badzip, 'a', compression=ZIP_DEFLATED) as z:
z.writestr(badfile, ''.join(['0'] * 100))
with LogCapture() as l:
assert not is_savezip_valid(badzip)
zi = z.NameToInfo[badfile]
l.check(('gnome.persist.save_load',
'WARNING',
("uncompressed filesize is {0} time compressed filesize."
"_max_compress_ratio must be less than {1}. Rejecting "
"zipfile".format(zi.file_size/zi.compress_size,
save_load._max_compress_ratio))))
def test_max_json_filesize(self):
'''
create a fake zip containing
'sample_data/boston_data/MerrimackMassCoastOSSM.json'
change _max_json_filesize 4K
'''
save_load._max_json_filesize = 8 * 1024
badzip = os.path.join(self.here,
'sample_data/badzip_max_json_filesize.zip')
filetoobig = 'filetoobig.json'
with ZipFile(badzip, 'w', compression=ZIP_DEFLATED) as z:
z.write(testdata['boston_data']['cats_ossm'], filetoobig)
with LogCapture() as lc:
assert not is_savezip_valid(badzip)
lc.check(('gnome.persist.save_load', 'WARNING',
'Filesize of {0} is {1}. It must be less than {2}. '
'Rejecting zipfile.'.format(
filetoobig, z.NameToInfo[filetoobig].file_size,
save_load._max_json_filesize)))
save_load._max_json_filesize = 1 * 1024
def test_max_json_filesize(self):
'''
create a fake zip containing
'sample_data/boston_data/MerrimackMassCoastOSSM.json'
change _max_json_filesize 4K
'''
save_load._max_json_filesize = 8 * 1024
badzip = os.path.join(self.here,
'sample_data/badzip_max_json_filesize.zip')
filetoobig = 'filetoobig.json'
with ZipFile(badzip, 'w', compression=ZIP_DEFLATED) as z:
z.write(testdata['boston_data']['cats_ossm'], filetoobig)
with LogCapture() as lc:
assert not is_savezip_valid(badzip)
lc.check(('gnome.persist.save_load',
'WARNING',
'Filesize of {0} is {1}. It must be less than {2}. '
'Rejecting zipfile.'
.format(filetoobig,
z.NameToInfo[filetoobig].file_size,
save_load._max_json_filesize)))
save_load._max_json_filesize = 1 * 1024
def test_invalid_zip(self):
''' invalid zipfile '''
with LogCapture() as lc:
assert not is_savezip_valid('junk.zip')
lc.check(('gnome.persist.save_load', 'WARNING',
'junk.zip is not a valid zipfile'))