def _check_upload_file(self, request, file_types, file_type_error_msg,
event, file_size_threshold):
"""
Check the type and size of uploaded file. If the file type is
unexpected or the size exceeds the threshold, log the error and return
to browser, otherwise, return None.
"""
# Check invalid file types
file_type_error = False
file_type = [
ft for ft in file_types if any(
str(request.POST['file'].file).lower().endswith(ext)
for ext in file_types[ft]['extension'])
]
# Check extension
if not file_type:
file_type_error = True
else:
file_type = file_type[0]
# Check mimetypes
if request.POST['file'].file.content_type not in file_types[
file_type]['mimetypes']:
file_type_error = True
else:
if 'magic' in file_types[file_type]:
# Check magic number
headers = file_types[file_type]['magic']
if request.POST['file'].file.read(
len(headers[0]) / 2).encode('hex') not in headers:
file_type_error = True
request.POST['file'].file.seek(0)
if file_type_error:
response = Response()
tracker.emit(event, {'uploadedFileName': 'FILE_TYPE_ERROR'})
response.status = 415
response.body = json.dumps({'error': file_type_error_msg})
response.headers['Content-Type'] = 'application/json'
return response
# Check whether file size exceeds threshold (30MB)
if request.POST['file'].file.size > file_size_threshold:
response = Response()
tracker.emit(event, {'uploadedFileName': 'FILE_SIZE_ERROR'})
response.status = 413
response.body = json.dumps({
'error':
self.ugettext('Size of uploaded file exceeds threshold')
})
response.headers['Content-Type'] = 'application/json'
return response
return file_type
def import_resources(self, request, _suffix=''):
"""
Import resources into the recommender.
"""
response = Response()
response.headers['Content-Type'] = 'application/json'
if not self.get_user_is_staff():
response.status = 403
response.body = json.dumps({'error': self.ugettext('Only staff can import resources')})
tracker.emit('import_resources', {'Status': 'NOT_A_STAFF'})
return response
# Check invalid file types
file_types = {
'json': {
'extension': [".json"],
'mimetypes': ['application/json', 'text/json', 'text/x-json']
}
}
file_type_error_msg = self.ugettext('Please submit the JSON file obtained with the download resources button')
result = self._check_upload_file(
request, file_types, file_type_error_msg, 'import_resources', 31457280
)
if isinstance(result, Response):
return result
try:
data = json.load(request.POST['file'].file)
self.flagged_accum_resources = data['flagged_accum_resources']
self.endorsed_recommendation_reasons = data['endorsed_recommendation_reasons']
self.endorsed_recommendation_ids = data['endorsed_recommendation_ids']
if 'removed_recommendations' in data:
self.removed_recommendations = data_structure_upgrade(data['removed_recommendations'])
data['removed_recommendations'] = self.removed_recommendations
self.recommendations = data_structure_upgrade(data['recommendations'])
data['recommendations'] = self.recommendations
tracker.emit('import_resources', {'Status': 'SUCCESS', 'data': data})
response.body = json.dumps(data, sort_keys=True)
response.status = 200
return response
except (ValueError, KeyError):
response.status = 415
response.body = json.dumps(
{'error': self.ugettext('Please submit the JSON file obtained with the download resources button')}
)
tracker.emit('import_resources', {'Status': 'FILE_FORMAT_ERROR'})
return response
except IOError:
return self._raise_pyfs_error('import_resources')
def import_resources(self, request, _suffix=''):
"""
Import resources into the recommender.
"""
response = Response()
response.headers['Content-Type'] = 'application/json'
if not self.get_user_is_staff():
response.status = 403
response.body = json.dumps({'error': self.ugettext('Only staff can import resources')})
tracker.emit('import_resources', {'Status': 'NOT_A_STAFF'})
return response
# Check invalid file types
file_types = {
'json': {
'extension': [".json"],
'mimetypes': ['application/json', 'text/json', 'text/x-json']
}
}
file_type_error_msg = self.ugettext('Please submit the JSON file obtained with the download resources button')
result = self._check_upload_file(
request, file_types, file_type_error_msg, 'import_resources', 31457280
)
if isinstance(result, Response):
return result
try:
data = json.load(request.POST['file'].file)
self.flagged_accum_resources = data['flagged_accum_resources']
self.endorsed_recommendation_reasons = data['endorsed_recommendation_reasons']
self.endorsed_recommendation_ids = data['endorsed_recommendation_ids']
if 'removed_recommendations' in data:
self.removed_recommendations = data_structure_upgrade(data['removed_recommendations'])
data['removed_recommendations'] = self.removed_recommendations
self.recommendations = data_structure_upgrade(data['recommendations'])
data['recommendations'] = self.recommendations
tracker.emit('import_resources', {'Status': 'SUCCESS', 'data': data})
response.body = json.dumps(data, sort_keys=True)
response.status = 200
return response
except (ValueError, KeyError):
response.status = 415
response.body = json.dumps(
{'error': self.ugettext('Please submit the JSON file obtained with the download resources button')}
)
tracker.emit('import_resources', {'Status': 'FILE_FORMAT_ERROR'})
return response
except IOError:
return self._raise_pyfs_error('import_resources')
def _check_upload_file(self, request, file_types, file_type_error_msg, event, file_size_threshold):
"""
Check the type and size of uploaded file. If the file type is
unexpected or the size exceeds the threshold, log the error and return
to browser, otherwise, return None.
"""
# Check invalid file types
file_type_error = False
file_type = [ft for ft in file_types
if any(str(request.POST['file'].file).lower().endswith(ext)
for ext in file_types[ft]['extension'])]
# Check extension
if not file_type:
file_type_error = True
else:
file_type = file_type[0]
# Check mimetypes
if request.POST['file'].file.content_type not in file_types[file_type]['mimetypes']:
file_type_error = True
else:
if 'magic' in file_types[file_type]:
# Check magic number
headers = file_types[file_type]['magic']
if request.POST['file'].file.read(len(headers[0]) / 2).encode('hex') not in headers:
file_type_error = True
request.POST['file'].file.seek(0)
if file_type_error:
response = Response()
tracker.emit(event, {'uploadedFileName': 'FILE_TYPE_ERROR'})
response.status = 415
response.body = json.dumps({'error': file_type_error_msg})
response.headers['Content-Type'] = 'application/json'
return response
# Check whether file size exceeds threshold (30MB)
if request.POST['file'].file.size > file_size_threshold:
response = Response()
tracker.emit(event, {'uploadedFileName': 'FILE_SIZE_ERROR'})
response.status = 413
response.body = json.dumps({'error': self.ugettext('Size of uploaded file exceeds threshold')})
response.headers['Content-Type'] = 'application/json'
return response
return file_type
def upload_screenshot(self, request, _suffix=''): # pylint: disable=unused-argument
"""
Upload a screenshot for an entry of resource as a preview (typically to S3 or filesystem).
Args:
request: HTTP POST request
request.POST['file'].file: the file to be uploaded
Returns:
response: HTTP response
response.body (response.responseText): name of the uploaded file
We validate that this is a valid JPG, GIF, or PNG by checking magic number, mimetype,
and extension all correspond. We also limit to 30MB. We save the file under its MD5
hash to (1) avoid name conflicts, (2) avoid race conditions and (3) save space.
"""
# Check invalid file types
image_types = {
'jpeg': {
'extension': [".jpeg", ".jpg"],
'mimetypes': ['image/jpeg', 'image/pjpeg'],
'magic': ["ffd8"]
},
'png': {
'extension': [".png"],
'mimetypes': ['image/png'],
'magic': ["89504e470d0a1a0a"]
},
'gif': {
'extension': [".gif"],
'mimetypes': ['image/gif'],
'magic': ["474946383961", "474946383761"]
}
}
file_type_error_msg = 'Please upload an image in GIF/JPG/PNG'
result = self._check_upload_file(
request, image_types, file_type_error_msg, 'upload_screenshot', 31457280
)
if isinstance(result, Response):
return result
try:
content = request.POST['file'].file.read()
file_id = hashlib.md5(content).hexdigest()
file_name = (file_id + '.' + result)
fhwrite = self.fs.open(file_name, "wb")
fhwrite.write(content)
fhwrite.close()
except IOError:
return self._raise_pyfs_error('upload_screenshot')
response = Response()
response.body = json.dumps({'file_name': str("fs://" + file_name)})
response.headers['Content-Type'] = 'application/json'
tracker.emit('upload_screenshot',
{'uploadedFileName': response.body})
response.status = 200
return response
def upload_screenshot(self, request, _suffix=''): # pylint: disable=unused-argument
"""
Upload a screenshot for an entry of resource as a preview (typically to S3 or filesystem).
Args:
request: HTTP POST request
request.POST['file'].file: the file to be uploaded
Returns:
response: HTTP response
response.body (response.responseText): name of the uploaded file
We validate that this is a valid JPG, GIF, or PNG by checking magic number, mimetype,
and extension all correspond. We also limit to 30MB. We save the file under its MD5
hash to (1) avoid name conflicts, (2) avoid race conditions and (3) save space.
"""
# Check invalid file types
image_types = {
'jpeg': {
'extension': [".jpeg", ".jpg"],
'mimetypes': ['image/jpeg', 'image/pjpeg'],
'magic': ["ffd8"]
},
'png': {
'extension': [".png"],
'mimetypes': ['image/png'],
'magic': ["89504e470d0a1a0a"]
},
'gif': {
'extension': [".gif"],
'mimetypes': ['image/gif'],
'magic': ["474946383961", "474946383761"]
}
}
file_type_error_msg = 'Please upload an image in GIF/JPG/PNG'
result = self._check_upload_file(
request, image_types, file_type_error_msg, 'upload_screenshot', 31457280
)
if isinstance(result, Response):
return result
try:
content = request.POST['file'].file.read()
file_id = hashlib.md5(content).hexdigest()
file_name = u'{}.{}'.format(file_id, result)
fhwrite = self.fs.open(file_name, "wb")
fhwrite.write(content)
fhwrite.close()
except IOError:
return self._raise_pyfs_error('upload_screenshot')
response = Response()
response.body = json.dumps({'file_name': str("fs://" + file_name)})
response.headers['Content-Type'] = 'application/json'
tracker.emit('upload_screenshot',
{'uploadedFileName': response.body})
response.status = 200
return response
def test_location():
res = Response()
res.status = '301'
res.location = '/test.html'
assert res.location == '/test.html'
req = Request.blank('/')
assert req.get_response(res).location == 'http://localhost/test.html'
res.location = '/test2.html'
assert req.get_response(res).location == 'http://localhost/test2.html'
def test_location():
res = Response()
res.status = "301"
res.location = "/test.html"
assert res.location == "/test.html"
req = Request.blank("/")
assert req.get_response(res).location == "http://localhost/test.html"
res.location = "/test2.html"
assert req.get_response(res).location == "http://localhost/test2.html"
def test_location():
res = Response()
res.status = '301'
res.location = '/test.html'
assert res.location == '/test.html'
req = Request.blank('/')
assert req.get_response(res).location == 'http://localhost/test.html'
res.location = '/test2.html'
assert req.get_response(res).location == 'http://localhost/test2.html'
def test_location():
res = Response()
res.status = "301"
res.location = "/test.html"
assert res.location == "/test.html"
req = Request.blank("/")
assert req.get_response(res).location == "http://localhost/test.html"
res.location = "/test2.html"
assert req.get_response(res).location == "http://localhost/test2.html"
def _raise_pyfs_error(self, event):
"""
Log and return an error if the pyfs is not properly set.
"""
response = Response()
error = self.ugettext('The configuration of pyfs is not properly set')
tracker.emit(event, {'uploadedFileName': 'IMPROPER_FS_SETUP'})
response.status = 404
response.body = json.dumps({'error': error})
response.headers['Content-Type'] = 'application/json'
return response
def _raise_pyfs_error(self, event):
"""
Log and return an error if the pyfs is not properly set.
"""
response = Response()
error = self.ugettext('The configuration of pyfs is not properly set')
tracker.emit(event, {'uploadedFileName': 'IMPROPER_FS_SETUP'})
response.status = 404
response.body = json.dumps({'error': error})
response.headers['Content-Type'] = 'application/json'
return response
def test_location_unicode():
environ = {
'REQUEST_METHOD': 'GET',
'wsgi.url_scheme': 'http',
'HTTP_HOST': u'test.com',
}
res = Response()
res.status = '301'
res.location = '/test.html'
def start_response(status, headerlist):
for (header, val) in headerlist:
if header.lower() == 'location':
assert val == 'http://test.com/test.html'
assert isinstance(val, str)
res(environ, start_response)
def test_location_unicode():
environ = {
"REQUEST_METHOD": "GET",
"wsgi.url_scheme": "http",
"HTTP_HOST": u"test.com",
}
res = Response()
res.status = "301"
res.location = "/test.html"
def start_response(status, headerlist):
for (header, val) in headerlist:
if header.lower() == "location":
assert val == "http://test.com/test.html"
assert isinstance(val, str)
res(environ, start_response)
def test_location_unicode():
environ = {
"REQUEST_METHOD": "GET",
"wsgi.url_scheme": "http",
"HTTP_HOST": u"test.com",
}
res = Response()
res.status = "301"
res.location = "/test.html"
def start_response(status, headerlist):
for (header, val) in headerlist:
if header.lower() == "location":
assert val == "http://test.com/test.html"
assert isinstance(val, str)
res(environ, start_response)
def test_location_unicode():
environ = {
'REQUEST_METHOD': 'GET',
'wsgi.url_scheme': 'http',
'HTTP_HOST': u'test.com',
}
res = Response()
res.status = '301'
res.location = '/test.html'
def start_response(status, headerlist):
for (header, val) in headerlist:
if header.lower() == 'location':
assert val == 'http://test.com/test.html'
assert isinstance(val, str)
res(environ, start_response)
def test_response_ok(self):
'''Test case that ensures response object behaves as expected. If this pass it guarantees webob version does not
break fantastico functionality.'''
response = Response()
self.assertEqual(200, response.status_code)
self.assertEqual("text/html", response.content_type)
response.charset = "utf8"
self.assertEqual("utf8", response.charset)
response.text = "test content"
self.assertEqual(b"test content", response.body)
response.body = b"test content"
self.assertEqual(b"test content", response.body)
response.status = 404
self.assertEqual(404, response.status_code)
response.content_type = "application/json"
self.assertEqual("application/json", response.content_type)
def test_response_ok(self):
'''Test case that ensures response object behaves as expected. If this pass it guarantees webob version does not
break fantastico functionality.'''
response = Response()
self.assertEqual(200, response.status_code)
self.assertEqual("text/html", response.content_type)
response.charset = "utf8"
self.assertEqual("utf8", response.charset)
response.text = "test content"
self.assertEqual(b"test content", response.body)
response.body = b"test content"
self.assertEqual(b"test content", response.body)
response.status = 404
self.assertEqual(404, response.status_code)
response.content_type = "application/json"
self.assertEqual("application/json", response.content_type)
def test_set_status():
res = Response()
res.status = "200"
assert res.status == "200 OK"
with pytest.raises(TypeError):
setattr(res, 'status', (200,))
def test_set_status():
res = Response()
res.status = "200"
eq_(res.status, "200 OK")
assert_raises(TypeError, setattr, res, 'status', (200,))
def test_set_status():
res = Response()
res.status = "200"
eq_(res.status, "200 OK")
assert_raises(TypeError, setattr, res, "status", float(200))
def test_set_status():
res = Response()
res.status = "200"
eq_(res.status, "200 OK")
assert_raises(TypeError, setattr, res, 'status', (200, ))
