def dfmp_get_thumbnail_url(context, data_dict):
''' Generates thumbnail and returns url. '''
_validate(data_dict, 'image_url', 'width', 'height')
import os
import ckan.lib.uploader as uploader
upload_path = uploader.get_storage_path()
# check if we can write a file somewhere
if not upload_path or not os.access(upload_path, os.W_OK):
raise ValidationError('Storage path directory should be specified. It should be writable.')
from PIL import Image, ImageOps
import requests
import io
import hashlib
size = (data_dict['width'], data_dict['height'])
file = io.BytesIO(requests.get(data_dict['image_url']).content)
try:
im = Image.open(file)
thumbnail = ImageOps.fit(
im,
size,
Image.ANTIALIAS
)
except IOError, e:
log.warn(e)
raise NotFound
def archive(self):
access_key = config.get('ckanext.s3archive.access_key')
secret_key = config.get('ckanext.s3archive.secret_key')
bucket_name = config.get('ckanext.s3archive.bucket')
if not access_key:
print 'ckanext.s3archive.access_key config argument not set'
return
if not secret_key:
print 'ckanext.s3archive.secret_key config argument not set'
return
if not bucket_name:
print 'ckanext.s3archive.bucket config argument not set'
return
storage_path = get_storage_path()
if not storage_path:
print 'ckan.storage_path not set in config'
return
resource_path = os.path.join(storage_path, 'resources')
def walk(bucket, dir, files):
for file in files:
full_path = os.path.join(resource_path, dir, file)
if not os.path.isfile(full_path) or full_path.endswith('~'):
continue
key_name = full_path[len(resource_path):]
for key in bucket.list(prefix=key_name.lstrip('/')):
key.delete()
resource_id = key_name.replace('/', '')
resource = model.Resource.get(resource_id)
if not resource:
continue
last_part = resource.url.split('/')[-1]
file_name = munge.munge_filename(last_part)
key_name = key_name + '/' + file_name
key = s3key.Key(bucket)
key.key = key_name
key.set_contents_from_filename(full_path)
print 'Archived %s' % key_name
os.remove(full_path)
conn = s3connection.S3Connection(access_key, secret_key)
bucket = conn.get_bucket(bucket_name)
try:
os.path.walk(resource_path, walk, bucket)
finally:
conn.close()
def filesystem_resource_download(package_type, id, resource_id, filename=None):
"""
A fallback view action to download resources from the
filesystem. A copy of the action from
`ckan.views.resource:download`.
Provide a direct download by either redirecting the user to the url
stored or downloading an uploaded file directly.
"""
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'auth_user_obj': g.userobj
}
preview = request.args.get(u'preview', False)
try:
rsc = get_action(u'resource_show')(context, {u'id': resource_id})
get_action(u'package_show')(context, {u'id': id})
except NotFound:
return abort(404, _(u'Resource not found'))
except NotAuthorized:
return abort(401, _('Unauthorised to read resource %s') % resource_id)
mimetype, enc = mimetypes.guess_type(rsc.get('url', ''))
if rsc.get(u'url_type') == u'upload':
if hasattr(DefaultResourceUpload, 'download'):
upload = DefaultResourceUpload(rsc)
try:
return upload.download(rsc['id'], filename)
except OSError:
# includes FileNotFoundError
return abort(404, _('Resource data not found'))
path = get_storage_path()
storage_path = os.path.join(path, 'resources')
directory = os.path.join(storage_path, resource_id[0:3],
resource_id[3:6])
filepath = os.path.join(directory, resource_id[6:])
if preview:
return flask.send_file(filepath, mimetype=mimetype)
else:
return flask.send_file(filepath)
elif u'url' not in rsc:
return abort(404, _(u'No download is available'))
return redirect(rsc[u'url'])
def __init__(self, object_type, old_filename=None):
""" Setup upload by creating a subdirectory of the storage directory
of name object_type. old_filename is the name of the file in the url
field last time"""
self.storage_path = None
self.filename = None
self.filepath = None
path = get_storage_path()
if not path:
return
self.storage_path = os.path.join(path, 'storage', 'uploads',
'organization')
_make_dirs_if_not_existing(self.storage_path)
self.object_type = object_type
self.old_filename = old_filename
self.old_filepath = None
def get_thumbnail(self, resolution, image):
import ckan.lib.uploader as uploader
import paste.fileapp
import mimetypes
filepath = uploader.get_storage_path() + '/thumbnails/' + resolution + '/' + image
log.warn(filepath)
fileapp = paste.fileapp.FileApp(filepath)
try:
status, headers, app_iter = request.call_application(fileapp)
except OSError:
base.abort(404, _('Resource data not found'))
response.headers.update(dict(headers))
content_type, content_enc = mimetypes.guess_type(image)
if content_type:
response.headers['Content-Type'] = content_type
response.status = status
return app_iter
def file_remove(context, data_dict):
resource_id = data_dict.get('id')
storage_path = uploader.get_storage_path()
directory = os.path.join(storage_path, 'resources', resource_id[0:3],
resource_id[3:6])
filepath = os.path.join(directory, resource_id[6:])
# print('################################## finish delete resource in filestore ##########################')
try:
os.remove(filepath)
# print('remove filepath {0}').format(filepath)
os.removedirs(directory)
# print('remove directory {0}').format(directory)
log.info(u'Resource file in %s has been deleted.' % filepath)
log.debug('Delete {0} resource in filestore'.format(resource_id))
except OSError, e:
log.debug(u'Error: %s - %s.' % (e.filename, e.strerror))
pass
def __init__(self, file_dict):
path = uploader.get_storage_path()
if not path:
self.storage_path = None
return
self.storage_path = os.path.join(path, 'global')
try:
os.makedirs(self.storage_path)
except OSError as e:
# errno 17 is file already exists
if e.errno != 17:
raise
self.filename = os.path.basename(file_dict.get('filename')) if file_dict.get('filename') else None
upload_field_storage = file_dict.pop('upload', None)
if isinstance(upload_field_storage, cgi.FieldStorage):
self._update_filename(upload_field_storage)
self.filename = munge.munge_filename(self.filename)
file_dict['filename'] = self.filename
self.upload_file = upload_field_storage.file
def image_uploader():
'''View function that renders the image uploader form.
Passes `'image_url': 'submitted-image-path'` and `'uploads': {'path',}'`
to template.
'''
try:
uploads_with_path = []
try:
# Get all the images from the home uploader.
path = uploader.get_storage_path()
storage_path = os.path.join(path, 'storage', 'uploads', 'home')
uploads = sorted(os.listdir(storage_path))
# Build the static content paths for each image.
for upload in uploads:
image_path = 'uploads/home/'
value = h.url_for_static('{0}{1}'.format(image_path, upload))
uploads_with_path.append(value)
except OSError:
log.error(
"The uploads directory does not exist yet. Upload an image.")
# This is somewhat odd, toolkit.render calls flask.render_template.
# But toolkit.render unpacks the args properly to access them in the
# template without needing 'extra_vars["data"]'. Lots of time trouble
# shooting to get this combo working.
return toolkit.render('admin/ontario_theme_image_uploader.html',
extra_vars={
'data': {},
'errors': {},
'image_url':
request.args.get("image_url", ''),
'uploads': uploads_with_path
})
except Exception as e:
log.error(e)
def resource_clean(context, data_dict):
""" Remove resource that is not referred by a package.
:context: TODO
:data_dict: TODO
:returns: TODO
"""
check_access('resource_clean', context, data_dict)
model = context['model']
confirmed = data_dict.get('confirmed', False)
res_indb = set([
r[0]
for r in sqlalchemy.sql.select([model.resource_table.c.id]).execute()
])
storage_path = os.path.join(get_storage_path(), 'resources')
dangling = []
for l in check_output(['find', storage_path, '-type', 'f']).split('\n'):
if l:
rid = ''.join(l.split('/')[-3:])
if rid not in res_indb:
dangling.append(l)
deleted = []
if confirmed:
for f in dangling:
os.remove(f)
deleted.append(f)
return {
'deleted': deleted,
'deleted_count': len(deleted),
'dangling': dangling,
'dangling_count': len(dangling)
}
def _merge_ueb_output_pkg_with_input_pkg(self, service_call_results, model_pkg_dataset_id):
"""
Merges the model run output data with the model input package
@param service_call_results: http service response object returned from the app server
@param model_pkg_dataset_id: id of the input model package
@return: success (True or False) and message ( a string value) representing the status to be set for model
package run status attribute
"""
source = 'uebpackage.tasks._merge_ueb_output_pkg_with_input_pkg():'
# save the output model pkg to temp directory
ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir
# create a directory for saving the file
# this will be a dir in the form of: /tmp/ckan/{random_id}
random_id = base.model.types.make_uuid()
destination_dir = os.path.join(ckan_default_dir, random_id)
os.makedirs(destination_dir)
ueb_output_pkg_filename = uebhelper.StringSettings.ueb_output_model_package_default_filename
ueb_output_pkg_file = os.path.join(destination_dir, ueb_output_pkg_filename)
bytes_to_read = 16 * 1024
success = False
message = uebhelper.StringSettings.app_server_job_status_package_retrieval_failed
try:
with open(ueb_output_pkg_file, 'wb') as file_obj:
while True:
data = service_call_results.read(bytes_to_read)
if not data:
break
file_obj.write(data)
except Exception as e:
log.error(source + 'Failed to save ueb model output package zip file to '
'temporary location for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
return success, message
log.info(source + 'ueb model output package zip file was saved to temporary location '
'for model package dataset ID: %s' % model_pkg_dataset_id)
# access the input model pkg zip file
model_pkg_dataset = uebhelper.get_package(model_pkg_dataset_id)
# get the package resource zip file
model_pkg_zip_file = model_pkg_dataset['resources'][0]
# get the storage path of the pkg zip file
ckan_storage_path = os.path.join(uploader.get_storage_path(), 'resources')
model_pkg_zip_file_path = os.path.join(ckan_storage_path, model_pkg_zip_file['id'][0:3],
model_pkg_zip_file['id'][3:6], model_pkg_zip_file['id'][6:])
'''
open the original input zip file in the append mode and then
read the output pkg zip file and append it to the original zip file
'''
is_merge_successful = False
try:
with zip.ZipFile(model_pkg_zip_file_path, 'a') as orig_file_obj:
zip_file_to_merge = zip.ZipFile(ueb_output_pkg_file, 'r')
for fname in zip_file_to_merge.namelist():
orig_file_obj.writestr(fname, zip_file_to_merge.open(fname).read())
is_merge_successful = True
except Exception as e:
log.error(source + 'Failed to merge output model pkg zip file with the input model pkg zip file '
'for model package dataset ID: %s \n '
'Exception: %s' % (model_pkg_dataset_id, e))
# update the model package dataset package_type to complete
if is_merge_successful:
data_dict = {'package_run_status': uebhelper.StringSettings.app_server_job_status_package_retrieval_success, 'package_type': u'Complete'}
success = True
message = uebhelper.StringSettings.app_server_job_status_package_retrieval_success
else:
data_dict = {'package_run_status': uebhelper.StringSettings.app_server_job_status_package_retrieval_failed}
update_msg = 'system auto updated ueb package dataset'
background_task = False
try:
updated_package = uebhelper.update_package(model_pkg_dataset_id, data_dict, update_msg, background_task)
log.info(source + 'UEB model package dataset was updated as a result of '
'receiving model output package for dataset:%s' % updated_package['name'])
except Exception as e:
log.error(source + 'Failed to update UEB model package dataset after '
'receiving model input package for dataset ID:%s \n'
'Exception: %s' % (model_pkg_dataset_id, e))
return success, message
def execute(self, pkg_id):
"""
Executes model input package on app server
@param pkg_id: id of the model input package to be executed
@return: ajax_response object
"""
source = 'uebpackage.uebexecute.execute():'
# get the model package
package = uebhelper.get_package(pkg_id)
ajax_response = uebhelper.AJAXResponse()
ajax_response.success = False
ajax_response.message = "Not a valid UEB model package for execution."
# check that it is a valid model package for execution
if package['type'] != "model-package":
return ajax_response.to_json()
if package.get('package_run_status', None) != 'Not yet submitted':
ajax_response.message = "This UEB model package has already been executed."
return ajax_response.to_json()
# get the package resource zip file
model_pkg_zip_file = package['resources'][0]
# get the storage path
ckan_storage_path = os.path.join(uploader.get_storage_path(), 'resources')
model_pkg_zip_file_path = os.path.join(ckan_storage_path, model_pkg_zip_file['id'][0:3],
model_pkg_zip_file['id'][3:6], model_pkg_zip_file['id'][6:])
# send request to app server
# TODO: read the app server address from config (.ini) file
service_host_address = uebhelper.StringSettings.app_server_host_address
service_request_url = uebhelper.StringSettings.app_server_api_run_ueb_url
connection = httplib.HTTPConnection(service_host_address)
headers = {'Content-Type': 'application/octet-stream', 'Connection': 'Keep-alive'}
# for debugging only
connection.set_debuglevel(1)
# Let's wait for 0.01 second before calling the webservice
# Otherwise sometime we get
# 104 error - Connection Reset by Peer
# ref: http://stackoverflow.com/questions/383738/104-connection-reset-by-peer-socket-error-or-when-does-closing-a-socket-resu
time.sleep(0.01)
with open(model_pkg_zip_file_path, 'r') as ueb_model_pkg_file:
connection.request('POST', service_request_url, ueb_model_pkg_file, headers)
service_call_results = connection.getresponse()
if service_call_results.status == httplib.OK:
log.info(source + 'Request to execute UEB was successful.')
service_response_data = service_call_results.read()
connection.close()
service_response_dict = json.loads(service_response_data)
ueb_run_job_id = service_response_dict.get('RunJobID', None)
response_msg = service_response_dict.get('Message', '')
if not ueb_run_job_id:
log.error(source + 'App server failed to process UEB run request.\n' + response_msg)
ajax_response.message = 'App server failed to process UEB run request.'
return ajax_response.to_json()
else:
connection.close()
log.error(source + 'App server error: Request to run UEB failed: %s' % service_call_results.reason)
ajax_response.message = 'App server failed to process UEB run request.'
return ajax_response.to_json()
if self._update_ueb_model_pkg_run_job_id(pkg_id, ueb_run_job_id) is None:
ajax_response.message = 'Failed to update job run ID for the model package.'
return ajax_response.to_json()
job_status_processing = uebhelper.StringSettings.app_server_job_status_processing
if self._update_ueb_model_pkg_run_status(pkg_id, job_status_processing) is None:
ajax_response.message = 'Failed to update run status for the model package.'
return ajax_response.to_json()
base.session.clear()
tk.c.ueb_run_job_id = ueb_run_job_id
ajax_response.success = True
ajax_response.json_data = job_status_processing
ajax_response.message = "Request to execute UEB was successful."
return ajax_response.to_json()
def make_app(conf, full_stack=True, static_files=True, **app_conf):
"""Create a Pylons WSGI application and return it
``conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# Configure the Pylons environment
load_environment(conf, app_conf)
# The Pylons WSGI app
app = PylonsApp()
# set pylons globals
app_globals.reset()
for plugin in PluginImplementations(IMiddleware):
app = plugin.make_middleware(app, config)
# Routing/Session/Cache Middleware
app = RoutesMiddleware(app, config['routes.map'])
# we want to be able to retrieve the routes middleware to be able to update
# the mapper. We store it in the pylons config to allow this.
config['routes.middleware'] = app
app = SessionMiddleware(app, config)
app = CacheMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
#app = QueueLogMiddleware(app)
# Fanstatic
if asbool(config.get('debug', False)):
fanstatic_config = {
'versioning': True,
'recompute_hashes': True,
'minified': False,
'bottom': True,
'bundle': False,
}
else:
fanstatic_config = {
'versioning': True,
'recompute_hashes': False,
'minified': True,
'bottom': True,
'bundle': True,
}
app = Fanstatic(app, **fanstatic_config)
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, conf, **config['pylons.errorware'])
# Display error documents for 401, 403, 404 status codes (and
# 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app, [400, 404])
else:
app = StatusCodeRedirect(app, [400, 404, 500])
# Initialize repoze.who
who_parser = WhoConfig(conf['here'])
who_parser.parse(open(app_conf['who.config_file']))
if asbool(config.get('openid_enabled', 'true')):
from repoze.who.plugins.openid.identification import OpenIdIdentificationPlugin
# Monkey patches for repoze.who.openid
# Fixes #1659 - enable log-out when CKAN mounted at non-root URL
from ckan.lib import repoze_patch
OpenIdIdentificationPlugin.identify = repoze_patch.identify
OpenIdIdentificationPlugin.redirect_to_logged_in = repoze_patch.redirect_to_logged_in
OpenIdIdentificationPlugin._redirect_to_loginform = repoze_patch._redirect_to_loginform
OpenIdIdentificationPlugin.challenge = repoze_patch.challenge
who_parser.identifiers = [i for i in who_parser.identifiers if \
not isinstance(i, OpenIdIdentificationPlugin)]
who_parser.challengers = [i for i in who_parser.challengers if \
not isinstance(i, OpenIdIdentificationPlugin)]
app = PluggableAuthenticationMiddleware(app,
who_parser.identifiers,
who_parser.authenticators,
who_parser.challengers,
who_parser.mdproviders,
who_parser.request_classifier,
who_parser.challenge_decider,
logging.getLogger('repoze.who'),
logging.WARN, # ignored
who_parser.remote_user_key,
)
# Establish the Registry for this application
app = RegistryManager(app)
app = I18nMiddleware(app, config)
if asbool(static_files):
# Serve static files
static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \
else int(config.get('ckan.static_max_age', 3600))
static_app = StaticURLParser(config['pylons.paths']['static_files'],
cache_max_age=static_max_age)
static_parsers = [static_app, app]
storage_directory = uploader.get_storage_path()
if storage_directory:
path = os.path.join(storage_directory, 'storage')
try:
os.makedirs(path)
except OSError, e:
## errno 17 is file already exists
if e.errno != 17:
raise
storage_app = StaticURLParser(path,
cache_max_age=static_max_age)
static_parsers.insert(0, storage_app)
# Configurable extra static file paths
extra_static_parsers = []
for public_path in config.get('extra_public_paths', '').split(','):
if public_path.strip():
extra_static_parsers.append(
StaticURLParser(public_path.strip(),
cache_max_age=static_max_age)
)
app = Cascade(extra_static_parsers + static_parsers)
def configure(self, config):
self.gapi_key = config.get('ckanext.geoview.gapi_key', None)
self.storage_path = get_storage_path()
self.geoserver_url = config.get('dadosabertos.geoserver.url', None)
self.geoserver_user = config.get('dadosabertos.geoserver.user', None)
self.geoserver_password = config.get('dadosabertos.geoserver.password', None)
def make_pylons_stack(conf, full_stack=True, static_files=True,
**app_conf):
"""Create a Pylons WSGI application and return it
``conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# The Pylons WSGI app
app = pylons_app = CKANPylonsApp()
for plugin in PluginImplementations(IMiddleware):
app = plugin.make_middleware(app, config)
# Routing/Session/Cache Middleware
app = RoutesMiddleware(app, config['routes.map'])
# we want to be able to retrieve the routes middleware to be able to update
# the mapper. We store it in the pylons config to allow this.
config['routes.middleware'] = app
app = SessionMiddleware(app, config)
app = CacheMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
# app = QueueLogMiddleware(app)
if asbool(config.get('ckan.use_pylons_response_cleanup_middleware',
True)):
app = execute_on_completion(app, config,
cleanup_pylons_response_string)
# Fanstatic
if asbool(config.get('debug', False)):
fanstatic_config = {
'versioning': True,
'recompute_hashes': True,
'minified': False,
'bottom': True,
'bundle': False,
}
else:
fanstatic_config = {
'versioning': True,
'recompute_hashes': False,
'minified': True,
'bottom': True,
'bundle': True,
}
app = Fanstatic(app, **fanstatic_config)
for plugin in PluginImplementations(IMiddleware):
try:
app = plugin.make_error_log_middleware(app, config)
except AttributeError:
log.critical('Middleware class {0} is missing the method'
'make_error_log_middleware.'
.format(plugin.__class__.__name__))
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, conf, **config['pylons.errorware'])
# Display error documents for 400, 403, 404 status codes (and
# 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app, [400, 403, 404])
else:
app = StatusCodeRedirect(app, [400, 403, 404, 500])
# Initialize repoze.who
who_parser = WhoConfig(conf['here'])
who_parser.parse(open(app_conf['who.config_file']))
app = PluggableAuthenticationMiddleware(
app,
who_parser.identifiers,
who_parser.authenticators,
who_parser.challengers,
who_parser.mdproviders,
who_parser.request_classifier,
who_parser.challenge_decider,
logging.getLogger('repoze.who'),
logging.WARN, # ignored
who_parser.remote_user_key
)
# Establish the Registry for this application
app = RegistryManager(app)
app = common_middleware.I18nMiddleware(app, config)
if asbool(static_files):
# Serve static files
static_max_age = None if not asbool(
config.get('ckan.cache_enabled')) \
else int(config.get('ckan.static_max_age', 3600))
static_app = StaticURLParser(
config['pylons.paths']['static_files'],
cache_max_age=static_max_age)
static_parsers = [static_app, app]
storage_directory = uploader.get_storage_path()
if storage_directory:
path = os.path.join(storage_directory, 'storage')
try:
os.makedirs(path)
except OSError, e:
# errno 17 is file already exists
if e.errno != 17:
raise
storage_app = StaticURLParser(path, cache_max_age=static_max_age)
static_parsers.insert(0, storage_app)
# Configurable extra static file paths
extra_static_parsers = []
for public_path in config.get(
'extra_public_paths', '').split(','):
if public_path.strip():
extra_static_parsers.append(
StaticURLParser(public_path.strip(),
cache_max_age=static_max_age)
)
app = Cascade(extra_static_parsers + static_parsers)
def __init__(self, resource):
path = get_storage_path()
config_mimetype_guess = config.get('ckan.mimetype_guess', 'file_ext')
if not path:
self.storage_path = None
return
self.storage_path = os.path.join(path, 'resources')
try:
os.makedirs(self.storage_path)
except OSError as e:
# errno 17 is file already exists
if e.errno != 17:
raise
self.filename = None
self.mimetype = None
url = resource.get('url')
upload_field_storage = resource.pop('upload', None)
self.clear = resource.pop('clear_upload', None)
if url and config_mimetype_guess == 'file_ext':
self.mimetype = mimetypes.guess_type(url)[0]
if isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):
self.filesize = 0 # bytes
self.filename = upload_field_storage.filename
# MODIFICATION START
self.filename = secure_filename(self.filename) # Overkill but I
# trust werkzueg over ckan.
# MODIFICATION END
self.filename = munge.munge_filename(self.filename)
resource['url'] = self.filename
resource['url_type'] = 'upload'
resource['last_modified'] = datetime.datetime.utcnow()
self.upload_file = _get_underlying_file(upload_field_storage)
self.upload_file.seek(0, os.SEEK_END)
self.filesize = self.upload_file.tell()
# go back to the beginning of the file buffer
self.upload_file.seek(0, os.SEEK_SET)
# MODIFICATION START
# Note: If resubmitting a failed form without clearing the file
# the ResourceUpload.upload function would be called skipping the
# init call.
if not allowed_file(self.filename):
log.error('Upload: Invalid upload file format.{}'.format(
self.filename))
# remove file - by default a resource can be added without any
# values
resource['url'] = None
resource['url_type'] = ''
raise logic.ValidationError({
'upload':
['Invalid upload file format, file has been removed.']
})
# MODIFICATION END
# check if the mimetype failed from guessing with the url
if not self.mimetype and config_mimetype_guess == 'file_ext':
self.mimetype = mimetypes.guess_type(self.filename)[0]
if not self.mimetype and config_mimetype_guess == 'file_contents':
try:
self.mimetype = magic.from_buffer(self.upload_file.read(),
mime=True)
self.upload_file.seek(0, os.SEEK_SET)
except IOError as e:
# Not that important if call above fails
self.mimetype = None
elif self.clear:
resource['url_type'] = ''
def make_flask_stack(conf: Union[Config, CKANConfig]) -> CKANApp:
""" This has to pass the flask app through all the same middleware that
Pylons used """
root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
debug = asbool(conf.get('debug', conf.get('DEBUG', False)))
testing = asbool(conf.get('testing', conf.get('TESTING', False)))
app = flask_app = CKANFlask(__name__, static_url_path='')
# Register storage for accessing group images, site logo, etc.
storage_folder = []
storage = uploader.get_storage_path()
if storage:
storage_folder = [os.path.join(storage, 'storage')]
# Static files folders (core and extensions)
public_folder = config.get_value(u'ckan.base_public_folder')
app.static_folder = config.get_value('extra_public_paths').split(',') + [
os.path.join(root, public_folder)
] + storage_folder
app.jinja_options = jinja_extensions.get_jinja_env_options()
app.jinja_env.policies['ext.i18n.trimmed'] = True
app.debug = debug
app.testing = testing
app.template_folder = os.path.join(root, 'templates')
app.app_ctx_globals_class = CKAN_AppCtxGlobals
app.url_rule_class = CKAN_Rule
# Update Flask config with the CKAN values. We use the common config
# object as values might have been modified on `load_environment`
if config:
app.config.update(config)
else:
app.config.update(conf)
# Do all the Flask-specific stuff before adding other middlewares
# Secret key needed for flask-debug-toolbar and sessions
if not app.config.get('SECRET_KEY'):
app.config['SECRET_KEY'] = config.get_value('beaker.session.secret')
if not app.config.get('SECRET_KEY'):
raise RuntimeError(u'You must provide a value for the secret key'
' with the SECRET_KEY config option')
root_path = config.get_value('ckan.root_path')
if debug:
from flask_debugtoolbar import DebugToolbarExtension
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
debug_ext = DebugToolbarExtension()
# register path that includes `ckan.site_root` before
# initializing debug app. In such a way, our route receives
# higher precedence.
# TODO: After removal of Pylons code, switch to
# `APPLICATION_ROOT` config value for flask application. Right
# now it's a bad option because we are handling both pylons
# and flask urls inside helpers and splitting this logic will
# bring us tons of headache.
if root_path:
app.add_url_rule(
root_path.replace('{{LANG}}', '').rstrip('/') +
'/_debug_toolbar/static/<path:filename>',
'_debug_toolbar.static', debug_ext.send_static_file)
debug_ext.init_app(app)
from werkzeug.debug import DebuggedApplication
app.wsgi_app = DebuggedApplication(app.wsgi_app, True)
# Use Beaker as the Flask session interface
class BeakerSessionInterface(SessionInterface):
def open_session(self, app: Any, request: Any):
if 'beaker.session' in request.environ:
return request.environ['beaker.session']
def save_session(self, app: Any, session: Any, response: Any):
session.save()
namespace = 'beaker.session.'
session_opts = {
k.replace('beaker.', ''): v
for k, v in config.items() if k.startswith(namespace)
}
if (not session_opts.get('session.data_dir')
and session_opts.get('session.type', 'file') == 'file'):
cache_dir = conf.get('cache_dir') or conf.get('cache.dir')
session_opts['session.data_dir'] = '{data_dir}/sessions'.format(
data_dir=cache_dir)
app.wsgi_app = RootPathMiddleware(app.wsgi_app)
app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts)
app.session_interface = BeakerSessionInterface()
# Add Jinja2 extensions and filters
app.jinja_env.filters['empty_and_escape'] = \
jinja_extensions.empty_and_escape
# Common handlers for all requests
#
# flask types do not mention that it's possible to return a response from
# the `before_request` callback
app.before_request(ckan_before_request)
app.after_request(ckan_after_request)
# Template context processors
app.context_processor(helper_functions)
app.context_processor(c_object)
app.context_processor(_ungettext_alias)
# Babel
_ckan_i18n_dir = i18n.get_ckan_i18n_dir()
pairs = [cast("tuple[str, str]", (_ckan_i18n_dir, u'ckan'))
] + [(p.i18n_directory(), p.i18n_domain())
for p in reversed(list(PluginImplementations(ITranslation)))]
i18n_dirs, i18n_domains = zip(*pairs)
app.config[u'BABEL_TRANSLATION_DIRECTORIES'] = ';'.join(i18n_dirs)
app.config[u'BABEL_DOMAIN'] = 'ckan'
app.config[u'BABEL_MULTIPLE_DOMAINS'] = ';'.join(i18n_domains)
app.config[u'BABEL_DEFAULT_TIMEZONE'] = str(h.get_display_timezone())
babel = CKANBabel(app)
babel.localeselector(get_locale)
# WebAssets
_setup_webassets(app)
# Auto-register all blueprints defined in the `views` folder
_register_core_blueprints(app)
_register_error_handler(app)
# Set up each IBlueprint extension as a Flask Blueprint
for plugin in PluginImplementations(IBlueprint):
plugin_blueprints = plugin.get_blueprint()
if not isinstance(plugin_blueprints, list):
plugin_blueprints = [plugin_blueprints]
for blueprint in plugin_blueprints:
app.register_extension_blueprint(blueprint)
lib_plugins.register_package_blueprints(app)
lib_plugins.register_group_blueprints(app)
# Start other middleware
for plugin in PluginImplementations(IMiddleware):
app = plugin.make_middleware(app, config)
for plugin in PluginImplementations(IMiddleware):
try:
app = plugin.make_error_log_middleware(app, config)
except AttributeError:
log.critical('Middleware class {0} is missing the method'
'make_error_log_middleware.'.format(
plugin.__class__.__name__))
# Initialize flask-login
login_manager = LoginManager()
login_manager.init_app(app)
# make anonymous_user an instance of CKAN custom class
login_manager.anonymous_user = model.AnonymousUser
# The name of the view to redirect to when the user needs to log in.
login_manager.login_view = config.get_value("ckan.auth.login_view")
@login_manager.user_loader
def load_user(user_id: str) -> Optional["model.User"]: # type: ignore
return model.User.get(user_id)
@login_manager.request_loader
def load_user_from_request(request): # type: ignore
user = _get_user_for_apitoken()
return user
# Update the main CKAN config object with the Flask specific keys
# that were set here or autogenerated
flask_config_keys = set(flask_app.config.keys()) - set(config.keys())
for key in flask_config_keys:
config[key] = flask_app.config[key]
# Prevent the host from request to be added to the new header location.
app = HostHeaderMiddleware(app)
app = I18nMiddleware(app)
if config.get_value('ckan.tracking_enabled'):
app = TrackingMiddleware(app, config)
# Add a reference to the actual Flask app so it's easier to access
# type_ignore_reason: custom attribute
app._wsgi_app = flask_app # type: ignore
return app
