def dfmp_get_thumbnail_url(context, data_dict): ''' Generates thumbnail and returns url. ''' _validate(data_dict, 'image_url', 'width', 'height') import os import ckan.lib.uploader as uploader upload_path = uploader.get_storage_path() # check if we can write a file somewhere if not upload_path or not os.access(upload_path, os.W_OK): raise ValidationError('Storage path directory should be specified. It should be writable.') from PIL import Image, ImageOps import requests import io import hashlib size = (data_dict['width'], data_dict['height']) file = io.BytesIO(requests.get(data_dict['image_url']).content) try: im = Image.open(file) thumbnail = ImageOps.fit( im, size, Image.ANTIALIAS ) except IOError, e: log.warn(e) raise NotFound
def archive(self): access_key = config.get('ckanext.s3archive.access_key') secret_key = config.get('ckanext.s3archive.secret_key') bucket_name = config.get('ckanext.s3archive.bucket') if not access_key: print 'ckanext.s3archive.access_key config argument not set' return if not secret_key: print 'ckanext.s3archive.secret_key config argument not set' return if not bucket_name: print 'ckanext.s3archive.bucket config argument not set' return storage_path = get_storage_path() if not storage_path: print 'ckan.storage_path not set in config' return resource_path = os.path.join(storage_path, 'resources') def walk(bucket, dir, files): for file in files: full_path = os.path.join(resource_path, dir, file) if not os.path.isfile(full_path) or full_path.endswith('~'): continue key_name = full_path[len(resource_path):] for key in bucket.list(prefix=key_name.lstrip('/')): key.delete() resource_id = key_name.replace('/', '') resource = model.Resource.get(resource_id) if not resource: continue last_part = resource.url.split('/')[-1] file_name = munge.munge_filename(last_part) key_name = key_name + '/' + file_name key = s3key.Key(bucket) key.key = key_name key.set_contents_from_filename(full_path) print 'Archived %s' % key_name os.remove(full_path) conn = s3connection.S3Connection(access_key, secret_key) bucket = conn.get_bucket(bucket_name) try: os.path.walk(resource_path, walk, bucket) finally: conn.close()
def filesystem_resource_download(package_type, id, resource_id, filename=None): """ A fallback view action to download resources from the filesystem. A copy of the action from `ckan.views.resource:download`. Provide a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = { u'model': model, u'session': model.Session, u'user': g.user, u'auth_user_obj': g.userobj } preview = request.args.get(u'preview', False) try: rsc = get_action(u'resource_show')(context, {u'id': resource_id}) get_action(u'package_show')(context, {u'id': id}) except NotFound: return abort(404, _(u'Resource not found')) except NotAuthorized: return abort(401, _('Unauthorised to read resource %s') % resource_id) mimetype, enc = mimetypes.guess_type(rsc.get('url', '')) if rsc.get(u'url_type') == u'upload': if hasattr(DefaultResourceUpload, 'download'): upload = DefaultResourceUpload(rsc) try: return upload.download(rsc['id'], filename) except OSError: # includes FileNotFoundError return abort(404, _('Resource data not found')) path = get_storage_path() storage_path = os.path.join(path, 'resources') directory = os.path.join(storage_path, resource_id[0:3], resource_id[3:6]) filepath = os.path.join(directory, resource_id[6:]) if preview: return flask.send_file(filepath, mimetype=mimetype) else: return flask.send_file(filepath) elif u'url' not in rsc: return abort(404, _(u'No download is available')) return redirect(rsc[u'url'])
def __init__(self, object_type, old_filename=None): """ Setup upload by creating a subdirectory of the storage directory of name object_type. old_filename is the name of the file in the url field last time""" self.storage_path = None self.filename = None self.filepath = None path = get_storage_path() if not path: return self.storage_path = os.path.join(path, 'storage', 'uploads', 'organization') _make_dirs_if_not_existing(self.storage_path) self.object_type = object_type self.old_filename = old_filename self.old_filepath = None
def get_thumbnail(self, resolution, image): import ckan.lib.uploader as uploader import paste.fileapp import mimetypes filepath = uploader.get_storage_path() + '/thumbnails/' + resolution + '/' + image log.warn(filepath) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: base.abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type(image) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter
def file_remove(context, data_dict): resource_id = data_dict.get('id') storage_path = uploader.get_storage_path() directory = os.path.join(storage_path, 'resources', resource_id[0:3], resource_id[3:6]) filepath = os.path.join(directory, resource_id[6:]) # print('################################## finish delete resource in filestore ##########################') try: os.remove(filepath) # print('remove filepath {0}').format(filepath) os.removedirs(directory) # print('remove directory {0}').format(directory) log.info(u'Resource file in %s has been deleted.' % filepath) log.debug('Delete {0} resource in filestore'.format(resource_id)) except OSError, e: log.debug(u'Error: %s - %s.' % (e.filename, e.strerror)) pass
def __init__(self, file_dict): path = uploader.get_storage_path() if not path: self.storage_path = None return self.storage_path = os.path.join(path, 'global') try: os.makedirs(self.storage_path) except OSError as e: # errno 17 is file already exists if e.errno != 17: raise self.filename = os.path.basename(file_dict.get('filename')) if file_dict.get('filename') else None upload_field_storage = file_dict.pop('upload', None) if isinstance(upload_field_storage, cgi.FieldStorage): self._update_filename(upload_field_storage) self.filename = munge.munge_filename(self.filename) file_dict['filename'] = self.filename self.upload_file = upload_field_storage.file
def image_uploader(): '''View function that renders the image uploader form. Passes `'image_url': 'submitted-image-path'` and `'uploads': {'path',}'` to template. ''' try: uploads_with_path = [] try: # Get all the images from the home uploader. path = uploader.get_storage_path() storage_path = os.path.join(path, 'storage', 'uploads', 'home') uploads = sorted(os.listdir(storage_path)) # Build the static content paths for each image. for upload in uploads: image_path = 'uploads/home/' value = h.url_for_static('{0}{1}'.format(image_path, upload)) uploads_with_path.append(value) except OSError: log.error( "The uploads directory does not exist yet. Upload an image.") # This is somewhat odd, toolkit.render calls flask.render_template. # But toolkit.render unpacks the args properly to access them in the # template without needing 'extra_vars["data"]'. Lots of time trouble # shooting to get this combo working. return toolkit.render('admin/ontario_theme_image_uploader.html', extra_vars={ 'data': {}, 'errors': {}, 'image_url': request.args.get("image_url", ''), 'uploads': uploads_with_path }) except Exception as e: log.error(e)
def resource_clean(context, data_dict): """ Remove resource that is not referred by a package. :context: TODO :data_dict: TODO :returns: TODO """ check_access('resource_clean', context, data_dict) model = context['model'] confirmed = data_dict.get('confirmed', False) res_indb = set([ r[0] for r in sqlalchemy.sql.select([model.resource_table.c.id]).execute() ]) storage_path = os.path.join(get_storage_path(), 'resources') dangling = [] for l in check_output(['find', storage_path, '-type', 'f']).split('\n'): if l: rid = ''.join(l.split('/')[-3:]) if rid not in res_indb: dangling.append(l) deleted = [] if confirmed: for f in dangling: os.remove(f) deleted.append(f) return { 'deleted': deleted, 'deleted_count': len(deleted), 'dangling': dangling, 'dangling_count': len(dangling) }
def _merge_ueb_output_pkg_with_input_pkg(self, service_call_results, model_pkg_dataset_id): """ Merges the model run output data with the model input package @param service_call_results: http service response object returned from the app server @param model_pkg_dataset_id: id of the input model package @return: success (True or False) and message ( a string value) representing the status to be set for model package run status attribute """ source = 'uebpackage.tasks._merge_ueb_output_pkg_with_input_pkg():' # save the output model pkg to temp directory ckan_default_dir = uebhelper.StringSettings.ckan_user_session_temp_dir # create a directory for saving the file # this will be a dir in the form of: /tmp/ckan/{random_id} random_id = base.model.types.make_uuid() destination_dir = os.path.join(ckan_default_dir, random_id) os.makedirs(destination_dir) ueb_output_pkg_filename = uebhelper.StringSettings.ueb_output_model_package_default_filename ueb_output_pkg_file = os.path.join(destination_dir, ueb_output_pkg_filename) bytes_to_read = 16 * 1024 success = False message = uebhelper.StringSettings.app_server_job_status_package_retrieval_failed try: with open(ueb_output_pkg_file, 'wb') as file_obj: while True: data = service_call_results.read(bytes_to_read) if not data: break file_obj.write(data) except Exception as e: log.error(source + 'Failed to save ueb model output package zip file to ' 'temporary location for model package dataset ID: %s \n ' 'Exception: %s' % (model_pkg_dataset_id, e)) return success, message log.info(source + 'ueb model output package zip file was saved to temporary location ' 'for model package dataset ID: %s' % model_pkg_dataset_id) # access the input model pkg zip file model_pkg_dataset = uebhelper.get_package(model_pkg_dataset_id) # get the package resource zip file model_pkg_zip_file = model_pkg_dataset['resources'][0] # get the storage path of the pkg zip file ckan_storage_path = os.path.join(uploader.get_storage_path(), 'resources') model_pkg_zip_file_path = os.path.join(ckan_storage_path, model_pkg_zip_file['id'][0:3], model_pkg_zip_file['id'][3:6], model_pkg_zip_file['id'][6:]) ''' open the original input zip file in the append mode and then read the output pkg zip file and append it to the original zip file ''' is_merge_successful = False try: with zip.ZipFile(model_pkg_zip_file_path, 'a') as orig_file_obj: zip_file_to_merge = zip.ZipFile(ueb_output_pkg_file, 'r') for fname in zip_file_to_merge.namelist(): orig_file_obj.writestr(fname, zip_file_to_merge.open(fname).read()) is_merge_successful = True except Exception as e: log.error(source + 'Failed to merge output model pkg zip file with the input model pkg zip file ' 'for model package dataset ID: %s \n ' 'Exception: %s' % (model_pkg_dataset_id, e)) # update the model package dataset package_type to complete if is_merge_successful: data_dict = {'package_run_status': uebhelper.StringSettings.app_server_job_status_package_retrieval_success, 'package_type': u'Complete'} success = True message = uebhelper.StringSettings.app_server_job_status_package_retrieval_success else: data_dict = {'package_run_status': uebhelper.StringSettings.app_server_job_status_package_retrieval_failed} update_msg = 'system auto updated ueb package dataset' background_task = False try: updated_package = uebhelper.update_package(model_pkg_dataset_id, data_dict, update_msg, background_task) log.info(source + 'UEB model package dataset was updated as a result of ' 'receiving model output package for dataset:%s' % updated_package['name']) except Exception as e: log.error(source + 'Failed to update UEB model package dataset after ' 'receiving model input package for dataset ID:%s \n' 'Exception: %s' % (model_pkg_dataset_id, e)) return success, message
def execute(self, pkg_id): """ Executes model input package on app server @param pkg_id: id of the model input package to be executed @return: ajax_response object """ source = 'uebpackage.uebexecute.execute():' # get the model package package = uebhelper.get_package(pkg_id) ajax_response = uebhelper.AJAXResponse() ajax_response.success = False ajax_response.message = "Not a valid UEB model package for execution." # check that it is a valid model package for execution if package['type'] != "model-package": return ajax_response.to_json() if package.get('package_run_status', None) != 'Not yet submitted': ajax_response.message = "This UEB model package has already been executed." return ajax_response.to_json() # get the package resource zip file model_pkg_zip_file = package['resources'][0] # get the storage path ckan_storage_path = os.path.join(uploader.get_storage_path(), 'resources') model_pkg_zip_file_path = os.path.join(ckan_storage_path, model_pkg_zip_file['id'][0:3], model_pkg_zip_file['id'][3:6], model_pkg_zip_file['id'][6:]) # send request to app server # TODO: read the app server address from config (.ini) file service_host_address = uebhelper.StringSettings.app_server_host_address service_request_url = uebhelper.StringSettings.app_server_api_run_ueb_url connection = httplib.HTTPConnection(service_host_address) headers = {'Content-Type': 'application/octet-stream', 'Connection': 'Keep-alive'} # for debugging only connection.set_debuglevel(1) # Let's wait for 0.01 second before calling the webservice # Otherwise sometime we get # 104 error - Connection Reset by Peer # ref: http://stackoverflow.com/questions/383738/104-connection-reset-by-peer-socket-error-or-when-does-closing-a-socket-resu time.sleep(0.01) with open(model_pkg_zip_file_path, 'r') as ueb_model_pkg_file: connection.request('POST', service_request_url, ueb_model_pkg_file, headers) service_call_results = connection.getresponse() if service_call_results.status == httplib.OK: log.info(source + 'Request to execute UEB was successful.') service_response_data = service_call_results.read() connection.close() service_response_dict = json.loads(service_response_data) ueb_run_job_id = service_response_dict.get('RunJobID', None) response_msg = service_response_dict.get('Message', '') if not ueb_run_job_id: log.error(source + 'App server failed to process UEB run request.\n' + response_msg) ajax_response.message = 'App server failed to process UEB run request.' return ajax_response.to_json() else: connection.close() log.error(source + 'App server error: Request to run UEB failed: %s' % service_call_results.reason) ajax_response.message = 'App server failed to process UEB run request.' return ajax_response.to_json() if self._update_ueb_model_pkg_run_job_id(pkg_id, ueb_run_job_id) is None: ajax_response.message = 'Failed to update job run ID for the model package.' return ajax_response.to_json() job_status_processing = uebhelper.StringSettings.app_server_job_status_processing if self._update_ueb_model_pkg_run_status(pkg_id, job_status_processing) is None: ajax_response.message = 'Failed to update run status for the model package.' return ajax_response.to_json() base.session.clear() tk.c.ueb_run_job_id = ueb_run_job_id ajax_response.success = True ajax_response.json_data = job_status_processing ajax_response.message = "Request to execute UEB was successful." return ajax_response.to_json()
def make_app(conf, full_stack=True, static_files=True, **app_conf): """Create a Pylons WSGI application and return it ``conf`` The inherited configuration for this application. Normally from the [DEFAULT] section of the Paste ini file. ``full_stack`` Whether this application provides a full WSGI stack (by default, meaning it handles its own exceptions and errors). Disable full_stack when this application is "managed" by another WSGI middleware. ``static_files`` Whether this application serves its own static files; disable when another web server is responsible for serving them. ``app_conf`` The application's local configuration. Normally specified in the [app:<name>] section of the Paste ini file (where <name> defaults to main). """ # Configure the Pylons environment load_environment(conf, app_conf) # The Pylons WSGI app app = PylonsApp() # set pylons globals app_globals.reset() for plugin in PluginImplementations(IMiddleware): app = plugin.make_middleware(app, config) # Routing/Session/Cache Middleware app = RoutesMiddleware(app, config['routes.map']) # we want to be able to retrieve the routes middleware to be able to update # the mapper. We store it in the pylons config to allow this. config['routes.middleware'] = app app = SessionMiddleware(app, config) app = CacheMiddleware(app, config) # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) #app = QueueLogMiddleware(app) # Fanstatic if asbool(config.get('debug', False)): fanstatic_config = { 'versioning': True, 'recompute_hashes': True, 'minified': False, 'bottom': True, 'bundle': False, } else: fanstatic_config = { 'versioning': True, 'recompute_hashes': False, 'minified': True, 'bottom': True, 'bundle': True, } app = Fanstatic(app, **fanstatic_config) if asbool(full_stack): # Handle Python exceptions app = ErrorHandler(app, conf, **config['pylons.errorware']) # Display error documents for 401, 403, 404 status codes (and # 500 when debug is disabled) if asbool(config['debug']): app = StatusCodeRedirect(app, [400, 404]) else: app = StatusCodeRedirect(app, [400, 404, 500]) # Initialize repoze.who who_parser = WhoConfig(conf['here']) who_parser.parse(open(app_conf['who.config_file'])) if asbool(config.get('openid_enabled', 'true')): from repoze.who.plugins.openid.identification import OpenIdIdentificationPlugin # Monkey patches for repoze.who.openid # Fixes #1659 - enable log-out when CKAN mounted at non-root URL from ckan.lib import repoze_patch OpenIdIdentificationPlugin.identify = repoze_patch.identify OpenIdIdentificationPlugin.redirect_to_logged_in = repoze_patch.redirect_to_logged_in OpenIdIdentificationPlugin._redirect_to_loginform = repoze_patch._redirect_to_loginform OpenIdIdentificationPlugin.challenge = repoze_patch.challenge who_parser.identifiers = [i for i in who_parser.identifiers if \ not isinstance(i, OpenIdIdentificationPlugin)] who_parser.challengers = [i for i in who_parser.challengers if \ not isinstance(i, OpenIdIdentificationPlugin)] app = PluggableAuthenticationMiddleware(app, who_parser.identifiers, who_parser.authenticators, who_parser.challengers, who_parser.mdproviders, who_parser.request_classifier, who_parser.challenge_decider, logging.getLogger('repoze.who'), logging.WARN, # ignored who_parser.remote_user_key, ) # Establish the Registry for this application app = RegistryManager(app) app = I18nMiddleware(app, config) if asbool(static_files): # Serve static files static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ else int(config.get('ckan.static_max_age', 3600)) static_app = StaticURLParser(config['pylons.paths']['static_files'], cache_max_age=static_max_age) static_parsers = [static_app, app] storage_directory = uploader.get_storage_path() if storage_directory: path = os.path.join(storage_directory, 'storage') try: os.makedirs(path) except OSError, e: ## errno 17 is file already exists if e.errno != 17: raise storage_app = StaticURLParser(path, cache_max_age=static_max_age) static_parsers.insert(0, storage_app) # Configurable extra static file paths extra_static_parsers = [] for public_path in config.get('extra_public_paths', '').split(','): if public_path.strip(): extra_static_parsers.append( StaticURLParser(public_path.strip(), cache_max_age=static_max_age) ) app = Cascade(extra_static_parsers + static_parsers)
def configure(self, config): self.gapi_key = config.get('ckanext.geoview.gapi_key', None) self.storage_path = get_storage_path() self.geoserver_url = config.get('dadosabertos.geoserver.url', None) self.geoserver_user = config.get('dadosabertos.geoserver.user', None) self.geoserver_password = config.get('dadosabertos.geoserver.password', None)
def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): """Create a Pylons WSGI application and return it ``conf`` The inherited configuration for this application. Normally from the [DEFAULT] section of the Paste ini file. ``full_stack`` Whether this application provides a full WSGI stack (by default, meaning it handles its own exceptions and errors). Disable full_stack when this application is "managed" by another WSGI middleware. ``static_files`` Whether this application serves its own static files; disable when another web server is responsible for serving them. ``app_conf`` The application's local configuration. Normally specified in the [app:<name>] section of the Paste ini file (where <name> defaults to main). """ # The Pylons WSGI app app = pylons_app = CKANPylonsApp() for plugin in PluginImplementations(IMiddleware): app = plugin.make_middleware(app, config) # Routing/Session/Cache Middleware app = RoutesMiddleware(app, config['routes.map']) # we want to be able to retrieve the routes middleware to be able to update # the mapper. We store it in the pylons config to allow this. config['routes.middleware'] = app app = SessionMiddleware(app, config) app = CacheMiddleware(app, config) # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) # app = QueueLogMiddleware(app) if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)): app = execute_on_completion(app, config, cleanup_pylons_response_string) # Fanstatic if asbool(config.get('debug', False)): fanstatic_config = { 'versioning': True, 'recompute_hashes': True, 'minified': False, 'bottom': True, 'bundle': False, } else: fanstatic_config = { 'versioning': True, 'recompute_hashes': False, 'minified': True, 'bottom': True, 'bundle': True, } app = Fanstatic(app, **fanstatic_config) for plugin in PluginImplementations(IMiddleware): try: app = plugin.make_error_log_middleware(app, config) except AttributeError: log.critical('Middleware class {0} is missing the method' 'make_error_log_middleware.' .format(plugin.__class__.__name__)) if asbool(full_stack): # Handle Python exceptions app = ErrorHandler(app, conf, **config['pylons.errorware']) # Display error documents for 400, 403, 404 status codes (and # 500 when debug is disabled) if asbool(config['debug']): app = StatusCodeRedirect(app, [400, 403, 404]) else: app = StatusCodeRedirect(app, [400, 403, 404, 500]) # Initialize repoze.who who_parser = WhoConfig(conf['here']) who_parser.parse(open(app_conf['who.config_file'])) app = PluggableAuthenticationMiddleware( app, who_parser.identifiers, who_parser.authenticators, who_parser.challengers, who_parser.mdproviders, who_parser.request_classifier, who_parser.challenge_decider, logging.getLogger('repoze.who'), logging.WARN, # ignored who_parser.remote_user_key ) # Establish the Registry for this application app = RegistryManager(app) app = common_middleware.I18nMiddleware(app, config) if asbool(static_files): # Serve static files static_max_age = None if not asbool( config.get('ckan.cache_enabled')) \ else int(config.get('ckan.static_max_age', 3600)) static_app = StaticURLParser( config['pylons.paths']['static_files'], cache_max_age=static_max_age) static_parsers = [static_app, app] storage_directory = uploader.get_storage_path() if storage_directory: path = os.path.join(storage_directory, 'storage') try: os.makedirs(path) except OSError, e: # errno 17 is file already exists if e.errno != 17: raise storage_app = StaticURLParser(path, cache_max_age=static_max_age) static_parsers.insert(0, storage_app) # Configurable extra static file paths extra_static_parsers = [] for public_path in config.get( 'extra_public_paths', '').split(','): if public_path.strip(): extra_static_parsers.append( StaticURLParser(public_path.strip(), cache_max_age=static_max_age) ) app = Cascade(extra_static_parsers + static_parsers)
def __init__(self, resource): path = get_storage_path() config_mimetype_guess = config.get('ckan.mimetype_guess', 'file_ext') if not path: self.storage_path = None return self.storage_path = os.path.join(path, 'resources') try: os.makedirs(self.storage_path) except OSError as e: # errno 17 is file already exists if e.errno != 17: raise self.filename = None self.mimetype = None url = resource.get('url') upload_field_storage = resource.pop('upload', None) self.clear = resource.pop('clear_upload', None) if url and config_mimetype_guess == 'file_ext': self.mimetype = mimetypes.guess_type(url)[0] if isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES): self.filesize = 0 # bytes self.filename = upload_field_storage.filename # MODIFICATION START self.filename = secure_filename(self.filename) # Overkill but I # trust werkzueg over ckan. # MODIFICATION END self.filename = munge.munge_filename(self.filename) resource['url'] = self.filename resource['url_type'] = 'upload' resource['last_modified'] = datetime.datetime.utcnow() self.upload_file = _get_underlying_file(upload_field_storage) self.upload_file.seek(0, os.SEEK_END) self.filesize = self.upload_file.tell() # go back to the beginning of the file buffer self.upload_file.seek(0, os.SEEK_SET) # MODIFICATION START # Note: If resubmitting a failed form without clearing the file # the ResourceUpload.upload function would be called skipping the # init call. if not allowed_file(self.filename): log.error('Upload: Invalid upload file format.{}'.format( self.filename)) # remove file - by default a resource can be added without any # values resource['url'] = None resource['url_type'] = '' raise logic.ValidationError({ 'upload': ['Invalid upload file format, file has been removed.'] }) # MODIFICATION END # check if the mimetype failed from guessing with the url if not self.mimetype and config_mimetype_guess == 'file_ext': self.mimetype = mimetypes.guess_type(self.filename)[0] if not self.mimetype and config_mimetype_guess == 'file_contents': try: self.mimetype = magic.from_buffer(self.upload_file.read(), mime=True) self.upload_file.seek(0, os.SEEK_SET) except IOError as e: # Not that important if call above fails self.mimetype = None elif self.clear: resource['url_type'] = ''
def make_flask_stack(conf: Union[Config, CKANConfig]) -> CKANApp: """ This has to pass the flask app through all the same middleware that Pylons used """ root = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) debug = asbool(conf.get('debug', conf.get('DEBUG', False))) testing = asbool(conf.get('testing', conf.get('TESTING', False))) app = flask_app = CKANFlask(__name__, static_url_path='') # Register storage for accessing group images, site logo, etc. storage_folder = [] storage = uploader.get_storage_path() if storage: storage_folder = [os.path.join(storage, 'storage')] # Static files folders (core and extensions) public_folder = config.get_value(u'ckan.base_public_folder') app.static_folder = config.get_value('extra_public_paths').split(',') + [ os.path.join(root, public_folder) ] + storage_folder app.jinja_options = jinja_extensions.get_jinja_env_options() app.jinja_env.policies['ext.i18n.trimmed'] = True app.debug = debug app.testing = testing app.template_folder = os.path.join(root, 'templates') app.app_ctx_globals_class = CKAN_AppCtxGlobals app.url_rule_class = CKAN_Rule # Update Flask config with the CKAN values. We use the common config # object as values might have been modified on `load_environment` if config: app.config.update(config) else: app.config.update(conf) # Do all the Flask-specific stuff before adding other middlewares # Secret key needed for flask-debug-toolbar and sessions if not app.config.get('SECRET_KEY'): app.config['SECRET_KEY'] = config.get_value('beaker.session.secret') if not app.config.get('SECRET_KEY'): raise RuntimeError(u'You must provide a value for the secret key' ' with the SECRET_KEY config option') root_path = config.get_value('ckan.root_path') if debug: from flask_debugtoolbar import DebugToolbarExtension app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False debug_ext = DebugToolbarExtension() # register path that includes `ckan.site_root` before # initializing debug app. In such a way, our route receives # higher precedence. # TODO: After removal of Pylons code, switch to # `APPLICATION_ROOT` config value for flask application. Right # now it's a bad option because we are handling both pylons # and flask urls inside helpers and splitting this logic will # bring us tons of headache. if root_path: app.add_url_rule( root_path.replace('{{LANG}}', '').rstrip('/') + '/_debug_toolbar/static/<path:filename>', '_debug_toolbar.static', debug_ext.send_static_file) debug_ext.init_app(app) from werkzeug.debug import DebuggedApplication app.wsgi_app = DebuggedApplication(app.wsgi_app, True) # Use Beaker as the Flask session interface class BeakerSessionInterface(SessionInterface): def open_session(self, app: Any, request: Any): if 'beaker.session' in request.environ: return request.environ['beaker.session'] def save_session(self, app: Any, session: Any, response: Any): session.save() namespace = 'beaker.session.' session_opts = { k.replace('beaker.', ''): v for k, v in config.items() if k.startswith(namespace) } if (not session_opts.get('session.data_dir') and session_opts.get('session.type', 'file') == 'file'): cache_dir = conf.get('cache_dir') or conf.get('cache.dir') session_opts['session.data_dir'] = '{data_dir}/sessions'.format( data_dir=cache_dir) app.wsgi_app = RootPathMiddleware(app.wsgi_app) app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts) app.session_interface = BeakerSessionInterface() # Add Jinja2 extensions and filters app.jinja_env.filters['empty_and_escape'] = \ jinja_extensions.empty_and_escape # Common handlers for all requests # # flask types do not mention that it's possible to return a response from # the `before_request` callback app.before_request(ckan_before_request) app.after_request(ckan_after_request) # Template context processors app.context_processor(helper_functions) app.context_processor(c_object) app.context_processor(_ungettext_alias) # Babel _ckan_i18n_dir = i18n.get_ckan_i18n_dir() pairs = [cast("tuple[str, str]", (_ckan_i18n_dir, u'ckan')) ] + [(p.i18n_directory(), p.i18n_domain()) for p in reversed(list(PluginImplementations(ITranslation)))] i18n_dirs, i18n_domains = zip(*pairs) app.config[u'BABEL_TRANSLATION_DIRECTORIES'] = ';'.join(i18n_dirs) app.config[u'BABEL_DOMAIN'] = 'ckan' app.config[u'BABEL_MULTIPLE_DOMAINS'] = ';'.join(i18n_domains) app.config[u'BABEL_DEFAULT_TIMEZONE'] = str(h.get_display_timezone()) babel = CKANBabel(app) babel.localeselector(get_locale) # WebAssets _setup_webassets(app) # Auto-register all blueprints defined in the `views` folder _register_core_blueprints(app) _register_error_handler(app) # Set up each IBlueprint extension as a Flask Blueprint for plugin in PluginImplementations(IBlueprint): plugin_blueprints = plugin.get_blueprint() if not isinstance(plugin_blueprints, list): plugin_blueprints = [plugin_blueprints] for blueprint in plugin_blueprints: app.register_extension_blueprint(blueprint) lib_plugins.register_package_blueprints(app) lib_plugins.register_group_blueprints(app) # Start other middleware for plugin in PluginImplementations(IMiddleware): app = plugin.make_middleware(app, config) for plugin in PluginImplementations(IMiddleware): try: app = plugin.make_error_log_middleware(app, config) except AttributeError: log.critical('Middleware class {0} is missing the method' 'make_error_log_middleware.'.format( plugin.__class__.__name__)) # Initialize flask-login login_manager = LoginManager() login_manager.init_app(app) # make anonymous_user an instance of CKAN custom class login_manager.anonymous_user = model.AnonymousUser # The name of the view to redirect to when the user needs to log in. login_manager.login_view = config.get_value("ckan.auth.login_view") @login_manager.user_loader def load_user(user_id: str) -> Optional["model.User"]: # type: ignore return model.User.get(user_id) @login_manager.request_loader def load_user_from_request(request): # type: ignore user = _get_user_for_apitoken() return user # Update the main CKAN config object with the Flask specific keys # that were set here or autogenerated flask_config_keys = set(flask_app.config.keys()) - set(config.keys()) for key in flask_config_keys: config[key] = flask_app.config[key] # Prevent the host from request to be added to the new header location. app = HostHeaderMiddleware(app) app = I18nMiddleware(app) if config.get_value('ckan.tracking_enabled'): app = TrackingMiddleware(app, config) # Add a reference to the actual Flask app so it's easier to access # type_ignore_reason: custom attribute app._wsgi_app = flask_app # type: ignore return app