def _read_config(cls): try: andino_config = cls._redis_cli().get('andino-config') gobar_config = json.loads(andino_config) except Exception: try: with open(GobArConfigController.CONFIG_PATH) as json_data: gobar_config = json.load(json_data) except Exception: gobar_config = {} try: is_redis_available() cls._redis_cli().set('andino-config', json.dumps(gobar_config)) except Exception: logger.error("Redis no se encuentra disponible!") return gobar_config
def load_environment(conf): """ Configure the Pylons environment via the ``pylons.config`` object. This code should only need to be run once. """ os.environ['CKAN_CONFIG'] = conf['__file__'] # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) valid_base_public_folder_names = ['public'] static_files = conf.get('ckan.base_public_folder', 'public') conf['ckan.base_public_folder'] = static_files if static_files not in valid_base_public_folder_names: raise CkanConfigurationException( 'You provided an invalid value for ckan.base_public_folder. ' 'Possible values are: "public".' ) log.info('Loading static files from %s' % static_files) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, static_files), templates=[]) # Initialize main CKAN config object config.update(conf) # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings msgs = ['^Unicode type received non-unicode bind param value', "^Did not recognize type 'BIGINT' of column 'size'", "^Did not recognize type 'tsvector' of column 'search_vector'" ] for msg in msgs: warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) # load all CKAN plugins p.load_all() # Check Redis availability if not is_redis_available(): log.critical('Could not connect to Redis.') app_globals.reset() # issue #3260: remove idle transaction # Session that was used for getting all config params nor committed, # neither removed and we have idle connection as result model.Session.commit() # Build JavaScript translations. Must be done after plugins have # been loaded. build_js_translations()
def load_environment(conf: Union[Config, CKANConfig]): """ Configure the Pylons environment via the ``pylons.config`` object. This code should only need to be run once. """ os.environ['CKAN_CONFIG'] = cast(str, conf['__file__']) valid_base_public_folder_names = ['public', 'public-bs3'] static_files = conf.get('ckan.base_public_folder', 'public') conf['ckan.base_public_folder'] = static_files if static_files not in valid_base_public_folder_names: raise CkanConfigurationException( 'You provided an invalid value for ckan.base_public_folder. ' 'Possible values are: "public" and "public-bs3".') log.info('Loading static files from %s' % static_files) # Initialize main CKAN config object config.update(conf) # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings msgs = [ '^Unicode type received non-unicode bind param value', "^Did not recognize type 'BIGINT' of column 'size'", "^Did not recognize type 'tsvector' of column 'search_vector'" ] for msg in msgs: warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) # load all CKAN plugins p.load_all() # Check Redis availability if not is_redis_available(): log.critical('Could not connect to Redis.') app_globals.reset() # Build JavaScript translations. Must be done after plugins have # been loaded. build_js_translations()
def load_environment(conf): """ Configure the Pylons environment via the ``pylons.config`` object. This code should only need to be run once. """ if six.PY2: # this must be run at a time when the env is semi-setup, thus inlined # here. Required by the deliverance plugin and iATI from pylons.wsgiapp import PylonsApp import pkg_resources find_controller_generic = getattr(PylonsApp.find_controller, '_old_find_controller', PylonsApp.find_controller) # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): if controller in self.controller_classes: return self.controller_classes[controller] # Check to see if its a dotted name if '.' in controller or ':' in controller: ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller)) if hasattr(ep, 'resolve'): # setuptools >= 10.2 mycontroller = ep.resolve() else: # setuptools >= 11.3 mycontroller = ep.load(False) self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) find_controller._old_find_controller = find_controller_generic PylonsApp.find_controller = find_controller os.environ['CKAN_CONFIG'] = conf['__file__'] # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) valid_base_public_folder_names = ['public'] static_files = conf.get('ckan.base_public_folder', 'public') conf['ckan.base_public_folder'] = static_files if static_files not in valid_base_public_folder_names: raise CkanConfigurationException( 'You provided an invalid value for ckan.base_public_folder. ' 'Possible values are: "public".') log.info('Loading static files from %s' % static_files) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, static_files), templates=[]) # Initialize main CKAN config object config.update(conf) if six.PY2: # Initialize Pylons own config object pylons_config.init_app(conf['global_conf'], conf, package='ckan', paths=paths) # Update the main CKAN config object with the Pylons specific stuff, # as it is quite hard to keep them separated. This should be removed # once Pylons support is dropped config.update(pylons_config) # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings msgs = [ '^Unicode type received non-unicode bind param value', "^Did not recognize type 'BIGINT' of column 'size'", "^Did not recognize type 'tsvector' of column 'search_vector'" ] for msg in msgs: warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) # load all CKAN plugins p.load_all() if not p.plugin_loaded('managed_search_schema'): search.check_solr_schema_version() # Check Redis availability if not is_redis_available(): log.critical('Could not connect to Redis.') app_globals.reset() # issue #3260: remove idle transaction # Session that was used for getting all config params nor committed, # neither removed and we have idle connection as result model.Session.commit() # Build JavaScript translations. Must be done after plugins have # been loaded. build_js_translations()
def load_environment(global_conf, app_conf): """ Configure the Pylons environment via the ``pylons.config`` object. This code should only need to be run once. """ # this must be run at a time when the env is semi-setup, thus inlined here. # Required by the deliverance plugin and iATI from pylons.wsgiapp import PylonsApp import pkg_resources find_controller_generic = PylonsApp.find_controller # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): if controller in self.controller_classes: return self.controller_classes[controller] # Check to see if its a dotted name if '.' in controller or ':' in controller: ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller)) if hasattr(ep, 'resolve'): # setuptools >= 10.2 mycontroller = ep.resolve() else: # setuptools >= 11.3 mycontroller = ep.load(False) self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) PylonsApp.find_controller = find_controller os.environ['CKAN_CONFIG'] = global_conf['__file__'] # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) valid_base_public_folder_names = ['public', 'public-bs2'] static_files = app_conf.get('ckan.base_public_folder', 'public') app_conf['ckan.base_public_folder'] = static_files if static_files not in valid_base_public_folder_names: raise CkanConfigurationException( 'You provided an invalid value for ckan.base_public_folder. ' 'Possible values are: "public" and "public-bs2".' ) log.info('Loading static files from %s' % static_files) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, static_files), templates=[]) # Initialize main CKAN config object config.update(global_conf) config.update(app_conf) # Initialize Pylons own config object pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths) # Update the main CKAN config object with the Pylons specific stuff, as it # quite hard to keep them separated. This should be removed once Pylons # support is dropped config.update(pylons_config) # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings msgs = ['^Unicode type received non-unicode bind param value', "^Did not recognize type 'BIGINT' of column 'size'", "^Did not recognize type 'tsvector' of column 'search_vector'" ] for msg in msgs: warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) # load all CKAN plugins p.load_all() # Check Redis availability if not is_redis_available(): log.critical('Could not connect to Redis.') app_globals.reset() # issue #3260: remove idle transaction # Session that was used for getting all config params nor committed, # neither removed and we have idle connection as result model.Session.commit() # Build JavaScript translations. Must be done after plugins have # been loaded. build_js_translations()
def load_environment(global_conf, app_conf): """ Configure the Pylons environment via the ``pylons.config`` object. This code should only need to be run once. """ # this must be run at a time when the env is semi-setup, thus inlined here. # Required by the deliverance plugin and iATI from pylons.wsgiapp import PylonsApp import pkg_resources find_controller_generic = PylonsApp.find_controller # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): if controller in self.controller_classes: return self.controller_classes[controller] # Check to see if its a dotted name if '.' in controller or ':' in controller: ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller)) if hasattr(ep, 'resolve'): # setuptools >= 10.2 mycontroller = ep.resolve() else: # setuptools >= 11.3 mycontroller = ep.load(False) self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) PylonsApp.find_controller = find_controller os.environ['CKAN_CONFIG'] = global_conf['__file__'] # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, 'public'), templates=[]) # Initialize main CKAN config object config.update(global_conf) config.update(app_conf) # Initialize Pylons own config object pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths) # Update the main CKAN config object with the Pylons specific stuff, as it # quite hard to keep them separated. This should be removed once Pylons # support is dropped config.update(pylons_config) # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings msgs = [ '^Unicode type received non-unicode bind param value', "^Did not recognize type 'BIGINT' of column 'size'", "^Did not recognize type 'tsvector' of column 'search_vector'" ] for msg in msgs: warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) # Check Redis availability if not is_redis_available(): log.critical('Could not connect to Redis.') # load all CKAN plugins p.load_all() app_globals.reset()
def validate_resource(self, id): if toolkit.request.method == 'POST': data = dict_fns.unflatten( tuplize_dict(parse_params(toolkit.request.POST))) check_schema = toolkit.request.params.get('check_schema') upload_data = toolkit.request.params.get('upload_data') file_path = data.get('file_path') # Logic for validating a resource against a specified schema if check_schema: schema = {'fields': []} fields = data.get('field_name') field_type = data.get('field_type') # Schema is populated from data entered by the user for i, field in enumerate(fields): schema['fields'].append({ 'name': field, 'type': field_type[i] }) # File is validated with Goodtables report = validate(file_path, schema=schema) log = logging.getLogger('ckanext.tayside') # Score is calculated based on Sir Tim Berners-Lee's five star # of openness sniffed_format = sniff_file_format(file_path, log) score = resource_format_scores().get(sniffed_format['format']) vars = { 'report': report, 'pkg_name': id, 'stars': score, 'file_path': file_path } return toolkit.render('tayside/package/validate_resource.html', extra_vars=vars) elif upload_data: # Handles creating a resource in CKAN # Metadata for the resource is stored in Redis r = redis.StrictRedis() data = json.loads(r.get(file_path)) data['package_id'] = id # Dataset's state is changed from 'draft' to 'active' toolkit.get_action('package_patch')({}, { 'id': id, 'state': 'active' }) # FieldStorage instance is created which is needed to upload # the file to Filestore and Datastore fs = cgi.FieldStorage() fs.file = fs.make_file() fs.filename = data.get('url') f = open(file_path, 'r') fs.file.write(f.read()) fs.file.seek(0) f.close() data['upload'] = fs try: toolkit.get_action('resource_create')({}, data) except Exception as e: vars = { 'upload_error': 'An error occured while creating the ' 'resource.', 'pkg_name': id } return toolkit.render( 'tayside/package/validate_resource.html', extra_vars=vars) # File is uploaded on Filestore, and now it is safe to be # removed from the temporary location os.remove(file_path) toolkit.redirect_to(controller='package', action='read', id=id) else: is_upload = isinstance(data.get('upload'), cgi.FieldStorage) supported_formats = ['csv', 'tsv', 'xls', 'xlsx', 'ods'] current_format = data.get('url').split('.')[-1] if is_upload: if current_format in supported_formats: # Logic for storing the file locally and extracting # it's headers (fields) storage_path = config.get('ckan.storage_path') file_path = storage_path + '/' + data.get('url') # Read the file buffer = data.get('upload').file buffer.seek(0) # Write the file locally f = open(file_path, 'w') f.write(buffer.read()) f.close() # Inspect the headers (fields) of the file with Stream(file_path, headers=1) as stream: fields = stream.headers vars = { 'fields': fields, 'pkg_name': id, 'file_path': file_path } if is_redis_available(): # Store the metadata of the resource in Redis for # later usage r = redis.StrictRedis() resource_data = { 'name': data.get('name'), 'description': data.get('description'), 'format': data.get('format'), 'url': data.get('url'), } r.set(file_path, json.dumps(resource_data)) # Store it for 1 day r.expire(file_path, 86400) else: return toolkit.render( 'tayside/package/validate_resource.html', {'redis_error': 'Redis not available'}) return toolkit.render( 'tayside/package/validate_resource.html', extra_vars=vars) else: vars = { 'format_error': 'Format not supported.', 'pkg_name': id } return toolkit.render( 'tayside/package/validate_resource.html', extra_vars=vars) vars = { 'format_error': 'No file provided for validation.', 'pkg_name': id } return toolkit.render('tayside/package/validate_resource.html', extra_vars=vars) else: return toolkit.render('tayside/package/validate_resource.html', {'pkg_name': id})
def load_environment(global_conf, app_conf): """ Configure the Pylons environment via the ``pylons.config`` object. This code should only need to be run once. """ # this must be run at a time when the env is semi-setup, thus inlined here. # Required by the deliverance plugin and iATI from pylons.wsgiapp import PylonsApp import pkg_resources find_controller_generic = PylonsApp.find_controller # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): if controller in self.controller_classes: return self.controller_classes[controller] # Check to see if its a dotted name if '.' in controller or ':' in controller: ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller)) if hasattr(ep, 'resolve'): # setuptools >= 10.2 mycontroller = ep.resolve() else: # setuptools >= 11.3 mycontroller = ep.load(False) self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) PylonsApp.find_controller = find_controller os.environ['CKAN_CONFIG'] = global_conf['__file__'] # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, 'public'), templates=[]) # Initialize main CKAN config object config.update(global_conf) config.update(app_conf) # Initialize Pylons own config object pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths) # Update the main CKAN config object with the Pylons specific stuff, as it # quite hard to keep them separated. This should be removed once Pylons # support is dropped config.update(pylons_config) # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings msgs = ['^Unicode type received non-unicode bind param value', "^Did not recognize type 'BIGINT' of column 'size'", "^Did not recognize type 'tsvector' of column 'search_vector'" ] for msg in msgs: warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) # Check Redis availability if not is_redis_available(): log.critical('Could not connect to Redis.') # load all CKAN plugins p.load_all() app_globals.reset()