def _read_config(cls):
try:
andino_config = cls._redis_cli().get('andino-config')
gobar_config = json.loads(andino_config)
except Exception:
try:
with open(GobArConfigController.CONFIG_PATH) as json_data:
gobar_config = json.load(json_data)
except Exception:
gobar_config = {}
try:
is_redis_available()
cls._redis_cli().set('andino-config', json.dumps(gobar_config))
except Exception:
logger.error("Redis no se encuentra disponible!")
return gobar_config
def load_environment(conf):
"""
Configure the Pylons environment via the ``pylons.config`` object. This
code should only need to be run once.
"""
os.environ['CKAN_CONFIG'] = conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
valid_base_public_folder_names = ['public']
static_files = conf.get('ckan.base_public_folder', 'public')
conf['ckan.base_public_folder'] = static_files
if static_files not in valid_base_public_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_public_folder. '
'Possible values are: "public".'
)
log.info('Loading static files from %s' % static_files)
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, static_files),
templates=[])
# Initialize main CKAN config object
config.update(conf)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = ['^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
# load all CKAN plugins
p.load_all()
# Check Redis availability
if not is_redis_available():
log.critical('Could not connect to Redis.')
app_globals.reset()
# issue #3260: remove idle transaction
# Session that was used for getting all config params nor committed,
# neither removed and we have idle connection as result
model.Session.commit()
# Build JavaScript translations. Must be done after plugins have
# been loaded.
build_js_translations()
def load_environment(conf: Union[Config, CKANConfig]):
"""
Configure the Pylons environment via the ``pylons.config`` object. This
code should only need to be run once.
"""
os.environ['CKAN_CONFIG'] = cast(str, conf['__file__'])
valid_base_public_folder_names = ['public', 'public-bs3']
static_files = conf.get('ckan.base_public_folder', 'public')
conf['ckan.base_public_folder'] = static_files
if static_files not in valid_base_public_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_public_folder. '
'Possible values are: "public" and "public-bs3".')
log.info('Loading static files from %s' % static_files)
# Initialize main CKAN config object
config.update(conf)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = [
'^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
# load all CKAN plugins
p.load_all()
# Check Redis availability
if not is_redis_available():
log.critical('Could not connect to Redis.')
app_globals.reset()
# Build JavaScript translations. Must be done after plugins have
# been loaded.
build_js_translations()
def load_environment(conf):
"""
Configure the Pylons environment via the ``pylons.config`` object. This
code should only need to be run once.
"""
if six.PY2:
# this must be run at a time when the env is semi-setup, thus inlined
# here. Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = getattr(PylonsApp.find_controller,
'_old_find_controller',
PylonsApp.find_controller)
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
if hasattr(ep, 'resolve'):
# setuptools >= 10.2
mycontroller = ep.resolve()
else:
# setuptools >= 11.3
mycontroller = ep.load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
find_controller._old_find_controller = find_controller_generic
PylonsApp.find_controller = find_controller
os.environ['CKAN_CONFIG'] = conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
valid_base_public_folder_names = ['public']
static_files = conf.get('ckan.base_public_folder', 'public')
conf['ckan.base_public_folder'] = static_files
if static_files not in valid_base_public_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_public_folder. '
'Possible values are: "public".')
log.info('Loading static files from %s' % static_files)
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, static_files),
templates=[])
# Initialize main CKAN config object
config.update(conf)
if six.PY2:
# Initialize Pylons own config object
pylons_config.init_app(conf['global_conf'],
conf,
package='ckan',
paths=paths)
# Update the main CKAN config object with the Pylons specific stuff,
# as it is quite hard to keep them separated. This should be removed
# once Pylons support is dropped
config.update(pylons_config)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = [
'^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
# load all CKAN plugins
p.load_all()
if not p.plugin_loaded('managed_search_schema'):
search.check_solr_schema_version()
# Check Redis availability
if not is_redis_available():
log.critical('Could not connect to Redis.')
app_globals.reset()
# issue #3260: remove idle transaction
# Session that was used for getting all config params nor committed,
# neither removed and we have idle connection as result
model.Session.commit()
# Build JavaScript translations. Must be done after plugins have
# been loaded.
build_js_translations()
def load_environment(global_conf, app_conf):
"""
Configure the Pylons environment via the ``pylons.config`` object. This
code should only need to be run once.
"""
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
if hasattr(ep, 'resolve'):
# setuptools >= 10.2
mycontroller = ep.resolve()
else:
# setuptools >= 11.3
mycontroller = ep.load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
valid_base_public_folder_names = ['public', 'public-bs2']
static_files = app_conf.get('ckan.base_public_folder', 'public')
app_conf['ckan.base_public_folder'] = static_files
if static_files not in valid_base_public_folder_names:
raise CkanConfigurationException(
'You provided an invalid value for ckan.base_public_folder. '
'Possible values are: "public" and "public-bs2".'
)
log.info('Loading static files from %s' % static_files)
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, static_files),
templates=[])
# Initialize main CKAN config object
config.update(global_conf)
config.update(app_conf)
# Initialize Pylons own config object
pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# Update the main CKAN config object with the Pylons specific stuff, as it
# quite hard to keep them separated. This should be removed once Pylons
# support is dropped
config.update(pylons_config)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = ['^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
# load all CKAN plugins
p.load_all()
# Check Redis availability
if not is_redis_available():
log.critical('Could not connect to Redis.')
app_globals.reset()
# issue #3260: remove idle transaction
# Session that was used for getting all config params nor committed,
# neither removed and we have idle connection as result
model.Session.commit()
# Build JavaScript translations. Must be done after plugins have
# been loaded.
build_js_translations()
def load_environment(global_conf, app_conf):
"""
Configure the Pylons environment via the ``pylons.config`` object. This
code should only need to be run once.
"""
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
if hasattr(ep, 'resolve'):
# setuptools >= 10.2
mycontroller = ep.resolve()
else:
# setuptools >= 11.3
mycontroller = ep.load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[])
# Initialize main CKAN config object
config.update(global_conf)
config.update(app_conf)
# Initialize Pylons own config object
pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# Update the main CKAN config object with the Pylons specific stuff, as it
# quite hard to keep them separated. This should be removed once Pylons
# support is dropped
config.update(pylons_config)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = [
'^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
# Check Redis availability
if not is_redis_available():
log.critical('Could not connect to Redis.')
# load all CKAN plugins
p.load_all()
app_globals.reset()
def validate_resource(self, id):
if toolkit.request.method == 'POST':
data = dict_fns.unflatten(
tuplize_dict(parse_params(toolkit.request.POST)))
check_schema = toolkit.request.params.get('check_schema')
upload_data = toolkit.request.params.get('upload_data')
file_path = data.get('file_path')
# Logic for validating a resource against a specified schema
if check_schema:
schema = {'fields': []}
fields = data.get('field_name')
field_type = data.get('field_type')
# Schema is populated from data entered by the user
for i, field in enumerate(fields):
schema['fields'].append({
'name': field,
'type': field_type[i]
})
# File is validated with Goodtables
report = validate(file_path, schema=schema)
log = logging.getLogger('ckanext.tayside')
# Score is calculated based on Sir Tim Berners-Lee's five star
# of openness
sniffed_format = sniff_file_format(file_path, log)
score = resource_format_scores().get(sniffed_format['format'])
vars = {
'report': report,
'pkg_name': id,
'stars': score,
'file_path': file_path
}
return toolkit.render('tayside/package/validate_resource.html',
extra_vars=vars)
elif upload_data:
# Handles creating a resource in CKAN
# Metadata for the resource is stored in Redis
r = redis.StrictRedis()
data = json.loads(r.get(file_path))
data['package_id'] = id
# Dataset's state is changed from 'draft' to 'active'
toolkit.get_action('package_patch')({}, {
'id': id,
'state': 'active'
})
# FieldStorage instance is created which is needed to upload
# the file to Filestore and Datastore
fs = cgi.FieldStorage()
fs.file = fs.make_file()
fs.filename = data.get('url')
f = open(file_path, 'r')
fs.file.write(f.read())
fs.file.seek(0)
f.close()
data['upload'] = fs
try:
toolkit.get_action('resource_create')({}, data)
except Exception as e:
vars = {
'upload_error': 'An error occured while creating the '
'resource.',
'pkg_name': id
}
return toolkit.render(
'tayside/package/validate_resource.html',
extra_vars=vars)
# File is uploaded on Filestore, and now it is safe to be
# removed from the temporary location
os.remove(file_path)
toolkit.redirect_to(controller='package', action='read', id=id)
else:
is_upload = isinstance(data.get('upload'), cgi.FieldStorage)
supported_formats = ['csv', 'tsv', 'xls', 'xlsx', 'ods']
current_format = data.get('url').split('.')[-1]
if is_upload:
if current_format in supported_formats:
# Logic for storing the file locally and extracting
# it's headers (fields)
storage_path = config.get('ckan.storage_path')
file_path = storage_path + '/' + data.get('url')
# Read the file
buffer = data.get('upload').file
buffer.seek(0)
# Write the file locally
f = open(file_path, 'w')
f.write(buffer.read())
f.close()
# Inspect the headers (fields) of the file
with Stream(file_path, headers=1) as stream:
fields = stream.headers
vars = {
'fields': fields,
'pkg_name': id,
'file_path': file_path
}
if is_redis_available():
# Store the metadata of the resource in Redis for
# later usage
r = redis.StrictRedis()
resource_data = {
'name': data.get('name'),
'description': data.get('description'),
'format': data.get('format'),
'url': data.get('url'),
}
r.set(file_path, json.dumps(resource_data))
# Store it for 1 day
r.expire(file_path, 86400)
else:
return toolkit.render(
'tayside/package/validate_resource.html',
{'redis_error': 'Redis not available'})
return toolkit.render(
'tayside/package/validate_resource.html',
extra_vars=vars)
else:
vars = {
'format_error': 'Format not supported.',
'pkg_name': id
}
return toolkit.render(
'tayside/package/validate_resource.html',
extra_vars=vars)
vars = {
'format_error': 'No file provided for validation.',
'pkg_name': id
}
return toolkit.render('tayside/package/validate_resource.html',
extra_vars=vars)
else:
return toolkit.render('tayside/package/validate_resource.html',
{'pkg_name': id})
def load_environment(global_conf, app_conf):
"""
Configure the Pylons environment via the ``pylons.config`` object. This
code should only need to be run once.
"""
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
if hasattr(ep, 'resolve'):
# setuptools >= 10.2
mycontroller = ep.resolve()
else:
# setuptools >= 11.3
mycontroller = ep.load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[])
# Initialize main CKAN config object
config.update(global_conf)
config.update(app_conf)
# Initialize Pylons own config object
pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# Update the main CKAN config object with the Pylons specific stuff, as it
# quite hard to keep them separated. This should be removed once Pylons
# support is dropped
config.update(pylons_config)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
msgs = ['^Unicode type received non-unicode bind param value',
"^Did not recognize type 'BIGINT' of column 'size'",
"^Did not recognize type 'tsvector' of column 'search_vector'"
]
for msg in msgs:
warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
# Check Redis availability
if not is_redis_available():
log.critical('Could not connect to Redis.')
# load all CKAN plugins
p.load_all()
app_globals.reset()
