def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%etalab2.fr%'),
# ):
# url = resource.url
# if url.startswith('http://ckan.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://ckan-hetic.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://www.etalab2.fr/'):
# resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/')
# else:
# print resource.url
# model.repo.commit_and_remove()
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%www.data.gouv.fr%'),
# ):
# if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'):
# resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/')
# model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://new.data.gouv.fr/%'), ):
resource.url = resource.url.replace('http://new.data.gouv.fr/',
'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('https://new.data.gouv.fr/%'), ):
resource.url = resource.url.replace('https://new.data.gouv.fr/',
'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
return 0
def mock_pylons():
from paste.registry import Registry
import pylons
from pylons.util import AttribSafeContextObj
import ckan.lib.app_globals as app_globals
from ckan.lib.cli import MockTranslator
from ckan.config.routing import make_map
from pylons.controllers.util import Request, Response
from routes.util import URLGenerator
class TestPylonsSession(dict):
last_accessed = None
def save(self):
pass
registry = Registry()
registry.prepare()
context_obj = AttribSafeContextObj()
registry.register(pylons.c, context_obj)
app_globals_obj = app_globals.app_globals
registry.register(pylons.g, app_globals_obj)
request_obj = Request(dict(HTTP_HOST="nohost", REQUEST_METHOD="GET"))
registry.register(pylons.request, request_obj)
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
registry.register(pylons.response, Response())
mapper = make_map()
registry.register(pylons.url, URLGenerator(mapper, {}))
registry.register(pylons.session, TestPylonsSession())
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'supplier_id',
model.PackageExtra.package_id.in_(
model.Session.query(model.Package.id).filter(
model.Package.name.like('%-fork-%'))),
):
model.Session.delete(package_extra)
model.repo.commit_and_remove()
return 0
def load_config(config: Any, load_site_user: bool = True):
conf = _get_config(config)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf)
# Set this internal test request context with the configured environment so
# it can be used when calling url_for from the CLI.
global _cli_test_request_context
app = make_app(conf)
flask_app = app.apps['flask_app']._wsgi_app
_cli_test_request_context = flask_app.test_request_context()
registry = Registry()
registry.prepare()
site_user = None
if model.user_table.exists() and load_site_user:
site_user = logic.get_action('get_site_user')({
'ignore_auth': True
}, {})
## give routes enough information to run url_for
parsed = urlparse(cast(str, conf.get('ckan.site_url', 'http://0.0.0.0')))
request_config = routes.request_config()
request_config.host = parsed.netloc + parsed.path
request_config.protocol = parsed.scheme
return site_user
def load_translations(lang):
# Register a translator in this thread so that
# the _() functions in logic layer can work
import ckan.lib.i18n
from paste.registry import Registry
from pylons import translator
from pylons import request
registry = Registry()
registry.prepare()
class FakePylons:
translator = None
fakepylons = FakePylons()
class FakeRequest:
# Stores details of the translator
environ = {'pylons.pylons': fakepylons}
registry.register(request, FakeRequest())
# create translator
ckan.lib.i18n.set_lang(lang)
# pull out translator and register it
registry.register(translator, fakepylons.translator)
def setup_class(cls):
cls.registry = Registry()
cls.registry.prepare()
cls.context_obj = AttribSafeContextObj()
cls.registry.register(pylons.c, cls.context_obj)
cls.app_globals_obj = app_globals.Globals()
cls.registry.register(pylons.g, cls.app_globals_obj)
cls.request_obj = Request(dict(HTTP_HOST="nohost"))
cls.registry.register(pylons.request, cls.request_obj)
cls.translator_obj = MockTranslator()
cls.registry.register(pylons.translator, cls.translator_obj)
cls.buffet = pylons.templating.Buffet('genshi',
template_root='ckan.templates')
cls.registry.register(pylons.buffet, cls.buffet)
cls.registry.register(pylons.response, Response())
mapper = make_map()
cls.registry.register(pylons.url, URLGenerator(mapper, {}))
cls.registry.register(pylons.session, TestSession())
# Templates often want to find out the request's routes info, so put
# some dummy values into the routes_dict, so the templates that do
# this don't cause an exception.
pylons.request.environ.update({
'pylons.routes_dict': {
'action': 'test-action',
'controller': 'test-package::',
}
})
def create_request(path, environ=None):
"""Helper used in test cases to quickly setup a request obj.
``path``
The path will become PATH_INFO
``environ``
Additional environment
Returns an instance of the `webob.Request` object.
"""
# setup the environ
if environ is None:
environ = {}
environ.update(default_environ)
# create a "blank" WebOb Request object
# using Pylon's Request which is a webob Request plus
# some compatibility methods
req = Request.blank(path, environ)
# setup a Registry
reg = environ.setdefault('paste.registry', Registry())
reg.prepare()
# setup pylons.request to point to our Registry
reg.register(pylons.request, req)
# setup tmpl context
tmpl_context._push_object(ContextObj())
url._push_object(URLGenerator(default_map, environ))
return req
def __call__(self, environ, start_response):
app_iter = None
reg = environ.setdefault('paste.registry', Registry())
reg.prepare()
if self.streaming:
return self.streaming_iter(reg, environ, start_response)
try:
app_iter = self.application(environ, start_response)
except Exception, e:
# Regardless of if the content is an iterable, generator, list
# or tuple, we clean-up right now. If its an iterable/generator
# care should be used to ensure the generator has its own ref
# to the actual object
if environ.get('paste.evalexception'):
# EvalException is present in the WSGI stack
expected = False
for expect in environ.get('paste.expected_exceptions', []):
if isinstance(e, expect):
expected = True
if not expected:
# An unexpected exception: save state for EvalException
restorer.save_registry_state(environ)
reg.cleanup()
raise
def setup_environment_and_database(env_dir=None, enabled_plugins=''):
global_config = {}
env_dir = env_dir or '/invalid'
app_instance_secret = secret.secret_string()
app_config = {
'plugins': enabled_plugins,
'sqlalchemy.url': 'sqlite://',
'layout_template': 'layout',
'external_template': 'false',
'image_dir': os.path.join(env_dir, 'images'),
'media_dir': os.path.join(env_dir, 'media'),
'beaker.session.secret': app_instance_secret,
'sa_auth.cookie_secret': app_instance_secret,
}
pylons_config = load_environment(global_config, app_config)
paste_registry = Registry()
paste_registry.prepare()
app_globals = pylons_config['pylons.app_globals']
register_instance(paste_registry, 'app_globals', app_globals)
pylons_config.update({
'paste.registry': paste_registry,
'pylons.pylons': pylons,
})
metadata.create_all(bind=DBSession.bind, checkfirst=True)
return pylons_config
def register_translator():
# Workaround until core translation function defaults to Flask
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
from pylons import translator
registry = Registry()
registry.prepare()
registry.register(translator, MockTranslator())
def create_fake_env():
registry = Registry()
registry.prepare()
registry.register(session, SessionObject({}))
registry.register(request, Request.blank('/bootstrap'))
request.identity = {}
return session, request
def setup_class(cls):
super(FunctionalTestBase, cls).setup_class()
core_helpers.reset_db()
# Fix ckanext-hierarchy "c"
c = pylons.util.AttribSafeContextObj()
registry = Registry()
registry.prepare()
registry.register(pylons.c, c)
def register_translator():
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
log.info('Translator object was registered for the background job')
def setup_class(cls):
# Hack because the hierarchy extension uses c in some methods
# that are called outside the context of a web request
c = pylons.util.AttribSafeContextObj()
registry = Registry()
registry.prepare()
registry.register(pylons.c, c)
super(AuthTestBase, cls).setup_class()
def _test_restorer(stack, data):
# We need to test the request's specific Registry. Initialize it here so we
# can use it later (RegistryManager will re-use one preexisting in the
# environ)
registry = Registry()
extra_environ={'paste.throw_errors': False,
'paste.registry': registry}
request_id = restorer.get_request_id(extra_environ)
app = TestApp(stack)
res = app.get('/', extra_environ=extra_environ, expect_errors=True)
# Ensure all the StackedObjectProxies are empty after the RegistryUsingApp
# raises an Exception
for stacked, proxied_obj, test_cleanup in data:
only_key = list(proxied_obj.keys())[0]
try:
assert only_key not in stacked
assert False
except TypeError:
# Definitely empty
pass
# Ensure the StackedObjectProxies & Registry 'work' in the simulated
# EvalException context
replace = {'replace': 'dict'}
new = {'new': 'object'}
restorer.restoration_begin(request_id)
try:
for stacked, proxied_obj, test_cleanup in data:
# Ensure our original data magically re-appears in this context
only_key, only_val = list(proxied_obj.items())[0]
assert only_key in stacked and stacked[only_key] == only_val
# Ensure the Registry still works
registry.prepare()
registry.register(stacked, new)
assert 'new' in stacked and stacked['new'] == 'object'
registry.cleanup()
# Back to the original (pre-prepare())
assert only_key in stacked and stacked[only_key] == only_val
registry.replace(stacked, replace)
assert 'replace' in stacked and stacked['replace'] == 'dict'
if test_cleanup:
registry.cleanup()
try:
stacked._current_obj()
assert False
except TypeError:
# Definitely empty
pass
finally:
restorer.restoration_end()
def __init__(self, conf=None):
self.config = load_config(conf)
# package_update needs a translator defined i.e. _()
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
self.app = make_app(self.config.global_conf, **self.config.local_conf)
def _paste_registry(pylons_request):
"Internal helper method to get a new registry."
paste_registry = None
is_request_registered = is_object_registered(pylons_request)
if is_request_registered:
paste_registry = pylons_request.environ.get('paste.registry')
if paste_registry is None:
paste_registry = Registry()
paste_registry.prepare()
if is_request_registered:
pylons_request.environ['paste.registry'] = paste_registry
return paste_registry
def register_translator():
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
global registry
registry = Registry()
registry.prepare()
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def load_config(path):
import paste.deploy
conf = paste.deploy.appconfig('config:' + path)
import ckan, pylons
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
from ckan.lib.cli import MockTranslator
from paste.registry import Registry
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def command(dry_run=False):
from ckan import model
from ckanext.dgu.lib.resource_formats import match
from running_stats import StatsList
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
if not dry_run:
model.repo.new_revision()
# Add canonised formats to map
for format_ in res_type_map.keys():
res_type_map[canonise(format_)] = res_type_map[format_]
log.info('Tidying resource types')
stats = StatsList()
res_query = model.Session.query(model.Resource)
log.info('Tidying formats. Resources=%i Canonised formats=%i',
res_query.count(), len(set(res_type_map.values())))
for res in res_query:
canonised_fmt = canonise(res.format or '')
if canonised_fmt in res_type_map:
improved_fmt = res_type_map[canonised_fmt]
else:
improved_fmt = tidy(res.format)
match_ = match(improved_fmt)
if match_:
improved_fmt = match_
if (improved_fmt or '') != (res.format or ''):
if not dry_run:
res.format = improved_fmt
stats.add(improved_fmt, res.format)
else:
stats.add('No change', res.format)
if not dry_run:
model.repo.commit_and_remove()
log.info('Stats report: %r', stats.report())
print stats.report()
log.info('Warnings (%i): %r', len(warnings), warnings)
def load_config(path):
import paste.deploy
from paste.registry import Registry
from command import MockTranslator
conf = paste.deploy.appconfig('config:' + os.path.abspath(path))
import ckan
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def __init__(self,
worker_id,
heartbeat_interval=60000,
action='update'):
super(UpdateIntentsWorker, self).__init__(worker_id, heartbeat_interval)
self.intents_worker = UserIntentsWorker(intents_extractor)
self.action = action
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
def fake_request(pylons_config,
server_name='mediacore.example',
language='en',
method='GET',
request_uri='/',
post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError(
'You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' %
(server_name, request_uri),
method.upper(),
request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediacore.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry)
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def fake_request(pylons_config, server_name='mediadrop.example', language='en',
method='GET', request_uri='/', post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError('You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri),
method.upper(), request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# TODO: Use ContextObj() for Pylons 0.10
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
mediacore_i18n_path = os.path.join(os.path.dirname(mediacore.__file__), 'i18n')
translator = Translator(language, dict(mediacore=mediacore_i18n_path))
# not sure why but sometimes pylons.translator is not a StackedObjectProxy
# but just a regular Translator.
if not hasattr(pylons.translator, '_push_object'):
pylons.translator = StackedObjectProxy()
paste_registry.replace(pylons.translator, translator)
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def register_translator():
# https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py
# If not set (in cli access), patch the a translator with a mock, so the
# _() functions in logic layer don't cause failure.
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
if 'registery' not in globals():
global registry
registry = Registry()
registry.prepare()
if 'translator_obj' not in globals():
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def _register_translator():
"""
Register a translator in this thread.
"""
global registry
try:
registry
except NameError:
registry = Registry()
registry.prepare()
global translator_obj
try:
translator_obj
except NameError:
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
try:
fileConfig(self.filename)
except Exception: pass
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
class ModelTest(object):
config = appconfig('config:test.ini', relative_to=conf_dir)
root_path = os.path.join(conf_dir, 'r2')
paths = {
'root': root_path,
'controllers': os.path.join(root_path, 'controllers'),
'templates': tmpl_dirs,
'static_files': os.path.join(root_path, 'public')
}
registry = Registry()
registry.prepare()
globals = Globals(config.global_conf, config.local_conf, paths)
registry.register(pylons.g, globals)
registry.register(pylons.translator, NullTranslations())
context_obj=ContextObj()
registry.register(pylons.c, context_obj)
r2.config.cache = globals.cache
def load_config(config, load_site_user=True):
conf = _get_config(config)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf)
# Set this internal test request context with the configured environment so
# it can be used when calling url_for from the CLI.
global _cli_test_request_context
app = make_app(conf)
flask_app = app.apps['flask_app']._wsgi_app
_cli_test_request_context = flask_app.test_request_context()
registry = Registry()
registry.prepare()
import pylons
registry.register(pylons.translator, MockTranslator())
site_user = None
if model.user_table.exists() and load_site_user:
# If the DB has already been initialized, create and register
# a pylons context object, and add the site user to it, so the
# auth works as in a normal web request
c = pylons.util.AttribSafeContextObj()
registry.register(pylons.c, c)
site_user = logic.get_action('get_site_user')({
'ignore_auth': True
}, {})
pylons.c.user = site_user['name']
pylons.c.userobj = model.User.get(site_user['name'])
## give routes enough information to run url_for
parsed = urlparse(conf.get('ckan.site_url', 'http://0.0.0.0'))
request_config = routes.request_config()
request_config.host = parsed.netloc + parsed.path
request_config.protocol = parsed.scheme
return site_user
