def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'supplier_id',
model.PackageExtra.package_id.in_(
model.Session.query(model.Package.id).filter(model.Package.name.like('%-fork-%'))),
):
model.Session.delete(package_extra)
model.repo.commit_and_remove()
return 0
class CkanCommand(paste.script.command.Command):
parser = paste.script.command.Command.standard_parser(verbose=True)
parser.add_option('-c', '--config', dest='config',
default='development.ini', help='Config file to use.')
parser.add_option('-f', '--file',
action='store',
dest='file_path',
help="File to dump results to (if needed)")
default_verbosity = 1
group_name = 'ckan'
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _setup_app(self):
cmd = paste.script.appinstall.SetupCommand('setup-app')
cmd.run([self.filename])
def setup_environment_and_database(env_dir=None, enabled_plugins=''):
global_config = {}
env_dir = env_dir or '/invalid'
app_instance_secret = secret.secret_string()
app_config = {
'plugins': enabled_plugins,
'sqlalchemy.url': 'sqlite://',
'layout_template': 'layout',
'external_template': 'false',
'image_dir': os.path.join(env_dir, 'images'),
'media_dir': os.path.join(env_dir, 'media'),
'beaker.session.secret': app_instance_secret,
'sa_auth.cookie_secret': app_instance_secret,
}
pylons_config = load_environment(global_config, app_config)
paste_registry = Registry()
paste_registry.prepare()
app_globals = pylons_config['pylons.app_globals']
register_instance(paste_registry, 'app_globals', app_globals)
pylons_config.update({
'paste.registry': paste_registry,
'pylons.pylons': pylons,
})
metadata.create_all(bind=DBSession.bind, checkfirst=True)
return pylons_config
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage_granularity',
model.PackageExtra.value == 'france',
):
package = package_extra.package
if package.private or package.state != 'active':
log.warning(u'Territorial coverage granularity of package {} must be manually corrected'.format(
package.name))
continue
package_extra.value = 'pays'
model.repo.commit_and_remove()
return 0
class HelloworldCommand(Command):
'''
A base class to use when the full Pylons environment should be loaded
'''
summary = __doc__.split("\n") [0]
usage = __doc__
group_name = "helloworld"
def __init__(self, name):
Command.__init__(self, name)
self.parser = Command.standard_parser(verbose=False)
self.parser.add_option('-c', '--config', dest='config', default='development.ini', help='Config file to use.')
return
def command(self):
pass
def _load_config(self):
from paste.deploy import appconfig
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig (self.filename)
conf = appconfig('config:' + self.filename)
# We have now loaded the config. Now we can import helloworld for the first time.
from helloworld.config.environment import load_environment
load_environment (conf.global_conf, conf.local_conf)
self.registry = Registry()
self.registry.prepare()
return
def setup_environment_and_database(env_dir=None, enabled_plugins=''):
global_config = {}
env_dir = env_dir or '/invalid'
app_instance_secret = secret.secret_string()
app_config = {
'plugins': enabled_plugins,
'sqlalchemy.url': 'sqlite://',
'layout_template': 'layout',
'external_template': 'false',
'image_dir': os.path.join(env_dir, 'images'),
'media_dir': os.path.join(env_dir, 'media'),
'beaker.session.secret': app_instance_secret,
'sa_auth.cookie_secret': app_instance_secret,
}
pylons_config = load_environment(global_config, app_config)
paste_registry = Registry()
paste_registry.prepare()
app_globals = pylons_config['pylons.app_globals']
register_instance(paste_registry, 'app_globals', app_globals)
pylons_config.update({
'paste.registry': paste_registry,
'pylons.pylons': pylons,
})
metadata.create_all(bind=DBSession.bind, checkfirst=True)
return pylons_config
class CkanCommand(paste.script.command.Command):
parser = paste.script.command.Command.standard_parser(verbose=True)
parser.add_option('-c', '--config', dest='config',
default='development.ini', help='Config file to use.')
parser.add_option('-f', '--file',
action='store',
dest='file_path',
help="File to dump results to (if needed)")
default_verbosity = 1
group_name = 'ckan'
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _setup_app(self):
cmd = paste.script.appinstall.SetupCommand('setup-app')
cmd.run([self.filename])
def load_config(config: Any, load_site_user: bool = True):
conf = _get_config(config)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf)
# Set this internal test request context with the configured environment so
# it can be used when calling url_for from the CLI.
global _cli_test_request_context
app = make_app(conf)
flask_app = app.apps['flask_app']._wsgi_app
_cli_test_request_context = flask_app.test_request_context()
registry = Registry()
registry.prepare()
site_user = None
if model.user_table.exists() and load_site_user:
site_user = logic.get_action('get_site_user')({
'ignore_auth': True
}, {})
## give routes enough information to run url_for
parsed = urlparse(cast(str, conf.get('ckan.site_url', 'http://0.0.0.0')))
request_config = routes.request_config()
request_config.host = parsed.netloc + parsed.path
request_config.protocol = parsed.scheme
return site_user
def register_translator():
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
log.info('Translator object was registered for the background job')
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%etalab2.fr%'),
# ):
# url = resource.url
# if url.startswith('http://ckan.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://ckan-hetic.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://www.etalab2.fr/'):
# resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/')
# else:
# print resource.url
# model.repo.commit_and_remove()
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%www.data.gouv.fr%'),
# ):
# if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'):
# resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/')
# model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://new.data.gouv.fr/%'),
):
resource.url = resource.url.replace('http://new.data.gouv.fr/', 'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('https://new.data.gouv.fr/%'),
):
resource.url = resource.url.replace('https://new.data.gouv.fr/', 'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
return 0
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-t', '--territoria-url', help = 'Territoria URL', required = True)
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
kind_code_name_by_kind_code = {}
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage',
):
if package_extra.value == 'Coutry/FR':
kind_code_name = 'Country/FR/FRANCE'
elif package_extra.value == 'InternationalOrganization/EU':
kind_code_name = 'InternationalOrganization/UE/UNION EUROPEENNE'
elif package_extra.value.count('/') == 1:
kind_code_name = kind_code_name_by_kind_code.get(package_extra.value)
if kind_code_name is None:
kind, code = package_extra.value.split('/')
try:
response = urllib2.urlopen(urlparse.urljoin(args.territoria_url,
'/api/v1/territory?{}'.format(urllib.urlencode(dict(
code = code,
kind = kind,
), doseq = True))))
except urllib2.HTTPError, response:
print package_extra.value
raise
response_dict = json.loads(response.read())
main_postal_distribution = response_dict['data']['main_postal_distribution']
kind_code_name_by_kind_code[package_extra.value] = kind_code_name = u'/'.join([kind, code,
main_postal_distribution])
print kind_code_name
else:
continue
package = package_extra.package
if package.private or package.state != 'active':
log.warning(u'Territorial coverage of package {} must be manually corrected'.format(package.name))
continue
package_extra.value = kind_code_name
def _paste_registry(pylons_request):
"Internal helper method to get a new registry."
paste_registry = None
is_request_registered = is_object_registered(pylons_request)
if is_request_registered:
paste_registry = pylons_request.environ.get('paste.registry')
if paste_registry is None:
paste_registry = Registry()
paste_registry.prepare()
if is_request_registered:
pylons_request.environ['paste.registry'] = paste_registry
return paste_registry
def register_translator():
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
global registry
registry = Registry()
registry.prepare()
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def _paste_registry(pylons_request):
"Internal helper method to get a new registry."
paste_registry = None
is_request_registered = is_object_registered(pylons_request)
if is_request_registered:
paste_registry = pylons_request.environ.get('paste.registry')
if paste_registry is None:
paste_registry = Registry()
paste_registry.prepare()
if is_request_registered:
pylons_request.environ['paste.registry'] = paste_registry
return paste_registry
def load_config(path):
import paste.deploy
conf = paste.deploy.appconfig('config:' + path)
import ckan, pylons
ckan.config.environment.load_environment(conf.global_conf,
conf.local_conf)
from ckan.lib.cli import MockTranslator
from paste.registry import Registry
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(pylons.translator, translator_obj)
def command(dry_run=False):
from ckan import model
from ckanext.dgu.lib.resource_formats import match
from running_stats import StatsList
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
if not dry_run:
model.repo.new_revision()
# Add canonised formats to map
for format_ in res_type_map.keys():
res_type_map[canonise(format_)] = res_type_map[format_]
log.info('Tidying resource types')
stats = StatsList()
res_query = model.Session.query(model.Resource)
log.info('Tidying formats. Resources=%i Canonised formats=%i',
res_query.count(), len(set(res_type_map.values())))
for res in res_query:
canonised_fmt = canonise(res.format or '')
if canonised_fmt in res_type_map:
improved_fmt = res_type_map[canonised_fmt]
else:
improved_fmt = tidy(res.format)
match_ = match(improved_fmt)
if match_:
improved_fmt = match_
if (improved_fmt or '') != (res.format or ''):
if not dry_run:
res.format = improved_fmt
stats.add(improved_fmt, res.format)
else:
stats.add('No change', res.format)
if not dry_run:
model.repo.commit_and_remove()
log.info('Stats report: %r', stats.report())
print stats.report()
log.info('Warnings (%i): %r', len(warnings), warnings)
def load_translations(lang):
# Register a translator in this thread so that
# the _() functions in logic layer can work
import ckan.lib.i18n
from paste.registry import Registry
from pylons import translator
from pylons import request
registry = Registry()
registry.prepare()
class FakePylons:
translator = None
fakepylons = FakePylons()
class FakeRequest:
# Stores details of the translator
environ = {'pylons.pylons': fakepylons}
registry.register(request, FakeRequest())
# create translator
ckan.lib.i18n.set_lang(lang)
# pull out translator and register it
registry.register(translator, fakepylons.translator)
def load_config(path):
import paste.deploy
from paste.registry import Registry
from command import MockTranslator
conf = paste.deploy.appconfig('config:' + os.path.abspath(path))
import ckan
ckan.config.environment.load_environment(conf.global_conf,
conf.local_conf)
registry=Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def create_fake_env():
registry = Registry()
registry.prepare()
registry.register(session, SessionObject({}))
registry.register(request, Request.blank('/bootstrap'))
request.identity = {}
return session, request
def _register_translator():
"""
Register a translator in this thread.
"""
global registry
try:
registry
except NameError:
registry = Registry()
registry.prepare()
global translator_obj
try:
translator_obj
except NameError:
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def register_translator():
# https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py
# If not set (in cli access), patch the a translator with a mock, so the
# _() functions in logic layer don't cause failure.
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
if 'registery' not in globals():
global registry
registry = Registry()
registry.prepare()
if 'translator_obj' not in globals():
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def __call__(self, environ, start_response):
app_iter = None
reg = environ.setdefault('paste.registry', Registry())
reg.prepare()
if self.streaming:
return self.streaming_iter(reg, environ, start_response)
try:
app_iter = self.application(environ, start_response)
except Exception, e:
# Regardless of if the content is an iterable, generator, list
# or tuple, we clean-up right now. If its an iterable/generator
# care should be used to ensure the generator has its own ref
# to the actual object
if environ.get('paste.evalexception'):
# EvalException is present in the WSGI stack
expected = False
for expect in environ.get('paste.expected_exceptions', []):
if isinstance(e, expect):
expected = True
if not expected:
# An unexpected exception: save state for EvalException
restorer.save_registry_state(environ)
reg.cleanup()
raise
def setup_class(cls):
cls.registry = Registry()
cls.registry.prepare()
cls.context_obj = AttribSafeContextObj()
cls.registry.register(pylons.c, cls.context_obj)
cls.app_globals_obj = app_globals.Globals()
cls.registry.register(pylons.g, cls.app_globals_obj)
cls.request_obj = Request(dict(HTTP_HOST="nohost"))
cls.registry.register(pylons.request, cls.request_obj)
cls.translator_obj = MockTranslator()
cls.registry.register(pylons.translator, cls.translator_obj)
cls.buffet = pylons.templating.Buffet('genshi',
template_root='ckan.templates')
cls.registry.register(pylons.buffet, cls.buffet)
cls.registry.register(pylons.response, Response())
mapper = make_map()
cls.registry.register(pylons.url, URLGenerator(mapper, {}))
cls.registry.register(pylons.session, TestSession())
# Templates often want to find out the request's routes info, so put
# some dummy values into the routes_dict, so the templates that do
# this don't cause an exception.
pylons.request.environ.update({
'pylons.routes_dict': {
'action': 'test-action',
'controller': 'test-package::',
}
})
def create_request(path, environ=None):
"""Helper used in test cases to quickly setup a request obj.
``path``
The path will become PATH_INFO
``environ``
Additional environment
Returns an instance of the `webob.Request` object.
"""
# setup the environ
if environ is None:
environ = {}
environ.update(default_environ)
# create a "blank" WebOb Request object
# using Pylon's Request which is a webob Request plus
# some compatibility methods
req = Request.blank(path, environ)
# setup a Registry
reg = environ.setdefault('paste.registry', Registry())
reg.prepare()
# setup pylons.request to point to our Registry
reg.register(pylons.request, req)
# setup tmpl context
tmpl_context._push_object(ContextObj())
url._push_object(URLGenerator(default_map, environ))
return req
def register_translator():
# Workaround until core translation function defaults to Flask
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
from pylons import translator
registry = Registry()
registry.prepare()
registry.register(translator, MockTranslator())
def setUp(self):
super(AutoCompleterTest, self).setUp()
# Création d'un environnement WSGI pour les tests.
# Par défaut, l'utilisateur n'est pas identifé
# et n'appartient donc à aucun groupe.
environ = {'repoze.what.credentials': {
'groups': [],
}
}
request = Request(environ)
config = VigiloAppConfig('TurboGears')
config.init_managers_predicate()
registry = Registry()
registry.prepare()
registry.register(tg.request, request)
registry.register(tg.config, config)
self.ctrl = AutoCompleteController()
# Par défaut, l'utilisateur n'est pas identifé.
tg.request.identity = {'repoze.who.userid': None}
self.accounts = (
(u'manager', u'managers'), # Accès à tout (manager)
(u'nobody', u'nobody'), # Accès à rien.
(u'direct', u'direct'), # Permissions directe sur l'objet.
(u'indirect', u'indirect'), # Permissions indirectes sur l'objet.
)
# Création des comptes utilisateurs.
for username, group in self.accounts:
functions.add_user(username, u'%s@test' % username,
u'', u'', group)
# Création de l'arborescence des objets.
self.create_deps()
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
try:
fileConfig(self.filename)
except Exception: pass
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def fake_request(pylons_config, server_name='mediacore.example', language='en',
method='GET', request_uri='/', post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError('You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri),
method.upper(), request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediacore.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry,
locale_dirs=pylons_config.get('locale_dirs'))
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def command(config_ini, nodepublisher_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config,'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
model.repo.new_revision()
log.info('Reading %s', nodepublisher_csv)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
nid, title = row
publishers[ int(nid) ] = munge_title_to_name(title)
# Mappings where we are getting rid of duplicate publishers
publishers[16268] = publishers[11408] # UKSA -> ONS
publishers[11606] = publishers[11408] # ONS
publishers[20054] = publishers[16248] # Met Office
publishers[33036] = publishers[15255] # Windsor & Maidenhead
publishers[32619] = publishers[33245] # Monmouthshire
publishers[12662] = publishers[11567] # NHS
update_datasets()
generate_harvest_publishers()
log.info('Warnings: %r', warnings)
def init_mediadrop(config_filename, here_dir=None, disable_logging=False):
if not os.path.exists(config_filename):
raise IOError('Config file %r does not exist.' % config_filename)
if here_dir is None:
here_dir = os.getcwd()
if not disable_logging:
logging_config.fileConfig(config_filename)
conf = appconfig('config:%s' % config_filename, relative_to=here_dir)
pylons_config = load_environment(conf.global_conf, conf.local_conf)
paste_registry = Registry()
paste_registry.prepare()
pylons_config['paste.registry'] = paste_registry
app_globals = pylons_config['pylons.app_globals']
register_instance(paste_registry, 'app_globals', app_globals)
register_instance(paste_registry, 'config', pylons_config)
fake_request(pylons_config, registry=paste_registry)
setup_global_translator(registry=paste_registry)
def fake_request(
pylons_config, server_name="mediadrop.example", language="en", method="GET", request_uri="/", post_vars=None
):
app_globals = pylons_config["pylons.app_globals"]
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != "POST":
raise ValueError("You must not specify post_vars for request method %r" % method)
wsgi_environ = create_wsgi_environ(
"http://%s%s" % (server_name, request_uri), method.upper(), request_body=post_vars
)
request = Request(wsgi_environ, charset="utf-8")
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type="application/xml", charset="utf-8")
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config["routes.map"], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediadrop.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry, locale_dirs=pylons_config.get("locale_dirs"))
wsgi_environ.update({"pylons.pylons": pylons, "paste.registry": paste_registry})
return request
def _load_config(self):
from paste.deploy import appconfig
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%etalab2.fr%'),
# ):
# url = resource.url
# if url.startswith('http://ckan.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://ckan-hetic.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://www.etalab2.fr/'):
# resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/')
# else:
# print resource.url
# model.repo.commit_and_remove()
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%www.data.gouv.fr%'),
# ):
# if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'):
# resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/')
# model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://new.data.gouv.fr/%'), ):
resource.url = resource.url.replace('http://new.data.gouv.fr/',
'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('https://new.data.gouv.fr/%'), ):
resource.url = resource.url.replace('https://new.data.gouv.fr/',
'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
return 0
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'supplier_id',
model.PackageExtra.package_id.in_(
model.Session.query(model.Package.id).filter(
model.Package.name.like('%-fork-%'))),
):
model.Session.delete(package_extra)
model.repo.commit_and_remove()
return 0
def fake_request(pylons_config, server_name='mediadrop.example', language='en',
method='GET', request_uri='/', post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError('You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri),
method.upper(), request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# TODO: Use ContextObj() for Pylons 0.10
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
mediacore_i18n_path = os.path.join(os.path.dirname(mediacore.__file__), 'i18n')
translator = Translator(language, dict(mediacore=mediacore_i18n_path))
# not sure why but sometimes pylons.translator is not a StackedObjectProxy
# but just a regular Translator.
if not hasattr(pylons.translator, '_push_object'):
pylons.translator = StackedObjectProxy()
paste_registry.replace(pylons.translator, translator)
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for organization_name, is_service_public in sorted(is_public_service_by_organization_name.iteritems()):
if not is_service_public:
continue
organization = model.Session.query(model.Group).filter(
model.Group.is_organization == True,
model.Group.name == organization_name,
).first()
if organization is None:
log.warning(u'Unknown organization: {}'.format(organization_name))
continue
if organization.certified_public_service is None:
log.info(u'Certifying "{}" as public service'.format(organization_name))
certified_public_service = etalab_model.CertifiedPublicService()
certified_public_service.organization_id = organization.id
model.Session.add(certified_public_service)
model.repo.commit_and_remove()
return 0
def setup_class(cls):
super(FunctionalTestBase, cls).setup_class()
core_helpers.reset_db()
# Fix ckanext-hierarchy "c"
c = pylons.util.AttribSafeContextObj()
registry = Registry()
registry.prepare()
registry.register(pylons.c, c)
def register_translator():
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
log.info('Translator object was registered for the background job')
def setup_class(cls):
# Hack because the hierarchy extension uses c in some methods
# that are called outside the context of a web request
c = pylons.util.AttribSafeContextObj()
registry = Registry()
registry.prepare()
registry.register(pylons.c, c)
super(AuthTestBase, cls).setup_class()
def __init__(self, conf=None):
self.config = load_config(conf)
# package_update needs a translator defined i.e. _()
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
self.app = make_app(self.config.global_conf, **self.config.local_conf)
def load_config(config, load_site_user=True):
conf = _get_config(config)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf)
# Set this internal test request context with the configured environment so
# it can be used when calling url_for from the CLI.
global _cli_test_request_context
app = make_app(conf)
flask_app = app.apps['flask_app']._wsgi_app
_cli_test_request_context = flask_app.test_request_context()
registry = Registry()
registry.prepare()
import pylons
registry.register(pylons.translator, MockTranslator())
site_user = None
if model.user_table.exists() and load_site_user:
# If the DB has already been initialized, create and register
# a pylons context object, and add the site user to it, so the
# auth works as in a normal web request
c = pylons.util.AttribSafeContextObj()
registry.register(pylons.c, c)
site_user = logic.get_action('get_site_user')({
'ignore_auth': True
}, {})
pylons.c.user = site_user['name']
pylons.c.userobj = model.User.get(site_user['name'])
## give routes enough information to run url_for
parsed = urlparse(conf.get('ckan.site_url', 'http://0.0.0.0'))
request_config = routes.request_config()
request_config.host = parsed.netloc + parsed.path
request_config.protocol = parsed.scheme
return site_user
def _load_config(self):
from paste.deploy import appconfig
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig (self.filename)
conf = appconfig('config:' + self.filename)
# We have now loaded the config. Now we can import helloworld for the first time.
from helloworld.config.environment import load_environment
load_environment (conf.global_conf, conf.local_conf)
self.registry = Registry()
self.registry.prepare()
return
def register_translator():
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
global registry
registry = Registry()
registry.prepare()
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def load_config(path):
import paste.deploy
conf = paste.deploy.appconfig('config:' + path)
import ckan, pylons
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
from ckan.lib.cli import MockTranslator
from paste.registry import Registry
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
class ModelTest(object):
config = appconfig('config:test.ini', relative_to=conf_dir)
root_path = os.path.join(conf_dir, 'r2')
paths = {
'root': root_path,
'controllers': os.path.join(root_path, 'controllers'),
'templates': tmpl_dirs,
'static_files': os.path.join(root_path, 'public')
}
registry = Registry()
registry.prepare()
globals = Globals(config.global_conf, config.local_conf, paths)
registry.register(pylons.g, globals)
registry.register(pylons.translator, NullTranslations())
context_obj=ContextObj()
registry.register(pylons.c, context_obj)
r2.config.cache = globals.cache
def command(dry_run=False):
from ckan import model
from ckanext.dgu.lib.resource_formats import match
from running_stats import StatsList
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
if not dry_run:
model.repo.new_revision()
# Add canonised formats to map
for format_ in res_type_map.keys():
res_type_map[canonise(format_)] = res_type_map[format_]
log.info('Tidying resource types')
stats = StatsList()
res_query = model.Session.query(model.Resource)
log.info('Tidying formats. Resources=%i Canonised formats=%i',
res_query.count(), len(set(res_type_map.values())))
for res in res_query:
canonised_fmt = canonise(res.format or '')
if canonised_fmt in res_type_map:
improved_fmt = res_type_map[canonised_fmt]
else:
improved_fmt = tidy(res.format)
match_ = match(improved_fmt)
if match_:
improved_fmt = match_
if (improved_fmt or '') != (res.format or ''):
if not dry_run:
res.format = improved_fmt
stats.add(improved_fmt, res.format)
else:
stats.add('No change', res.format)
if not dry_run:
model.repo.commit_and_remove()
log.info('Stats report: %r', stats.report())
print stats.report()
log.info('Warnings (%i): %r', len(warnings), warnings)
def setup(self):
self.registry=Registry()
self.registry.prepare()
self.context_obj=ContextObj()
self.registry.register(pylons.c, self.context_obj)
pylons.c.errors = None
self.request_obj=Request(dict(HTTP_HOST="nohost"))
self.registry.register(pylons.request, self.request_obj)
self.translator_obj=MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
self.buffet = pylons.templating.Buffet('genshi', template_root='ckan.templates')
self.registry.register(pylons.buffet, self.buffet)
self.registry.register(pylons.response, Response())
self.registry.register(pylons.url, None)
def load_config(path):
import paste.deploy
from paste.registry import Registry
from command import MockTranslator
conf = paste.deploy.appconfig('config:' + os.path.abspath(path))
import ckan
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def fake_request(pylons_config,
server_name='mediacore.example',
language='en',
method='GET',
request_uri='/',
post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError(
'You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' %
(server_name, request_uri),
method.upper(),
request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediacore.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry)
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def __init__(self,
worker_id,
heartbeat_interval=60000,
action='update'):
super(UpdateIntentsWorker, self).__init__(worker_id, heartbeat_interval)
self.intents_worker = UserIntentsWorker(intents_extractor)
self.action = action
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
def __init__(self):
super(MockedJobController, self).__init__()
# Register into paste the fake request and response objects
registry = Registry()
registry.prepare()
registry.register(request, MockRequest())
registry.register(response, MockResponse())
# This is expected to be set by Pylons, so we set it here
self._py_object = type(
"", (),
{"__init__": (lambda s, **kwargs: s.__dict__.update(kwargs))})(
request=request, response=response)
# Inject fake proxy, so the submission works
delegated = Credential(dlg_id='12345',
dn='/DN=1234',
proxy='',
termination_time=datetime.utcnow() +
timedelta(hours=5))
Session.merge(delegated)
Session.commit()
def __init__(self):
super(MockedJobController, self).__init__()
# Register into paste the fake request and response objects
registry = Registry()
registry.prepare()
registry.register(request, MockRequest())
registry.register(response, MockResponse())
# This is expected to be set by Pylons, so we set it here
self._py_object = type("", (), {
"__init__": (lambda s, **kwargs: s.__dict__.update(kwargs))
})(request=request, response=response)
# Inject fake proxy, so the submission works
delegated = Credential(
dlg_id='12345', dn='/DN=1234', proxy='',
termination_time=datetime.utcnow() + timedelta(hours=5)
)
Session.merge(delegated)
Session.commit()
def load_translations(lang):
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from pylons import request
registry = Registry()
registry.prepare()
class FakePylons:
translator = None
fakepylons = FakePylons()
class FakeRequest:
# Stores details of the translator
environ = {'pylons.pylons': fakepylons}
registry.register(request, FakeRequest())
# create translator
i18n.set_lang(lang)
# pull out translator and register it
registry.register(translator, fakepylons.translator)
def command(input_csv, config_ini, commit=False):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
from ckan import model
from ckan.logic import get_action
from ckan.lib.cli import MockTranslator
model.init_model(engine)
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
ctx = {
'model': model, 'session': model.Session,
'user': get_action('get_site_user')({'model': model,'ignore_auth': True}, {})['name']
}
if commit:
rev = model.repo.new_revision()
packages_to_check = set()
reader = csv.reader(open(input_csv, 'r'))
for row in reader:
# For each URL in the csv, get the list of resources referencing
# that URL
resources = model.Session.query(model.Resource)\
.filter(model.Resource.state=='active')\
.filter(model.Resource.url==row[0]).all()
for resource in resources:
# For each resource, add the package to the list
packages_to_check.add(resource.get_package_id())
# Delete the resource
resource.state = 'deleted'
model.Session.add(resource)
if commit:
model.Session.commit()
print "Deleted resource: {0}".format(resource.id)
stats.increment("Deleted resource")
for pid in packages_to_check:
# for each package we need to check, see if it has any
# resources left, it not, delete it.
pkg = model.Package.get(pid)
if len(pkg.resources) == 0:
pkg.state = 'deleted'
model.Session.add(pkg)
if commit:
model.Session.commit()
stats.increment('Deleted packages')
print "Deleted package: {0}".format(pkg.name)
if commit:
model.repo.commit_and_remove()
else:
print ""
print '*' * 60
print "DON'T PANIC, this was a dry run, nothing was committed"
print '*' * 60
print ''
print '*' * 60, 'Deletion Report'
print stats.report(order_by_title=True)
