class HelloworldCommand(Command):
'''
A base class to use when the full Pylons environment should be loaded
'''
summary = __doc__.split("\n") [0]
usage = __doc__
group_name = "helloworld"
def __init__(self, name):
Command.__init__(self, name)
self.parser = Command.standard_parser(verbose=False)
self.parser.add_option('-c', '--config', dest='config', default='development.ini', help='Config file to use.')
return
def command(self):
pass
def _load_config(self):
from paste.deploy import appconfig
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig (self.filename)
conf = appconfig('config:' + self.filename)
# We have now loaded the config. Now we can import helloworld for the first time.
from helloworld.config.environment import load_environment
load_environment (conf.global_conf, conf.local_conf)
self.registry = Registry()
self.registry.prepare()
return
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%etalab2.fr%'),
# ):
# url = resource.url
# if url.startswith('http://ckan.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://ckan-hetic.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://www.etalab2.fr/'):
# resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/')
# else:
# print resource.url
# model.repo.commit_and_remove()
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%www.data.gouv.fr%'),
# ):
# if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'):
# resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/')
# model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://new.data.gouv.fr/%'), ):
resource.url = resource.url.replace('http://new.data.gouv.fr/',
'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('https://new.data.gouv.fr/%'), ):
resource.url = resource.url.replace('https://new.data.gouv.fr/',
'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
return 0
class CkanCommand(paste.script.command.Command):
parser = paste.script.command.Command.standard_parser(verbose=True)
parser.add_option('-c', '--config', dest='config',
default='development.ini', help='Config file to use.')
parser.add_option('-f', '--file',
action='store',
dest='file_path',
help="File to dump results to (if needed)")
default_verbosity = 1
group_name = 'ckan'
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _setup_app(self):
cmd = paste.script.appinstall.SetupCommand('setup-app')
cmd.run([self.filename])
def mock_pylons():
from paste.registry import Registry
import pylons
from pylons.util import AttribSafeContextObj
import ckan.lib.app_globals as app_globals
from ckan.lib.cli import MockTranslator
from ckan.config.routing import make_map
from pylons.controllers.util import Request, Response
from routes.util import URLGenerator
class TestPylonsSession(dict):
last_accessed = None
def save(self):
pass
registry = Registry()
registry.prepare()
context_obj = AttribSafeContextObj()
registry.register(pylons.c, context_obj)
app_globals_obj = app_globals.app_globals
registry.register(pylons.g, app_globals_obj)
request_obj = Request(dict(HTTP_HOST="nohost", REQUEST_METHOD="GET"))
registry.register(pylons.request, request_obj)
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
registry.register(pylons.response, Response())
mapper = make_map()
registry.register(pylons.url, URLGenerator(mapper, {}))
registry.register(pylons.session, TestPylonsSession())
def setup_environment_and_database(env_dir=None, enabled_plugins=''):
global_config = {}
env_dir = env_dir or '/invalid'
app_instance_secret = secret.secret_string()
app_config = {
'plugins': enabled_plugins,
'sqlalchemy.url': 'sqlite://',
'layout_template': 'layout',
'external_template': 'false',
'image_dir': os.path.join(env_dir, 'images'),
'media_dir': os.path.join(env_dir, 'media'),
'beaker.session.secret': app_instance_secret,
'sa_auth.cookie_secret': app_instance_secret,
}
pylons_config = load_environment(global_config, app_config)
paste_registry = Registry()
paste_registry.prepare()
app_globals = pylons_config['pylons.app_globals']
register_instance(paste_registry, 'app_globals', app_globals)
pylons_config.update({
'paste.registry': paste_registry,
'pylons.pylons': pylons,
})
metadata.create_all(bind=DBSession.bind, checkfirst=True)
return pylons_config
def load_config(config: Any, load_site_user: bool = True):
conf = _get_config(config)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf)
# Set this internal test request context with the configured environment so
# it can be used when calling url_for from the CLI.
global _cli_test_request_context
app = make_app(conf)
flask_app = app.apps['flask_app']._wsgi_app
_cli_test_request_context = flask_app.test_request_context()
registry = Registry()
registry.prepare()
site_user = None
if model.user_table.exists() and load_site_user:
site_user = logic.get_action('get_site_user')({
'ignore_auth': True
}, {})
## give routes enough information to run url_for
parsed = urlparse(cast(str, conf.get('ckan.site_url', 'http://0.0.0.0')))
request_config = routes.request_config()
request_config.host = parsed.netloc + parsed.path
request_config.protocol = parsed.scheme
return site_user
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage_granularity',
model.PackageExtra.value == 'france',
):
package = package_extra.package
if package.private or package.state != 'active':
log.warning(u'Territorial coverage granularity of package {} must be manually corrected'.format(
package.name))
continue
package_extra.value = 'pays'
model.repo.commit_and_remove()
return 0
def setUp(self):
super(AutoCompleterTest, self).setUp()
# Création d'un environnement WSGI pour les tests.
# Par défaut, l'utilisateur n'est pas identifé
# et n'appartient donc à aucun groupe.
environ = {'repoze.what.credentials': {
'groups': [],
}
}
request = Request(environ)
config = VigiloAppConfig('TurboGears')
config.init_managers_predicate()
registry = Registry()
registry.prepare()
registry.register(tg.request, request)
registry.register(tg.config, config)
self.ctrl = AutoCompleteController()
# Par défaut, l'utilisateur n'est pas identifé.
tg.request.identity = {'repoze.who.userid': None}
self.accounts = (
(u'manager', u'managers'), # Accès à tout (manager)
(u'nobody', u'nobody'), # Accès à rien.
(u'direct', u'direct'), # Permissions directe sur l'objet.
(u'indirect', u'indirect'), # Permissions indirectes sur l'objet.
)
# Création des comptes utilisateurs.
for username, group in self.accounts:
functions.add_user(username, u'%s@test' % username,
u'', u'', group)
# Création de l'arborescence des objets.
self.create_deps()
def load_translations(lang):
# Register a translator in this thread so that
# the _() functions in logic layer can work
import ckan.lib.i18n
from paste.registry import Registry
from pylons import translator
from pylons import request
registry = Registry()
registry.prepare()
class FakePylons:
translator = None
fakepylons = FakePylons()
class FakeRequest:
# Stores details of the translator
environ = {'pylons.pylons': fakepylons}
registry.register(request, FakeRequest())
# create translator
ckan.lib.i18n.set_lang(lang)
# pull out translator and register it
registry.register(translator, fakepylons.translator)
def setup_environment_and_database(env_dir=None, enabled_plugins=''):
global_config = {}
env_dir = env_dir or '/invalid'
app_instance_secret = secret.secret_string()
app_config = {
'plugins': enabled_plugins,
'sqlalchemy.url': 'sqlite://',
'layout_template': 'layout',
'external_template': 'false',
'image_dir': os.path.join(env_dir, 'images'),
'media_dir': os.path.join(env_dir, 'media'),
'beaker.session.secret': app_instance_secret,
'sa_auth.cookie_secret': app_instance_secret,
}
pylons_config = load_environment(global_config, app_config)
paste_registry = Registry()
paste_registry.prepare()
app_globals = pylons_config['pylons.app_globals']
register_instance(paste_registry, 'app_globals', app_globals)
pylons_config.update({
'paste.registry': paste_registry,
'pylons.pylons': pylons,
})
metadata.create_all(bind=DBSession.bind, checkfirst=True)
return pylons_config
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'supplier_id',
model.PackageExtra.package_id.in_(
model.Session.query(model.Package.id).filter(
model.Package.name.like('%-fork-%'))),
):
model.Session.delete(package_extra)
model.repo.commit_and_remove()
return 0
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'supplier_id',
model.PackageExtra.package_id.in_(
model.Session.query(model.Package.id).filter(model.Package.name.like('%-fork-%'))),
):
model.Session.delete(package_extra)
model.repo.commit_and_remove()
return 0
class CkanCommand(paste.script.command.Command):
parser = paste.script.command.Command.standard_parser(verbose=True)
parser.add_option('-c', '--config', dest='config',
default='development.ini', help='Config file to use.')
parser.add_option('-f', '--file',
action='store',
dest='file_path',
help="File to dump results to (if needed)")
default_verbosity = 1
group_name = 'ckan'
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _setup_app(self):
cmd = paste.script.appinstall.SetupCommand('setup-app')
cmd.run([self.filename])
def _register_mock_translator():
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
def create_fake_env():
registry = Registry()
registry.prepare()
registry.register(session, SessionObject({}))
registry.register(request, Request.blank('/bootstrap'))
request.identity = {}
return session, request
def register_translator():
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
log.info('Translator object was registered for the background job')
def register_translator():
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
log.info('Translator object was registered for the background job')
def setup_class(cls):
super(FunctionalTestBase, cls).setup_class()
core_helpers.reset_db()
# Fix ckanext-hierarchy "c"
c = pylons.util.AttribSafeContextObj()
registry = Registry()
registry.prepare()
registry.register(pylons.c, c)
def setup_class(cls):
# Hack because the hierarchy extension uses c in some methods
# that are called outside the context of a web request
c = pylons.util.AttribSafeContextObj()
registry = Registry()
registry.prepare()
registry.register(pylons.c, c)
super(AuthTestBase, cls).setup_class()
def _test_restorer(stack, data):
# We need to test the request's specific Registry. Initialize it here so we
# can use it later (RegistryManager will re-use one preexisting in the
# environ)
registry = Registry()
extra_environ={'paste.throw_errors': False,
'paste.registry': registry}
request_id = restorer.get_request_id(extra_environ)
app = TestApp(stack)
res = app.get('/', extra_environ=extra_environ, expect_errors=True)
# Ensure all the StackedObjectProxies are empty after the RegistryUsingApp
# raises an Exception
for stacked, proxied_obj, test_cleanup in data:
only_key = list(proxied_obj.keys())[0]
try:
assert only_key not in stacked
assert False
except TypeError:
# Definitely empty
pass
# Ensure the StackedObjectProxies & Registry 'work' in the simulated
# EvalException context
replace = {'replace': 'dict'}
new = {'new': 'object'}
restorer.restoration_begin(request_id)
try:
for stacked, proxied_obj, test_cleanup in data:
# Ensure our original data magically re-appears in this context
only_key, only_val = list(proxied_obj.items())[0]
assert only_key in stacked and stacked[only_key] == only_val
# Ensure the Registry still works
registry.prepare()
registry.register(stacked, new)
assert 'new' in stacked and stacked['new'] == 'object'
registry.cleanup()
# Back to the original (pre-prepare())
assert only_key in stacked and stacked[only_key] == only_val
registry.replace(stacked, replace)
assert 'replace' in stacked and stacked['replace'] == 'dict'
if test_cleanup:
registry.cleanup()
try:
stacked._current_obj()
assert False
except TypeError:
# Definitely empty
pass
finally:
restorer.restoration_end()
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%etalab2.fr%'),
# ):
# url = resource.url
# if url.startswith('http://ckan.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://ckan-hetic.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://www.etalab2.fr/'):
# resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/')
# else:
# print resource.url
# model.repo.commit_and_remove()
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%www.data.gouv.fr%'),
# ):
# if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'):
# resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/')
# model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://new.data.gouv.fr/%'),
):
resource.url = resource.url.replace('http://new.data.gouv.fr/', 'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('https://new.data.gouv.fr/%'),
):
resource.url = resource.url.replace('https://new.data.gouv.fr/', 'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
return 0
def __init__(self, conf=None):
self.config = load_config(conf)
# package_update needs a translator defined i.e. _()
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
self.app = make_app(self.config.global_conf, **self.config.local_conf)
def _paste_registry(pylons_request):
"Internal helper method to get a new registry."
paste_registry = None
is_request_registered = is_object_registered(pylons_request)
if is_request_registered:
paste_registry = pylons_request.environ.get('paste.registry')
if paste_registry is None:
paste_registry = Registry()
paste_registry.prepare()
if is_request_registered:
pylons_request.environ['paste.registry'] = paste_registry
return paste_registry
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-t', '--territoria-url', help = 'Territoria URL', required = True)
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
kind_code_name_by_kind_code = {}
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage',
):
if package_extra.value == 'Coutry/FR':
kind_code_name = 'Country/FR/FRANCE'
elif package_extra.value == 'InternationalOrganization/EU':
kind_code_name = 'InternationalOrganization/UE/UNION EUROPEENNE'
elif package_extra.value.count('/') == 1:
kind_code_name = kind_code_name_by_kind_code.get(package_extra.value)
if kind_code_name is None:
kind, code = package_extra.value.split('/')
try:
response = urllib2.urlopen(urlparse.urljoin(args.territoria_url,
'/api/v1/territory?{}'.format(urllib.urlencode(dict(
code = code,
kind = kind,
), doseq = True))))
except urllib2.HTTPError, response:
print package_extra.value
raise
response_dict = json.loads(response.read())
main_postal_distribution = response_dict['data']['main_postal_distribution']
kind_code_name_by_kind_code[package_extra.value] = kind_code_name = u'/'.join([kind, code,
main_postal_distribution])
print kind_code_name
else:
continue
package = package_extra.package
if package.private or package.state != 'active':
log.warning(u'Territorial coverage of package {} must be manually corrected'.format(package.name))
continue
package_extra.value = kind_code_name
def _paste_registry(pylons_request):
"Internal helper method to get a new registry."
paste_registry = None
is_request_registered = is_object_registered(pylons_request)
if is_request_registered:
paste_registry = pylons_request.environ.get('paste.registry')
if paste_registry is None:
paste_registry = Registry()
paste_registry.prepare()
if is_request_registered:
pylons_request.environ['paste.registry'] = paste_registry
return paste_registry
def register_translator():
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
global registry
registry = Registry()
registry.prepare()
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def register_translator():
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
global registry
registry = Registry()
registry.prepare()
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def load_config(path):
import paste.deploy
conf = paste.deploy.appconfig('config:' + path)
import ckan, pylons
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
from ckan.lib.cli import MockTranslator
from paste.registry import Registry
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def load_config(path):
import paste.deploy
conf = paste.deploy.appconfig('config:' + path)
import ckan, pylons
ckan.config.environment.load_environment(conf.global_conf,
conf.local_conf)
from ckan.lib.cli import MockTranslator
from paste.registry import Registry
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(pylons.translator, translator_obj)
def command(dry_run=False):
from ckan import model
from ckanext.dgu.lib.resource_formats import match
from running_stats import StatsList
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
if not dry_run:
model.repo.new_revision()
# Add canonised formats to map
for format_ in res_type_map.keys():
res_type_map[canonise(format_)] = res_type_map[format_]
log.info('Tidying resource types')
stats = StatsList()
res_query = model.Session.query(model.Resource)
log.info('Tidying formats. Resources=%i Canonised formats=%i',
res_query.count(), len(set(res_type_map.values())))
for res in res_query:
canonised_fmt = canonise(res.format or '')
if canonised_fmt in res_type_map:
improved_fmt = res_type_map[canonised_fmt]
else:
improved_fmt = tidy(res.format)
match_ = match(improved_fmt)
if match_:
improved_fmt = match_
if (improved_fmt or '') != (res.format or ''):
if not dry_run:
res.format = improved_fmt
stats.add(improved_fmt, res.format)
else:
stats.add('No change', res.format)
if not dry_run:
model.repo.commit_and_remove()
log.info('Stats report: %r', stats.report())
print stats.report()
log.info('Warnings (%i): %r', len(warnings), warnings)
def command(dry_run=False):
from ckan import model
from ckanext.dgu.lib.resource_formats import match
from running_stats import StatsList
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
if not dry_run:
model.repo.new_revision()
# Add canonised formats to map
for format_ in res_type_map.keys():
res_type_map[canonise(format_)] = res_type_map[format_]
log.info('Tidying resource types')
stats = StatsList()
res_query = model.Session.query(model.Resource)
log.info('Tidying formats. Resources=%i Canonised formats=%i',
res_query.count(), len(set(res_type_map.values())))
for res in res_query:
canonised_fmt = canonise(res.format or '')
if canonised_fmt in res_type_map:
improved_fmt = res_type_map[canonised_fmt]
else:
improved_fmt = tidy(res.format)
match_ = match(improved_fmt)
if match_:
improved_fmt = match_
if (improved_fmt or '') != (res.format or ''):
if not dry_run:
res.format = improved_fmt
stats.add(improved_fmt, res.format)
else:
stats.add('No change', res.format)
if not dry_run:
model.repo.commit_and_remove()
log.info('Stats report: %r', stats.report())
print stats.report()
log.info('Warnings (%i): %r', len(warnings), warnings)
def load_config(path):
import paste.deploy
from paste.registry import Registry
from command import MockTranslator
conf = paste.deploy.appconfig('config:' + os.path.abspath(path))
import ckan
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def fake_request(pylons_config,
server_name='mediacore.example',
language='en',
method='GET',
request_uri='/',
post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError(
'You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' %
(server_name, request_uri),
method.upper(),
request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediacore.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry)
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def __init__(self,
worker_id,
heartbeat_interval=60000,
action='update'):
super(UpdateIntentsWorker, self).__init__(worker_id, heartbeat_interval)
self.intents_worker = UserIntentsWorker(intents_extractor)
self.action = action
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
def fake_request(pylons_config, server_name='mediadrop.example', language='en',
method='GET', request_uri='/', post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError('You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri),
method.upper(), request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# TODO: Use ContextObj() for Pylons 0.10
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
mediacore_i18n_path = os.path.join(os.path.dirname(mediacore.__file__), 'i18n')
translator = Translator(language, dict(mediacore=mediacore_i18n_path))
# not sure why but sometimes pylons.translator is not a StackedObjectProxy
# but just a regular Translator.
if not hasattr(pylons.translator, '_push_object'):
pylons.translator = StackedObjectProxy()
paste_registry.replace(pylons.translator, translator)
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def load_config(path):
import paste.deploy
from paste.registry import Registry
from command import MockTranslator
conf = paste.deploy.appconfig('config:' + os.path.abspath(path))
import ckan
ckan.config.environment.load_environment(conf.global_conf,
conf.local_conf)
registry=Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def _register_translator():
"""
Register a translator in this thread.
"""
global registry
try:
registry
except NameError:
registry = Registry()
registry.prepare()
global translator_obj
try:
translator_obj
except NameError:
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def _register_translator():
"""
Register a translator in this thread.
"""
global registry
try:
registry
except NameError:
registry = Registry()
registry.prepare()
global translator_obj
try:
translator_obj
except NameError:
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def register_translator():
# https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py
# If not set (in cli access), patch the a translator with a mock, so the
# _() functions in logic layer don't cause failure.
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
if 'registery' not in globals():
global registry
registry = Registry()
registry.prepare()
if 'translator_obj' not in globals():
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def register_translator():
# https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py
# If not set (in cli access), patch the a translator with a mock, so the
# _() functions in logic layer don't cause failure.
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
if 'registery' not in globals():
global registry
registry = Registry()
registry.prepare()
if 'translator_obj' not in globals():
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def load_config(config, load_site_user=True):
conf = _get_config(config)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf)
# Set this internal test request context with the configured environment so
# it can be used when calling url_for from the CLI.
global _cli_test_request_context
app = make_app(conf)
flask_app = app.apps['flask_app']._wsgi_app
_cli_test_request_context = flask_app.test_request_context()
registry = Registry()
registry.prepare()
import pylons
registry.register(pylons.translator, MockTranslator())
site_user = None
if model.user_table.exists() and load_site_user:
# If the DB has already been initialized, create and register
# a pylons context object, and add the site user to it, so the
# auth works as in a normal web request
c = pylons.util.AttribSafeContextObj()
registry.register(pylons.c, c)
site_user = logic.get_action('get_site_user')({
'ignore_auth': True
}, {})
pylons.c.user = site_user['name']
pylons.c.userobj = model.User.get(site_user['name'])
## give routes enough information to run url_for
parsed = urlparse(conf.get('ckan.site_url', 'http://0.0.0.0'))
request_config = routes.request_config()
request_config.host = parsed.netloc + parsed.path
request_config.protocol = parsed.scheme
return site_user
def fake_request(pylons_config, server_name='mediacore.example', language='en',
method='GET', request_uri='/', post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError('You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri),
method.upper(), request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediacore.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry,
locale_dirs=pylons_config.get('locale_dirs'))
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def command(config_ini, nodepublisher_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config,'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
model.repo.new_revision()
log.info('Reading %s', nodepublisher_csv)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
nid, title = row
publishers[ int(nid) ] = munge_title_to_name(title)
# Mappings where we are getting rid of duplicate publishers
publishers[16268] = publishers[11408] # UKSA -> ONS
publishers[11606] = publishers[11408] # ONS
publishers[20054] = publishers[16248] # Met Office
publishers[33036] = publishers[15255] # Windsor & Maidenhead
publishers[32619] = publishers[33245] # Monmouthshire
publishers[12662] = publishers[11567] # NHS
update_datasets()
generate_harvest_publishers()
log.info('Warnings: %r', warnings)
def command(config_ini, nodepublisher_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config,'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
model.repo.new_revision()
log.info('Reading %s', nodepublisher_csv)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
nid, title = row
publishers[ int(nid) ] = munge_title_to_name(title)
# Mappings where we are getting rid of duplicate publishers
publishers[16268] = publishers[11408] # UKSA -> ONS
publishers[11606] = publishers[11408] # ONS
publishers[20054] = publishers[16248] # Met Office
publishers[33036] = publishers[15255] # Windsor & Maidenhead
publishers[32619] = publishers[33245] # Monmouthshire
publishers[12662] = publishers[11567] # NHS
update_datasets()
generate_harvest_publishers()
log.info('Warnings: %r', warnings)
def init_mediadrop(config_filename, here_dir=None, disable_logging=False):
if not os.path.exists(config_filename):
raise IOError('Config file %r does not exist.' % config_filename)
if here_dir is None:
here_dir = os.getcwd()
if not disable_logging:
logging_config.fileConfig(config_filename)
conf = appconfig('config:%s' % config_filename, relative_to=here_dir)
pylons_config = load_environment(conf.global_conf, conf.local_conf)
paste_registry = Registry()
paste_registry.prepare()
pylons_config['paste.registry'] = paste_registry
app_globals = pylons_config['pylons.app_globals']
register_instance(paste_registry, 'app_globals', app_globals)
register_instance(paste_registry, 'config', pylons_config)
fake_request(pylons_config, registry=paste_registry)
setup_global_translator(registry=paste_registry)
def fake_request(
pylons_config, server_name="mediadrop.example", language="en", method="GET", request_uri="/", post_vars=None
):
app_globals = pylons_config["pylons.app_globals"]
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != "POST":
raise ValueError("You must not specify post_vars for request method %r" % method)
wsgi_environ = create_wsgi_environ(
"http://%s%s" % (server_name, request_uri), method.upper(), request_body=post_vars
)
request = Request(wsgi_environ, charset="utf-8")
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type="application/xml", charset="utf-8")
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config["routes.map"], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediadrop.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry, locale_dirs=pylons_config.get("locale_dirs"))
wsgi_environ.update({"pylons.pylons": pylons, "paste.registry": paste_registry})
return request
def __init__(self):
super(MockedJobController, self).__init__()
# Register into paste the fake request and response objects
registry = Registry()
registry.prepare()
registry.register(request, MockRequest())
registry.register(response, MockResponse())
# This is expected to be set by Pylons, so we set it here
self._py_object = type("", (), {
"__init__": (lambda s, **kwargs: s.__dict__.update(kwargs))
})(request=request, response=response)
# Inject fake proxy, so the submission works
delegated = Credential(
dlg_id='12345', dn='/DN=1234', proxy='',
termination_time=datetime.utcnow() + timedelta(hours=5)
)
Session.merge(delegated)
Session.commit()
def __init__(self):
super(MockedJobController, self).__init__()
# Register into paste the fake request and response objects
registry = Registry()
registry.prepare()
registry.register(request, MockRequest())
registry.register(response, MockResponse())
# This is expected to be set by Pylons, so we set it here
self._py_object = type(
"", (),
{"__init__": (lambda s, **kwargs: s.__dict__.update(kwargs))})(
request=request, response=response)
# Inject fake proxy, so the submission works
delegated = Credential(dlg_id='12345',
dn='/DN=1234',
proxy='',
termination_time=datetime.utcnow() +
timedelta(hours=5))
Session.merge(delegated)
Session.commit()
class PylonsTestCase(object):
"""A basic test case which allows access to pylons.c and pylons.request.
"""
def setup(self):
self.registry=Registry()
self.registry.prepare()
self.context_obj=ContextObj()
self.registry.register(pylons.c, self.context_obj)
pylons.c.errors = None
self.request_obj=Request(dict(HTTP_HOST="nohost"))
self.registry.register(pylons.request, self.request_obj)
self.translator_obj=MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
self.buffet = pylons.templating.Buffet('genshi', template_root='ckan.templates')
self.registry.register(pylons.buffet, self.buffet)
self.registry.register(pylons.response, Response())
self.registry.register(pylons.url, None)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage_granularity',
model.PackageExtra.value == 'france',
):
package = package_extra.package
if package.private or package.state != 'active':
log.warning(
u'Territorial coverage granularity of package {} must be manually corrected'
.format(package.name))
continue
package_extra.value = 'pays'
model.repo.commit_and_remove()
return 0
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for organization_name, is_service_public in sorted(is_public_service_by_organization_name.iteritems()):
if not is_service_public:
continue
organization = model.Session.query(model.Group).filter(
model.Group.is_organization == True,
model.Group.name == organization_name,
).first()
if organization is None:
log.warning(u'Unknown organization: {}'.format(organization_name))
continue
if organization.certified_public_service is None:
log.info(u'Certifying "{}" as public service'.format(organization_name))
certified_public_service = etalab_model.CertifiedPublicService()
certified_public_service.organization_id = organization.id
model.Session.add(certified_public_service)
model.repo.commit_and_remove()
return 0
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for organization_name, is_service_public in sorted(is_public_service_by_organization_name.iteritems()):
if not is_service_public:
continue
organization = model.Session.query(model.Group).filter(
model.Group.is_organization == True,
model.Group.name == organization_name,
).first()
if organization is None:
log.warning(u'Unknown organization: {}'.format(organization_name))
continue
if organization.certified_public_service is None:
log.info(u'Certifying "{}" as public service'.format(organization_name))
certified_public_service = etalab_model.CertifiedPublicService()
certified_public_service.organization_id = organization.id
model.Session.add(certified_public_service)
model.repo.commit_and_remove()
return 0
def load_translations(lang):
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from pylons import request
registry = Registry()
registry.prepare()
class FakePylons:
translator = None
fakepylons = FakePylons()
class FakeRequest:
# Stores details of the translator
environ = {'pylons.pylons': fakepylons}
registry.register(request, FakeRequest())
# create translator
i18n.set_lang(lang)
# pull out translator and register it
registry.register(translator, fakepylons.translator)
def command(input_csv, config_ini, commit=False):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
from ckan import model
from ckan.logic import get_action
from ckan.lib.cli import MockTranslator
model.init_model(engine)
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
ctx = {
'model': model, 'session': model.Session,
'user': get_action('get_site_user')({'model': model,'ignore_auth': True}, {})['name']
}
if commit:
rev = model.repo.new_revision()
packages_to_check = set()
reader = csv.reader(open(input_csv, 'r'))
for row in reader:
# For each URL in the csv, get the list of resources referencing
# that URL
resources = model.Session.query(model.Resource)\
.filter(model.Resource.state=='active')\
.filter(model.Resource.url==row[0]).all()
for resource in resources:
# For each resource, add the package to the list
packages_to_check.add(resource.get_package_id())
# Delete the resource
resource.state = 'deleted'
model.Session.add(resource)
if commit:
model.Session.commit()
print "Deleted resource: {0}".format(resource.id)
stats.increment("Deleted resource")
for pid in packages_to_check:
# for each package we need to check, see if it has any
# resources left, it not, delete it.
pkg = model.Package.get(pid)
if len(pkg.resources) == 0:
pkg.state = 'deleted'
model.Session.add(pkg)
if commit:
model.Session.commit()
stats.increment('Deleted packages')
print "Deleted package: {0}".format(pkg.name)
if commit:
model.repo.commit_and_remove()
else:
print ""
print '*' * 60
print "DON'T PANIC, this was a dry run, nothing was committed"
print '*' * 60
print ''
print '*' * 60, 'Deletion Report'
print stats.report(order_by_title=True)
def migrate_v3_create_datasets(source_ids=None):
import pylons
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
sources = []
if not source_ids:
sources = model.Session.query(HarvestSource).all()
else:
sources = model.Session.query(HarvestSource) \
.filter(HarvestSource.id.in_(source_ids)) \
.all()
if not sources:
log.debug('No harvest sources to migrate')
return
site_user_name = logic.get_action('get_site_user')({'model': model, 'ignore_auth': True},{})['name']
context = {'model': model,
'session': model.Session,
'user': site_user_name, # TODO: auth of existing sources?
'return_id_only': True,
'extras_as_string': True,
}
def gen_new_name(title):
name = munge_title_to_name(title).replace('_', '-')
while '--' in name:
name = name.replace('--', '-')
pkg_obj = Session.query(Package).filter(Package.name == name).first()
if pkg_obj:
return name + str(uuid.uuid4())[:5]
else:
return name
for source in sources:
if 'id' in context:
del context['id']
if 'package' in context:
del context['package']
# Check if package already exists
try:
logic.get_action('package_show')(context, {'id': source.id})
continue
except logic.NotFound:
pass
package_dict = {
'id': source.id,
'name': gen_new_name(source.title) if source.title else source.id,
'title': source.title if source.title else source.url,
'notes': source.description,
'url': source.url,
'type': 'harvest',
'source_type': source.type,
'config': source.config,
'frequency': source.frequency,
}
context['message'] = 'Created package for harvest source {0}'.format(source.id)
try:
new_package_id = logic.get_action('package_create')(context, package_dict)
if new_package_id != source.id or not context['return_id_only']:
# this check only makes sense if we are sure we are returning
# the package id not the package object
raise PackageIdHarvestSourceIdMismatch
log.info('Created new package for source {0} ({1})'.format(source.id, source.url))
except logic.ValidationError,e:
log.error('Validation Error: %s' % str(e.error_summary))
def migrate_v3_create_datasets(source_ids=None):
import pylons
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
sources = []
if not source_ids:
sources = model.Session.query(HarvestSource).all()
else:
sources = model.Session.query(HarvestSource).filter(HarvestSource.id.in_(source_ids)).all()
if not sources:
log.debug("No harvest sources to migrate")
return
site_user_name = logic.get_action("get_site_user")({"model": model, "ignore_auth": True}, {})["name"]
context = {
"model": model,
"session": model.Session,
"user": site_user_name, # TODO: auth of existing sources?
"return_id_only": True,
"extras_as_string": True,
}
def gen_new_name(title):
name = munge_title_to_name(title).replace("_", "-")
while "--" in name:
name = name.replace("--", "-")
pkg_obj = Session.query(Package).filter(Package.name == name).first()
if pkg_obj:
return name + str(uuid.uuid4())[:5]
else:
return name
for source in sources:
if "id" in context:
del context["id"]
if "package" in context:
del context["package"]
# Check if package already exists
try:
logic.get_action("package_show")(context, {"id": source.id})
continue
except logic.NotFound:
pass
package_dict = {
"id": source.id,
"name": gen_new_name(source.title) if source.title else source.id,
"title": source.title if source.title else source.url,
"notes": source.description,
"url": source.url,
"type": "harvest",
"source_type": source.type,
"config": source.config,
"frequency": source.frequency,
}
context["message"] = "Created package for harvest source {0}".format(source.id)
try:
new_package_id = logic.get_action("package_create")(context, package_dict)
if new_package_id != source.id or not context["return_id_only"]:
# this check only makes sense if we are sure we are returning
# the package id not the package object
raise PackageIdHarvestSourceIdMismatch
log.info("Created new package for source {0} ({1})".format(source.id, source.url))
except logic.ValidationError, e:
log.error("Validation Error: %s" % str(e.error_summary))
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('user', help = 'name of email of user')
parser.add_argument('-d', '--dry-run', action = 'store_true',
help = "simulate harvesting, don't update CKAN repository")
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
user_name = args.user.lower().replace('.', '-dot-').replace('@', '-at-')
user = model.Session.query(model.User).filter(model.User.name == user_name).one()
assert user is not None, 'Unknown user: {}'.format(user_name)
for membership_request in model.Session.query(youckan_model.MembershipRequest).filter(
youckan_model.MembershipRequest.user_id == user.id,
):
log.warning(u'Deleting membership request: {}'.format(membership_request))
if not args.dry_run:
model.Session.delete(membership_request)
model.Session.commit()
for related in model.Session.query(model.Related).filter(model.Related.owner_id == user.id):
log.warning(u'Deleting related: {}'.format(related))
if not args.dry_run:
model.Session.delete(related)
model.Session.commit()
if not args.dry_run:
model.repo.new_revision()
for user_object_role in model.Session.query(model.UserObjectRole).filter(
model.UserObjectRole.user_id == user.id,
model.UserObjectRole.role == 'admin',
):
if user_object_role.context == 'Group':
group = user_object_role.group
log.warning(u'Deleting group or organization: {}'.format(group))
if not args.dry_run:
model.Session.query(etalab_model.CertifiedPublicService).filter(
etalab_model.CertifiedPublicService.organization_id == group.id,
).delete()
model.Session.delete(group)
else:
assert user_object_role.context == 'Package', 'Unexpected context for role: {}'.format(
user_object_role.context)
package = user_object_role.package
# Delete resource_revision before purging package, to avoid IntegrityError: update or delete on table
# "resource_group" violates foreign key constraint "resource_revision_resource_group_id_fkey" on table
# "resource_revision".
for resource_group in model.Session.query(model.ResourceGroup).filter(
model.ResourceGroup.package_id == package.id,
):
for resource_revision in model.Session.query(model.ResourceRevision).filter(
model.ResourceRevision.resource_group_id == resource_group.id,
):
if not args.dry_run:
log.warning(u'Deleting resource_revision')
model.Session.delete(resource_revision)
# Delete package_relationship_revision before purging package, to avoid IntegrityError: update or
# delete on table "package" violates foreign key constraint
# "package_relationship_revision_subject_package_id_fkey" on table "package_relationship_revision".
for package_relationship_revision in model.Session.query(model.PackageRelationshipRevision).filter(
model.PackageRelationshipRevision.subject_package_id == package.id,
):
if not args.dry_run:
log.warning(u'Deleting package_relationship_revision')
model.Session.delete(package_relationship_revision)
for package_relationship_revision in model.Session.query(model.PackageRelationshipRevision).filter(
model.PackageRelationshipRevision.object_package_id == package.id,
):
if not args.dry_run:
log.warning(u'Deleting package_relationship_revision')
model.Session.delete(package_relationship_revision)
log.warning(u'Deleting package: {}'.format(package))
if not args.dry_run:
model.Session.delete(package)
if not args.dry_run:
model.repo.commit_and_remove()
if not args.dry_run:
log.warning(u'Deleting user: {}'.format(user))
model.Session.delete(user)
model.Session.commit()
return 0
