def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage_granularity',
model.PackageExtra.value == 'france',
):
package = package_extra.package
if package.private or package.state != 'active':
log.warning(u'Territorial coverage granularity of package {} must be manually corrected'.format(
package.name))
continue
package_extra.value = 'pays'
model.repo.commit_and_remove()
return 0
def setUp(self):
super(AutoCompleterTest, self).setUp()
# Création d'un environnement WSGI pour les tests.
# Par défaut, l'utilisateur n'est pas identifé
# et n'appartient donc à aucun groupe.
environ = {'repoze.what.credentials': {
'groups': [],
}
}
request = Request(environ)
config = VigiloAppConfig('TurboGears')
config.init_managers_predicate()
registry = Registry()
registry.prepare()
registry.register(tg.request, request)
registry.register(tg.config, config)
self.ctrl = AutoCompleteController()
# Par défaut, l'utilisateur n'est pas identifé.
tg.request.identity = {'repoze.who.userid': None}
self.accounts = (
(u'manager', u'managers'), # Accès à tout (manager)
(u'nobody', u'nobody'), # Accès à rien.
(u'direct', u'direct'), # Permissions directe sur l'objet.
(u'indirect', u'indirect'), # Permissions indirectes sur l'objet.
)
# Création des comptes utilisateurs.
for username, group in self.accounts:
functions.add_user(username, u'%s@test' % username,
u'', u'', group)
# Création de l'arborescence des objets.
self.create_deps()
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'supplier_id',
model.PackageExtra.package_id.in_(
model.Session.query(model.Package.id).filter(model.Package.name.like('%-fork-%'))),
):
model.Session.delete(package_extra)
model.repo.commit_and_remove()
return 0
class CkanCommand(paste.script.command.Command):
parser = paste.script.command.Command.standard_parser(verbose=True)
parser.add_option('-c', '--config', dest='config',
default='development.ini', help='Config file to use.')
parser.add_option('-f', '--file',
action='store',
dest='file_path',
help="File to dump results to (if needed)")
default_verbosity = 1
group_name = 'ckan'
def _load_config(self):
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' % self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry=Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _setup_app(self):
cmd = paste.script.appinstall.SetupCommand('setup-app')
cmd.run([self.filename])
def register_translator():
from paste.registry import Registry
import pylons
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
log.info('Translator object was registered for the background job')
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%etalab2.fr%'),
# ):
# url = resource.url
# if url.startswith('http://ckan.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://ckan-hetic.etalab2.fr/'):
# resource.url = resource.url.replace('http://ckan-hetic.etalab2.fr/', 'http://www.data.gouv.fr/fr/')
# elif url.startswith('http://www.etalab2.fr/'):
# resource.url = resource.url.replace('http://www.etalab2.fr/', 'http://www.data.gouv.fr/')
# else:
# print resource.url
# model.repo.commit_and_remove()
# model.repo.new_revision()
# for resource in model.Session.query(model.Resource).filter(
# model.Resource.url.like('%www.data.gouv.fr%'),
# ):
# if resource.url.startswith('http://www.data.gouv.fr/') and not resource.url.startswith('http://www.data.gouv.fr/var'):
# resource.url = resource.url.replace('http://www.data.gouv.fr/', 'http://new.data.gouv.fr/')
# model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://new.data.gouv.fr/%'),
):
resource.url = resource.url.replace('http://new.data.gouv.fr/', 'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
model.repo.new_revision()
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('https://new.data.gouv.fr/%'),
):
resource.url = resource.url.replace('https://new.data.gouv.fr/', 'https://www.data.gouv.fr/')
model.repo.commit_and_remove()
return 0
def _test_restorer(stack, data):
# We need to test the request's specific Registry. Initialize it here so we
# can use it later (RegistryManager will re-use one preexisting in the
# environ)
registry = Registry()
extra_environ={'paste.throw_errors': False,
'paste.registry': registry}
request_id = restorer.get_request_id(extra_environ)
app = TestApp(stack)
res = app.get('/', extra_environ=extra_environ, expect_errors=True)
# Ensure all the StackedObjectProxies are empty after the RegistryUsingApp
# raises an Exception
for stacked, proxied_obj, test_cleanup in data:
only_key = list(proxied_obj.keys())[0]
try:
assert only_key not in stacked
assert False
except TypeError:
# Definitely empty
pass
# Ensure the StackedObjectProxies & Registry 'work' in the simulated
# EvalException context
replace = {'replace': 'dict'}
new = {'new': 'object'}
restorer.restoration_begin(request_id)
try:
for stacked, proxied_obj, test_cleanup in data:
# Ensure our original data magically re-appears in this context
only_key, only_val = list(proxied_obj.items())[0]
assert only_key in stacked and stacked[only_key] == only_val
# Ensure the Registry still works
registry.prepare()
registry.register(stacked, new)
assert 'new' in stacked and stacked['new'] == 'object'
registry.cleanup()
# Back to the original (pre-prepare())
assert only_key in stacked and stacked[only_key] == only_val
registry.replace(stacked, replace)
assert 'replace' in stacked and stacked['replace'] == 'dict'
if test_cleanup:
registry.cleanup()
try:
stacked._current_obj()
assert False
except TypeError:
# Definitely empty
pass
finally:
restorer.restoration_end()
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-t', '--territoria-url', help = 'Territoria URL', required = True)
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
kind_code_name_by_kind_code = {}
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage',
):
if package_extra.value == 'Coutry/FR':
kind_code_name = 'Country/FR/FRANCE'
elif package_extra.value == 'InternationalOrganization/EU':
kind_code_name = 'InternationalOrganization/UE/UNION EUROPEENNE'
elif package_extra.value.count('/') == 1:
kind_code_name = kind_code_name_by_kind_code.get(package_extra.value)
if kind_code_name is None:
kind, code = package_extra.value.split('/')
try:
response = urllib2.urlopen(urlparse.urljoin(args.territoria_url,
'/api/v1/territory?{}'.format(urllib.urlencode(dict(
code = code,
kind = kind,
), doseq = True))))
except urllib2.HTTPError, response:
print package_extra.value
raise
response_dict = json.loads(response.read())
main_postal_distribution = response_dict['data']['main_postal_distribution']
kind_code_name_by_kind_code[package_extra.value] = kind_code_name = u'/'.join([kind, code,
main_postal_distribution])
print kind_code_name
else:
continue
package = package_extra.package
if package.private or package.state != 'active':
log.warning(u'Territorial coverage of package {} must be manually corrected'.format(package.name))
continue
package_extra.value = kind_code_name
def load_config(path):
import paste.deploy
conf = paste.deploy.appconfig('config:' + path)
import ckan, pylons
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
from ckan.lib.cli import MockTranslator
from paste.registry import Registry
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def register_translator():
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
global registry
registry = Registry()
registry.prepare()
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def register_translator():
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
global registry
registry=Registry()
registry.prepare()
global translator_obj
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
def load_config(path):
import paste.deploy
conf = paste.deploy.appconfig('config:' + path)
import ckan, pylons
ckan.config.environment.load_environment(conf.global_conf,
conf.local_conf)
from ckan.lib.cli import MockTranslator
from paste.registry import Registry
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(pylons.translator, translator_obj)
def command(dry_run=False):
from ckan import model
from ckanext.dgu.lib.resource_formats import match
from running_stats import StatsList
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
if not dry_run:
model.repo.new_revision()
# Add canonised formats to map
for format_ in res_type_map.keys():
res_type_map[canonise(format_)] = res_type_map[format_]
log.info('Tidying resource types')
stats = StatsList()
res_query = model.Session.query(model.Resource)
log.info('Tidying formats. Resources=%i Canonised formats=%i',
res_query.count(), len(set(res_type_map.values())))
for res in res_query:
canonised_fmt = canonise(res.format or '')
if canonised_fmt in res_type_map:
improved_fmt = res_type_map[canonised_fmt]
else:
improved_fmt = tidy(res.format)
match_ = match(improved_fmt)
if match_:
improved_fmt = match_
if (improved_fmt or '') != (res.format or ''):
if not dry_run:
res.format = improved_fmt
stats.add(improved_fmt, res.format)
else:
stats.add('No change', res.format)
if not dry_run:
model.repo.commit_and_remove()
log.info('Stats report: %r', stats.report())
print stats.report()
log.info('Warnings (%i): %r', len(warnings), warnings)
def command(dry_run=False):
from ckan import model
from ckanext.dgu.lib.resource_formats import match
from running_stats import StatsList
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
if not dry_run:
model.repo.new_revision()
# Add canonised formats to map
for format_ in res_type_map.keys():
res_type_map[canonise(format_)] = res_type_map[format_]
log.info('Tidying resource types')
stats = StatsList()
res_query = model.Session.query(model.Resource)
log.info('Tidying formats. Resources=%i Canonised formats=%i',
res_query.count(), len(set(res_type_map.values())))
for res in res_query:
canonised_fmt = canonise(res.format or '')
if canonised_fmt in res_type_map:
improved_fmt = res_type_map[canonised_fmt]
else:
improved_fmt = tidy(res.format)
match_ = match(improved_fmt)
if match_:
improved_fmt = match_
if (improved_fmt or '') != (res.format or ''):
if not dry_run:
res.format = improved_fmt
stats.add(improved_fmt, res.format)
else:
stats.add('No change', res.format)
if not dry_run:
model.repo.commit_and_remove()
log.info('Stats report: %r', stats.report())
print stats.report()
log.info('Warnings (%i): %r', len(warnings), warnings)
def __init__(self,
worker_id,
heartbeat_interval=60000,
action='update'):
super(UpdateIntentsWorker, self).__init__(worker_id, heartbeat_interval)
self.intents_worker = UserIntentsWorker(intents_extractor)
self.action = action
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
def load_config(path):
import paste.deploy
from paste.registry import Registry
from command import MockTranslator
conf = paste.deploy.appconfig('config:' + os.path.abspath(path))
import ckan
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def fake_request(pylons_config,
server_name='mediacore.example',
language='en',
method='GET',
request_uri='/',
post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError(
'You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' %
(server_name, request_uri),
method.upper(),
request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediacore.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry)
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def fake_request(pylons_config, server_name='mediadrop.example', language='en',
method='GET', request_uri='/', post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError('You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri),
method.upper(), request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# TODO: Use ContextObj() for Pylons 0.10
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
mediacore_i18n_path = os.path.join(os.path.dirname(mediacore.__file__), 'i18n')
translator = Translator(language, dict(mediacore=mediacore_i18n_path))
# not sure why but sometimes pylons.translator is not a StackedObjectProxy
# but just a regular Translator.
if not hasattr(pylons.translator, '_push_object'):
pylons.translator = StackedObjectProxy()
paste_registry.replace(pylons.translator, translator)
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def load_config(path):
import paste.deploy
from paste.registry import Registry
from command import MockTranslator
conf = paste.deploy.appconfig('config:' + os.path.abspath(path))
import ckan
ckan.config.environment.load_environment(conf.global_conf,
conf.local_conf)
registry=Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def _register_translator():
"""
Register a translator in this thread.
"""
global registry
try:
registry
except NameError:
registry = Registry()
registry.prepare()
global translator_obj
try:
translator_obj
except NameError:
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def register_translator():
# https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py
# If not set (in cli access), patch the a translator with a mock, so the
# _() functions in logic layer don't cause failure.
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
if 'registery' not in globals():
global registry
registry = Registry()
registry.prepare()
if 'translator_obj' not in globals():
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def register_translator():
# https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py
# If not set (in cli access), patch the a translator with a mock, so the
# _() functions in logic layer don't cause failure.
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
if 'registery' not in globals():
global registry
registry = Registry()
registry.prepare()
if 'translator_obj' not in globals():
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def _register_translator():
"""
Register a translator in this thread.
"""
global registry
try:
registry
except NameError:
registry = Registry()
registry.prepare()
global translator_obj
try:
translator_obj
except NameError:
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def fake_request(pylons_config, server_name='mediacore.example', language='en',
method='GET', request_uri='/', post_vars=None):
app_globals = pylons_config['pylons.app_globals']
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != 'POST':
raise ValueError('You must not specify post_vars for request method %r' % method)
wsgi_environ = create_wsgi_environ('http://%s%s' % (server_name, request_uri),
method.upper(), request_body=post_vars)
request = Request(wsgi_environ, charset='utf-8')
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type='application/xml', charset='utf-8')
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config['routes.map'], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediacore.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry,
locale_dirs=pylons_config.get('locale_dirs'))
wsgi_environ.update({
'pylons.pylons': pylons,
'paste.registry': paste_registry,
})
return request
def load_config(config, load_site_user=True):
conf = _get_config(config)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf)
# Set this internal test request context with the configured environment so
# it can be used when calling url_for from the CLI.
global _cli_test_request_context
app = make_app(conf)
flask_app = app.apps['flask_app']._wsgi_app
_cli_test_request_context = flask_app.test_request_context()
registry = Registry()
registry.prepare()
import pylons
registry.register(pylons.translator, MockTranslator())
site_user = None
if model.user_table.exists() and load_site_user:
# If the DB has already been initialized, create and register
# a pylons context object, and add the site user to it, so the
# auth works as in a normal web request
c = pylons.util.AttribSafeContextObj()
registry.register(pylons.c, c)
site_user = logic.get_action('get_site_user')({
'ignore_auth': True
}, {})
pylons.c.user = site_user['name']
pylons.c.userobj = model.User.get(site_user['name'])
## give routes enough information to run url_for
parsed = urlparse(conf.get('ckan.site_url', 'http://0.0.0.0'))
request_config = routes.request_config()
request_config.host = parsed.netloc + parsed.path
request_config.protocol = parsed.scheme
return site_user
def command(config_ini, nodepublisher_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config,'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
model.repo.new_revision()
log.info('Reading %s', nodepublisher_csv)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
nid, title = row
publishers[ int(nid) ] = munge_title_to_name(title)
# Mappings where we are getting rid of duplicate publishers
publishers[16268] = publishers[11408] # UKSA -> ONS
publishers[11606] = publishers[11408] # ONS
publishers[20054] = publishers[16248] # Met Office
publishers[33036] = publishers[15255] # Windsor & Maidenhead
publishers[32619] = publishers[33245] # Monmouthshire
publishers[12662] = publishers[11567] # NHS
update_datasets()
generate_harvest_publishers()
log.info('Warnings: %r', warnings)
def command(config_ini, nodepublisher_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config,'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
model.repo.new_revision()
log.info('Reading %s', nodepublisher_csv)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
nid, title = row
publishers[ int(nid) ] = munge_title_to_name(title)
# Mappings where we are getting rid of duplicate publishers
publishers[16268] = publishers[11408] # UKSA -> ONS
publishers[11606] = publishers[11408] # ONS
publishers[20054] = publishers[16248] # Met Office
publishers[33036] = publishers[15255] # Windsor & Maidenhead
publishers[32619] = publishers[33245] # Monmouthshire
publishers[12662] = publishers[11567] # NHS
update_datasets()
generate_harvest_publishers()
log.info('Warnings: %r', warnings)
def fake_request(
pylons_config, server_name="mediadrop.example", language="en", method="GET", request_uri="/", post_vars=None
):
app_globals = pylons_config["pylons.app_globals"]
pylons.app_globals._push_object(app_globals)
if post_vars and method.upper() != "POST":
raise ValueError("You must not specify post_vars for request method %r" % method)
wsgi_environ = create_wsgi_environ(
"http://%s%s" % (server_name, request_uri), method.upper(), request_body=post_vars
)
request = Request(wsgi_environ, charset="utf-8")
request.language = language
request.settings = app_globals.settings
pylons.request._push_object(request)
response = Response(content_type="application/xml", charset="utf-8")
pylons.response._push_object(response)
session = SessionObject(wsgi_environ)
pylons.session._push_object(session)
routes_url = URLGenerator(pylons_config["routes.map"], wsgi_environ)
pylons.url._push_object(routes_url)
# Use ContextObj() when we get rid of 'pylons.strict_tmpl_context=False' in
# mediadrop.lib.environment
tmpl_context = AttribSafeContextObj()
tmpl_context.paginators = Bunch()
pylons.tmpl_context._push_object(tmpl_context)
# some parts of Pylons (e.g. Pylons.controllers.core.WSGIController)
# use the '.c' alias instead.
pylons.c = pylons.tmpl_context
paste_registry = Registry()
paste_registry.prepare()
engines = create_tw_engine_manager(app_globals)
host_framework = PylonsHostFramework(engines=engines)
paste_registry.register(tw.framework, host_framework)
setup_translator(language=language, registry=paste_registry, locale_dirs=pylons_config.get("locale_dirs"))
wsgi_environ.update({"pylons.pylons": pylons, "paste.registry": paste_registry})
return request
def __init__(self):
super(MockedJobController, self).__init__()
# Register into paste the fake request and response objects
registry = Registry()
registry.prepare()
registry.register(request, MockRequest())
registry.register(response, MockResponse())
# This is expected to be set by Pylons, so we set it here
self._py_object = type("", (), {
"__init__": (lambda s, **kwargs: s.__dict__.update(kwargs))
})(request=request, response=response)
# Inject fake proxy, so the submission works
delegated = Credential(
dlg_id='12345', dn='/DN=1234', proxy='',
termination_time=datetime.utcnow() + timedelta(hours=5)
)
Session.merge(delegated)
Session.commit()
class CkanCommand(paste.script.command.Command):
parser = paste.script.command.Command.standard_parser(verbose=True)
parser.add_option('-c',
'--config',
dest='config',
default='development.ini',
help='Config file to use.')
parser.add_option('-f',
'--file',
action='store',
dest='file_path',
help="File to dump results to (if needed)")
default_verbosity = 1
group_name = 'ckan'
def _load_config(self):
from paste.deploy import appconfig
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' %
self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
assert 'ckan' not in dir() # otherwise loggers would be disabled
# We have now loaded the config. Now we can import ckan for the
# first time.
from ckan.config.environment import load_environment
load_environment(conf.global_conf, conf.local_conf)
self.registry = Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _setup_app(self):
cmd = paste.script.appinstall.SetupCommand('setup-app')
cmd.run([self.filename])
class CkanCommand(paste.script.command.Command):
parser = paste.script.command.Command.standard_parser(verbose=True)
parser.add_option('-c',
'--config',
dest='config',
default='development.ini',
help='Config file to use.')
parser.add_option('-f',
'--file',
action='store',
dest='file_path',
help="File to dump results to (if needed)")
default_verbosity = 1
group_name = 'ckan'
def _load_config(self):
# Avoids vdm logging warning
logging.basicConfig(level=logging.ERROR)
from paste.deploy import appconfig
from ckan.config.environment import load_environment
if not self.options.config:
msg = 'No config file supplied'
raise self.BadCommand(msg)
self.filename = os.path.abspath(self.options.config)
if not os.path.exists(self.filename):
raise AssertionError('Config filename %r does not exist.' %
self.filename)
fileConfig(self.filename)
conf = appconfig('config:' + self.filename)
load_environment(conf.global_conf, conf.local_conf)
self.registry = Registry()
self.registry.prepare()
import pylons
self.translator_obj = MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
def _setup_app(self):
cmd = paste.script.appinstall.SetupCommand('setup-app')
cmd.run([self.filename])
def __init__(self):
super(MockedJobController, self).__init__()
# Register into paste the fake request and response objects
registry = Registry()
registry.prepare()
registry.register(request, MockRequest())
registry.register(response, MockResponse())
# This is expected to be set by Pylons, so we set it here
self._py_object = type(
"", (),
{"__init__": (lambda s, **kwargs: s.__dict__.update(kwargs))})(
request=request, response=response)
# Inject fake proxy, so the submission works
delegated = Credential(dlg_id='12345',
dn='/DN=1234',
proxy='',
termination_time=datetime.utcnow() +
timedelta(hours=5))
Session.merge(delegated)
Session.commit()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage_granularity',
model.PackageExtra.value == 'france',
):
package = package_extra.package
if package.private or package.state != 'active':
log.warning(
u'Territorial coverage granularity of package {} must be manually corrected'
.format(package.name))
continue
package_extra.value = 'pays'
model.repo.commit_and_remove()
return 0
def load_translations(lang):
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from pylons import request
registry = Registry()
registry.prepare()
class FakePylons:
translator = None
fakepylons = FakePylons()
class FakeRequest:
# Stores details of the translator
environ = {'pylons.pylons': fakepylons}
registry.register(request, FakeRequest())
# create translator
i18n.set_lang(lang)
# pull out translator and register it
registry.register(translator, fakepylons.translator)
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for organization_name, is_service_public in sorted(is_public_service_by_organization_name.iteritems()):
if not is_service_public:
continue
organization = model.Session.query(model.Group).filter(
model.Group.is_organization == True,
model.Group.name == organization_name,
).first()
if organization is None:
log.warning(u'Unknown organization: {}'.format(organization_name))
continue
if organization.certified_public_service is None:
log.info(u'Certifying "{}" as public service'.format(organization_name))
certified_public_service = etalab_model.CertifiedPublicService()
certified_public_service.organization_id = organization.id
model.Session.add(certified_public_service)
model.repo.commit_and_remove()
return 0
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for organization_name, is_service_public in sorted(is_public_service_by_organization_name.iteritems()):
if not is_service_public:
continue
organization = model.Session.query(model.Group).filter(
model.Group.is_organization == True,
model.Group.name == organization_name,
).first()
if organization is None:
log.warning(u'Unknown organization: {}'.format(organization_name))
continue
if organization.certified_public_service is None:
log.info(u'Certifying "{}" as public service'.format(organization_name))
certified_public_service = etalab_model.CertifiedPublicService()
certified_public_service.organization_id = organization.id
model.Session.add(certified_public_service)
model.repo.commit_and_remove()
return 0
def load_translations(lang):
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from pylons import request
registry = Registry()
registry.prepare()
class FakePylons:
translator = None
fakepylons = FakePylons()
class FakeRequest:
# Stores details of the translator
environ = {'pylons.pylons': fakepylons}
registry.register(request, FakeRequest())
# create translator
i18n.set_lang(lang)
# pull out translator and register it
registry.register(translator, fakepylons.translator)
def bootstrap(command, conf, vars):
"""Place any commands to setup bq here"""
# <websetup.bootstrap.before.auth
from sqlalchemy.exc import IntegrityError
from bq.data_service.model import Taggable, Tag, BQUser, ModuleExecution
registry = Registry()
registry.prepare()
registry.register(session, SessionObject({}))
registry.register(request, Request.blank('/bootstrap'))
request.identity = {}
log.info('BEGIN boostrap')
try:
initial_mex = ModuleExecution(mex_id = False, owner_id = False)
initial_mex.mex = initial_mex
initial_mex.name = "initialization"
initial_mex.type = "initialization"
initial_mex.hidden = True
model.DBSession.add(initial_mex)
model.DBSession.flush()
admin = model.User(
user_name = u"admin",
display_name = config.get('bisque.admin_display_name', u'Bisque admin'))
admin._email_address = config.get('bisque.admin_email', u'*****@*****.**')
admin.password = u'admin'
# password = u'admin')
#admin.password = u'admin'
model.DBSession.add(admin)
for g in [ u'admins', u'managers' ] :
group = model.Group()
group.group_name = g
group.display_name = u'Administrators Group'
group.users.append(admin)
model.DBSession.add(group)
permission = model.Permission()
permission.permission_name = u'root'
permission.description = u'This permission give an administrative right to the bearer'
permission.groups.append(group)
model.DBSession.add(permission)
#model.DBSession.flush()
# This commit will setup the BQUser also
transaction.commit()
except IntegrityError:
print 'Warning, there was a problem adding your auth data, it may have already been added:'
#import traceback
#print traceback.format_exc()
transaction.abort()
print 'Continuing with bootstrapping...'
try:
######
#
#from bq.data_service.model import UniqueName
initial_mex = model.DBSession.query(ModuleExecution).first()
session['mex_id'] = initial_mex.id
#request.identity['bisque.mex_id'] = initial_mex.id
admin = model.DBSession.query(BQUser).filter_by(resource_name = 'admin').first()
admin.mex_id = initial_mex.id
initial_mex.owner = admin
session['user'] = admin
system = model.DBSession.query(Taggable).filter_by (resource_type='system').first()
if system is None:
system_prefs = defaults_path('preferences.xml.default')
if os.path.exists(system_prefs):
with open (system_prefs) as f:
system = bisquik2db (f)
system.permission = 'published'
else:
print( "Couldn't find %s: using minimal default preferences" % system_prefs)
system = Taggable(resource_type = 'system')
version = Tag(parent = system)
version.name ='version'
version.value = __VERSION__
prefs = Tag(parent = system)
prefs.name = 'Preferences'
model.DBSession.add(system)
transaction.commit()
except IntegrityError:
log.exception ( 'Warning, there was a problem adding your system object, it may have already been added:')
#import traceback
#print traceback.format_exc()
transaction.abort()
log.warning ( 'Continuing with bootstrapping...' )
log.info('END boostrap')
def _get_or_create_user(self, env):
#WSGI Variables
#Shib-Application-ID 'default'
#Shib-Authentication-Instant '2012-08-13T12:04:22.492Z'
#Shib-Authentication-Method 'urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport'
#Shib-AuthnContext-Class 'urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport'
#Shib-Identity-Provider 'https://idp.example.com/idp/shibboleth'
#Shib-Session-ID '_7ec5a681e6dbae627c1cefcc7cb4d56a'
#Shib-Session-Index '39dafd8477850f5e0b968e3561570197f2109948c1d374a7a2b4c9a7adbf8628'
#cn 'My Other Self'
#givenName 'My Other Self'
#mail '*****@*****.**'
eppn = env.get(self.eppn, None)
fullname = env.get(self.fullname, None)
email = env.get(self.mail, None)
if not eppn or not fullname:
log.debug(
'Environ does not contain eppn or cn attributes, user not loaded.')
return None
user = model.Session.query(model.User).autoflush(False) \
.filter_by(name=eppn).first()
# Check if user information from shibboleth has changed
if user:
if (user.fullname != fullname or user.email != email):
log.debug('User attributes modified, updating.')
user.fullname = fullname
user.email = email
else: # user is None:
log.debug('User does not exists, creating new one.')
basename = eppn
username = basename
suffix = 0
while not model.User.check_name_available(username):
suffix += 1
username = basename + str(suffix)
user = model.User(name=username,
fullname=fullname,
email=email,
openid=eppn)
model.Session.add(user)
model.Session.flush()
log.info('Created new user {usr}'.format(usr=fullname))
groups = env.get(self.groups, None)
if groups:
groups = groups.split(";")
log.debug("groups: {}".format(sorted(groups)))
orgs = toolkit.get_action('group_list')(data_dict={
'all_fields': True,
'include_extras': True
})
log.debug("orgs: {}".format(orgs))
add_member = toolkit.get_action('group_member_create')
# Ensure there's a pylons.translator object
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
for o in orgs:
for e in o['extras']:
if e['key'] == 'ecgroup' and e['value'] in groups:
log.debug("Adding {} to {}".format(user.name, o['name']))
add_member(context={
'user': '******'
}, data_dict={
'id': o['name'],
'username': user.name,
'role': 'member'
})
model.Session.commit()
model.Session.remove()
return user
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
# Purge groups & organizations.
bad_groups_name = []
while True:
model.repo.new_revision()
group = model.Session.query(model.Group).filter(
model.Group.state == 'deleted',
sa.not_(model.Group.name.in_(bad_groups_name)) if bad_groups_name else None,
).first()
if group is None:
break
name = group.name
title = group.title
model.Session.query(etalab_model.CertifiedPublicService).filter(
etalab_model.CertifiedPublicService.organization_id == group.id,
).delete()
group.purge()
log.info(u'Purged group {} - {}'.format(name, title))
try:
model.repo.commit_and_remove()
except sqlalchemy.exc.IntegrityError:
log.exception(u'An integrity error while purging {} - {}'.format(name, title))
bad_groups_name.append(name)
# Purge packages.
bad_packages_name = []
while True:
model.repo.new_revision()
package = model.Session.query(model.Package).filter(
model.Package.state == 'deleted',
sa.not_(model.Package.name.in_(bad_packages_name)) if bad_packages_name else None,
).first()
if package is None:
break
name = package.name
title = package.title
# Delete resource_revision before purging package, to avoid IntegrityError: update or delete on table
# "resource_group" violates foreign key constraint "resource_revision_resource_group_id_fkey" on table
# "resource_revision".
for resource_group in model.Session.query(model.ResourceGroup).filter(
model.ResourceGroup.package_id == package.id,
):
for resource_revision in model.Session.query(model.ResourceRevision).filter(
model.ResourceRevision.resource_group_id == resource_group.id,
):
model.Session.delete(resource_revision)
# Delete package_relationship_revision before purging package, to avoid IntegrityError: update or
# delete on table "package" violates foreign key constraint
# "package_relationship_revision_subject_package_id_fkey" on table "package_relationship_revision".
for package_relationship_revision in model.Session.query(model.PackageRelationshipRevision).filter(
model.PackageRelationshipRevision.subject_package_id == package.id,
):
model.Session.delete(package_relationship_revision)
for package_relationship_revision in model.Session.query(model.PackageRelationshipRevision).filter(
model.PackageRelationshipRevision.object_package_id == package.id,
):
model.Session.delete(package_relationship_revision)
package.purge()
log.info(u'Purged package {} - {}'.format(name, title))
try:
model.repo.commit_and_remove()
except sqlalchemy.exc.IntegrityError:
log.exception(u'An integrity error while purging {} - {}'.format(name, title))
bad_packages_name.append(name)
# Purge resources.
bad_resources_id = []
while True:
model.repo.new_revision()
resource = model.Session.query(model.Resource).filter(
model.Resource.state == 'deleted',
sa.not_(model.Resource.id.in_(bad_resources_id)) if bad_resources_id else None,
).first()
if resource is None:
break
id = resource.id
name = resource.name
resource.purge()
log.info(u'Purged resource {} - {}'.format(id, name))
try:
model.repo.commit_and_remove()
except sqlalchemy.exc.IntegrityError:
log.exception(u'An integrity error while purging {} - {}'.format(id, name))
bad_resources_id.append(id)
# Delete unused tags.
for tag in model.Session.query(model.Tag):
package_tag = model.Session.query(model.PackageTag).filter(
model.PackageTag.tag_id == tag.id,
).first()
if package_tag is None:
model.Session.delete(tag)
log.info(u'Deleted unused tag {}'.format(tag.name))
model.Session.commit()
return 0
class PylonsTestCase(object):
"""A basic test case which allows access to pylons.c and pylons.request.
"""
def setup(self):
self.registry=Registry()
self.registry.prepare()
self.context_obj=ContextObj()
self.registry.register(pylons.c, self.context_obj)
pylons.c.errors = None
self.request_obj=Request(dict(HTTP_HOST="nohost"))
self.registry.register(pylons.request, self.request_obj)
self.translator_obj=MockTranslator()
self.registry.register(pylons.translator, self.translator_obj)
self.buffet = pylons.templating.Buffet('genshi', template_root='ckan.templates')
self.registry.register(pylons.buffet, self.buffet)
self.registry.register(pylons.response, Response())
self.registry.register(pylons.url, None)
def command(input_csv, config_ini, commit=False):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
from ckan import model
from ckan.logic import get_action
from ckan.lib.cli import MockTranslator
model.init_model(engine)
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
ctx = {
'model':
model,
'session':
model.Session,
'user':
get_action('get_site_user')({
'model': model,
'ignore_auth': True
}, {})['name']
}
if commit:
rev = model.repo.new_revision()
packages_to_check = set()
reader = csv.reader(open(input_csv, 'r'))
for row in reader:
# For each URL in the csv, get the list of resources referencing
# that URL
resources = model.Session.query(model.Resource)\
.filter(model.Resource.state=='active')\
.filter(model.Resource.url==row[0]).all()
for resource in resources:
# For each resource, add the package to the list
packages_to_check.add(resource.get_package_id())
# Delete the resource
resource.state = 'deleted'
model.Session.add(resource)
if commit:
model.Session.commit()
print "Deleted resource: {0}".format(resource.id)
stats.increment("Deleted resource")
for pid in packages_to_check:
# for each package we need to check, see if it has any
# resources left, it not, delete it.
pkg = model.Package.get(pid)
if len(pkg.resources) == 0:
pkg.state = 'deleted'
model.Session.add(pkg)
if commit:
model.Session.commit()
stats.increment('Deleted packages')
print "Deleted package: {0}".format(pkg.name)
if commit:
model.repo.commit_and_remove()
else:
print ""
print '*' * 60
print "DON'T PANIC, this was a dry run, nothing was committed"
print '*' * 60
print ''
print '*' * 60, 'Deletion Report'
print stats.report(order_by_title=True)
def mock_objects(config):
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
h.config = config
def command(dry_run=False):
from ckan import model
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
global_log.info('Tidying package fields')
stats = StatsList()
if not dry_run:
rev = model.repo.new_revision()
rev.message = 'Package fields migration'
for pkg in model.Session.query(model.Package)\
.filter_by(state='active')\
.order_by(model.Package.name):
# field map
for existing_fields, destination_field in field_map.items():
value = pkg.extras.get(destination_field)
if value:
continue
for existing_field in existing_fields:
if hasattr(pkg, existing_field):
value = getattr(pkg, existing_field)
else:
value = pkg.extras.get(existing_field)
if value:
value = value.strip()
if value:
# take the first hit
continue
if not dry_run:
pkg.extras[destination_field] = value or ''
# delete existing field values
for existing_field in existing_fields:
if hasattr(pkg, existing_field):
setattr(pkg, existing_field, '')
elif existing_field in pkg.extras:
del pkg.extras[existing_field]
if value:
stats.add('Merged to field "%s"' % destination_field, pkg.name)
else:
stats.add('Not merged to field "%s"' % destination_field, pkg.name)
# move url to additional resource
if pkg.url:
stats.add('Url moved to additional resource', value)
if not dry_run:
if not pkg.resource_groups:
res_group = model.ResourceGroup(label="default")
pkg.resource_groups.append(res_group)
res_group = pkg.resource_groups[0]
res = model.Resource(format='HTML', resource_type='documentation',
url=pkg.url, description='Web page about the data')
res_group.resources.append(res)
model.Session.add(res)
#pkg.url = ''
stats.add('URL moved to additional resource', pkg.name)
else:
stats.add('No URL to move to additional resource', pkg.name)
# delete fields
for field in delete_fields:
if field in pkg.extras:
if not dry_run:
del pkg.extras[field]
stats.add('Deleted field "%s"' % field, pkg.name)
else:
stats.add('No field to delete "%s"' % field, pkg.name)
if not dry_run:
model.repo.commit_and_remove()
global_log.info(stats.report())
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-t',
'--territoria-url',
help='Territoria URL',
required=True)
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
kind_code_name_by_kind_code = {}
for package_extra in model.Session.query(model.PackageExtra).filter(
model.PackageExtra.key == 'territorial_coverage', ):
if package_extra.value == 'Coutry/FR':
kind_code_name = 'Country/FR/FRANCE'
elif package_extra.value == 'InternationalOrganization/EU':
kind_code_name = 'InternationalOrganization/UE/UNION EUROPEENNE'
elif package_extra.value.count('/') == 1:
kind_code_name = kind_code_name_by_kind_code.get(
package_extra.value)
if kind_code_name is None:
kind, code = package_extra.value.split('/')
try:
response = urllib2.urlopen(
urlparse.urljoin(
args.territoria_url, '/api/v1/territory?{}'.format(
urllib.urlencode(dict(
code=code,
kind=kind,
),
doseq=True))))
except urllib2.HTTPError, response:
print package_extra.value
raise
response_dict = json.loads(response.read())
main_postal_distribution = response_dict['data'][
'main_postal_distribution']
kind_code_name_by_kind_code[
package_extra.value] = kind_code_name = u'/'.join(
[kind, code, main_postal_distribution])
print kind_code_name
else:
continue
package = package_extra.package
if package.private or package.state != 'active':
log.warning(
u'Territorial coverage of package {} must be manually corrected'
.format(package.name))
continue
package_extra.value = kind_code_name
def command(input_csv, config_ini, commit=False):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
from ckan import model
from ckan.logic import get_action
from ckan.lib.cli import MockTranslator
model.init_model(engine)
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
ctx = {
'model': model, 'session': model.Session,
'user': get_action('get_site_user')({'model': model,'ignore_auth': True}, {})['name']
}
if commit:
rev = model.repo.new_revision()
packages_to_check = set()
reader = csv.reader(open(input_csv, 'r'))
for row in reader:
# For each URL in the csv, get the list of resources referencing
# that URL
resources = model.Session.query(model.Resource)\
.filter(model.Resource.state=='active')\
.filter(model.Resource.url==row[0]).all()
for resource in resources:
# For each resource, add the package to the list
packages_to_check.add(resource.get_package_id())
# Delete the resource
resource.state = 'deleted'
model.Session.add(resource)
if commit:
model.Session.commit()
print "Deleted resource: {0}".format(resource.id)
stats.increment("Deleted resource")
for pid in packages_to_check:
# for each package we need to check, see if it has any
# resources left, it not, delete it.
pkg = model.Package.get(pid)
if len(pkg.resources) == 0:
pkg.state = 'deleted'
model.Session.add(pkg)
if commit:
model.Session.commit()
stats.increment('Deleted packages')
print "Deleted package: {0}".format(pkg.name)
if commit:
model.repo.commit_and_remove()
else:
print ""
print '*' * 60
print "DON'T PANIC, this was a dry run, nothing was committed"
print '*' * 60
print ''
print '*' * 60, 'Deletion Report'
print stats.report(order_by_title=True)
def mock_pylons():
from paste.registry import Registry
import pylons
from pylons.util import AttribSafeContextObj
import ckan.lib.app_globals as app_globals
from ckan.lib.cli import MockTranslator
from ckan.config.routing import make_map
from pylons.controllers.util import Request, Response
from routes.util import URLGenerator
class TestPylonsSession(dict):
last_accessed = None
def save(self):
pass
registry = Registry()
registry.prepare()
context_obj = AttribSafeContextObj()
registry.register(pylons.c, context_obj)
app_globals_obj = app_globals.app_globals
registry.register(pylons.g, app_globals_obj)
request_obj = Request(dict(HTTP_HOST="nohost", REQUEST_METHOD="GET"))
registry.register(pylons.request, request_obj)
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
registry.register(pylons.response, Response())
mapper = make_map()
registry.register(pylons.url, URLGenerator(mapper, {}))
registry.register(pylons.session, TestPylonsSession())
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
model.repo.new_revision()
for package in model.Session.query(model.Package).filter(
model.Package.owner_org != None,
model.Package.state != 'deleted',
):
organization = model.Session.query(model.Group).get(package.owner_org)
if organization is None and package.state != 'active':
log.warning(
u'Purging package "{}" whose organization is missing'.format(
package.name))
package.purge()
continue
assert organization is not None
assert organization.is_organization
assert organization.state != 'deleted', str((organization, package))
member = model.Session.query(model.Member).filter(
model.Member.group_id == organization.id,
model.Member.state == 'active',
model.Member.table_id == package.id,
).first()
if member is None:
log.warning(
u'Repairing organization "{}" package "{}" membership'.format(
organization.name, package.name))
member = model.Session.query(model.Member).filter(
model.Member.group_id == organization.id,
model.Member.table_id == package.id,
).first()
assert member is not None
if member.capacity != 'organization':
member.capacity = 'organization'
member.state = 'active'
assert member.table_name == 'package'
else:
if member.capacity != 'organization':
log.warning(
u'Repairing capacity organization "{}" package "{}" membership'
.format(organization, package))
member.capacity = 'organization'
assert member.table_name == 'package'
continue
for organization in model.Session.query(model.Group).filter(
model.Group.is_organization == True,
model.Group.state == 'active',
):
for member in model.Session.query(model.Member).filter(
model.Member.capacity != 'organization',
model.Member.group_id == organization.id,
model.Member.state == 'active',
model.Member.table_name == 'package',
):
package = model.Session.query(model.Package).get(member.table_id)
if package is None:
log.warning(
u"Purging member of organization {} with capacity {}, whose package doesn't exist"
.format(organization.name, member.capacity))
member.purge()
else:
log.warning(
u'Repairing capacity organization "{}" package "{}" membership'
.format(organization.name, package))
member.capacity = 'organization'
member_by_table_id_by_group_id = {}
for member in model.Session.query(model.Member).filter(
model.Member.state == 'active', ):
member_by_table_id = member_by_table_id_by_group_id.setdefault(
member.group_id, {})
if member.table_id in member_by_table_id:
log.warning(
u"Group {} contains several time the same object:\n {}\n {}".
format(member.group.name, member_by_table_id[member.table_id],
member))
member.purge()
continue
member_by_table_id[member.table_id] = member
model.repo.commit_and_remove()
return 0
dest='subcatalogs',
default=False,
help="Enable subcatalogs handling (dct:hasPart support)")
args = parser.parse_args()
contents = args.file.read()
config.update({DCAT_EXPOSE_SUBCATALOGS: args.subcatalogs})
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
if args.mode == 'produce':
serializer = RDFSerializer(profiles=args.profile,
compatibility_mode=args.compat_mode)
dataset = json.loads(contents)
out = serializer.serialize_dataset(dataset, _format=args.format)
print(out)
else:
parser = RDFParser(profiles=args.profile,
compatibility_mode=args.compat_mode)
parser.parse(contents, _format=args.format)
ckan_datasets = [d for d in parser.datasets()]
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('user', help = 'name of email of user')
parser.add_argument('-d', '--dry-run', action = 'store_true',
help = "simulate harvesting, don't update CKAN repository")
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
user_name = args.user.lower().replace('.', '-dot-').replace('@', '-at-')
user = model.Session.query(model.User).filter(model.User.name == user_name).one()
assert user is not None, 'Unknown user: {}'.format(user_name)
for membership_request in model.Session.query(youckan_model.MembershipRequest).filter(
youckan_model.MembershipRequest.user_id == user.id,
):
log.warning(u'Deleting membership request: {}'.format(membership_request))
if not args.dry_run:
model.Session.delete(membership_request)
model.Session.commit()
for related in model.Session.query(model.Related).filter(model.Related.owner_id == user.id):
log.warning(u'Deleting related: {}'.format(related))
if not args.dry_run:
model.Session.delete(related)
model.Session.commit()
if not args.dry_run:
model.repo.new_revision()
for user_object_role in model.Session.query(model.UserObjectRole).filter(
model.UserObjectRole.user_id == user.id,
model.UserObjectRole.role == 'admin',
):
if user_object_role.context == 'Group':
group = user_object_role.group
log.warning(u'Deleting group or organization: {}'.format(group))
if not args.dry_run:
model.Session.query(etalab_model.CertifiedPublicService).filter(
etalab_model.CertifiedPublicService.organization_id == group.id,
).delete()
model.Session.delete(group)
else:
assert user_object_role.context == 'Package', 'Unexpected context for role: {}'.format(
user_object_role.context)
package = user_object_role.package
# Delete resource_revision before purging package, to avoid IntegrityError: update or delete on table
# "resource_group" violates foreign key constraint "resource_revision_resource_group_id_fkey" on table
# "resource_revision".
for resource_group in model.Session.query(model.ResourceGroup).filter(
model.ResourceGroup.package_id == package.id,
):
for resource_revision in model.Session.query(model.ResourceRevision).filter(
model.ResourceRevision.resource_group_id == resource_group.id,
):
if not args.dry_run:
log.warning(u'Deleting resource_revision')
model.Session.delete(resource_revision)
# Delete package_relationship_revision before purging package, to avoid IntegrityError: update or
# delete on table "package" violates foreign key constraint
# "package_relationship_revision_subject_package_id_fkey" on table "package_relationship_revision".
for package_relationship_revision in model.Session.query(model.PackageRelationshipRevision).filter(
model.PackageRelationshipRevision.subject_package_id == package.id,
):
if not args.dry_run:
log.warning(u'Deleting package_relationship_revision')
model.Session.delete(package_relationship_revision)
for package_relationship_revision in model.Session.query(model.PackageRelationshipRevision).filter(
model.PackageRelationshipRevision.object_package_id == package.id,
):
if not args.dry_run:
log.warning(u'Deleting package_relationship_revision')
model.Session.delete(package_relationship_revision)
log.warning(u'Deleting package: {}'.format(package))
if not args.dry_run:
model.Session.delete(package)
if not args.dry_run:
model.repo.commit_and_remove()
if not args.dry_run:
log.warning(u'Deleting user: {}'.format(user))
model.Session.delete(user)
model.Session.commit()
return 0
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('user', help='name of email of user')
parser.add_argument(
'-d',
'--dry-run',
action='store_true',
help="simulate harvesting, don't update CKAN repository")
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
user_name = args.user.lower().replace('.', '-dot-').replace('@', '-at-')
user = model.Session.query(
model.User).filter(model.User.name == user_name).one()
assert user is not None, 'Unknown user: {}'.format(user_name)
for membership_request in model.Session.query(
youckan_model.MembershipRequest).filter(
youckan_model.MembershipRequest.user_id == user.id, ):
log.warning(
u'Deleting membership request: {}'.format(membership_request))
if not args.dry_run:
model.Session.delete(membership_request)
model.Session.commit()
for related in model.Session.query(
model.Related).filter(model.Related.owner_id == user.id):
log.warning(u'Deleting related: {}'.format(related))
if not args.dry_run:
model.Session.delete(related)
model.Session.commit()
if not args.dry_run:
model.repo.new_revision()
for user_object_role in model.Session.query(model.UserObjectRole).filter(
model.UserObjectRole.user_id == user.id,
model.UserObjectRole.role == 'admin',
):
if user_object_role.context == 'Group':
group = user_object_role.group
log.warning(u'Deleting group or organization: {}'.format(group))
if not args.dry_run:
model.Session.query(
etalab_model.CertifiedPublicService).filter(
etalab_model.CertifiedPublicService.organization_id ==
group.id, ).delete()
model.Session.delete(group)
else:
assert user_object_role.context == 'Package', 'Unexpected context for role: {}'.format(
user_object_role.context)
package = user_object_role.package
# Delete resource_revision before purging package, to avoid IntegrityError: update or delete on table
# "resource_group" violates foreign key constraint "resource_revision_resource_group_id_fkey" on table
# "resource_revision".
for resource_group in model.Session.query(
model.ResourceGroup).filter(
model.ResourceGroup.package_id == package.id, ):
for resource_revision in model.Session.query(
model.ResourceRevision).filter(
model.ResourceRevision.resource_group_id ==
resource_group.id, ):
if not args.dry_run:
log.warning(u'Deleting resource_revision')
model.Session.delete(resource_revision)
# Delete package_relationship_revision before purging package, to avoid IntegrityError: update or
# delete on table "package" violates foreign key constraint
# "package_relationship_revision_subject_package_id_fkey" on table "package_relationship_revision".
for package_relationship_revision in model.Session.query(
model.PackageRelationshipRevision).filter(
model.PackageRelationshipRevision.subject_package_id ==
package.id, ):
if not args.dry_run:
log.warning(u'Deleting package_relationship_revision')
model.Session.delete(package_relationship_revision)
for package_relationship_revision in model.Session.query(
model.PackageRelationshipRevision).filter(
model.PackageRelationshipRevision.object_package_id ==
package.id, ):
if not args.dry_run:
log.warning(u'Deleting package_relationship_revision')
model.Session.delete(package_relationship_revision)
log.warning(u'Deleting package: {}'.format(package))
if not args.dry_run:
model.Session.delete(package)
if not args.dry_run:
model.repo.commit_and_remove()
if not args.dry_run:
log.warning(u'Deleting user: {}'.format(user))
model.Session.delete(user)
model.Session.commit()
return 0
def migrate_v3_create_datasets(source_ids=None):
import pylons
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
sources = []
if not source_ids:
sources = model.Session.query(HarvestSource).all()
else:
sources = model.Session.query(HarvestSource) \
.filter(HarvestSource.id.in_(source_ids)) \
.all()
if not sources:
log.debug('No harvest sources to migrate')
return
site_user_name = logic.get_action('get_site_user')({'model': model, 'ignore_auth': True},{})['name']
context = {'model': model,
'session': model.Session,
'user': site_user_name, # TODO: auth of existing sources?
'return_id_only': True,
'extras_as_string': True,
}
def gen_new_name(title):
name = munge_title_to_name(title).replace('_', '-')
while '--' in name:
name = name.replace('--', '-')
pkg_obj = Session.query(Package).filter(Package.name == name).first()
if pkg_obj:
return name + str(uuid.uuid4())[:5]
else:
return name
for source in sources:
if 'id' in context:
del context['id']
if 'package' in context:
del context['package']
# Check if package already exists
try:
logic.get_action('package_show')(context, {'id': source.id})
continue
except logic.NotFound:
pass
package_dict = {
'id': source.id,
'name': gen_new_name(source.title) if source.title else source.id,
'title': source.title if source.title else source.url,
'notes': source.description,
'url': source.url,
'type': 'harvest',
'source_type': source.type,
'config': source.config,
'frequency': source.frequency,
}
context['message'] = 'Created package for harvest source {0}'.format(source.id)
try:
new_package_id = logic.get_action('package_create')(context, package_dict)
if new_package_id != source.id or not context['return_id_only']:
# this check only makes sense if we are sure we are returning
# the package id not the package object
raise PackageIdHarvestSourceIdMismatch
log.info('Created new package for source {0} ({1})'.format(source.id, source.url))
except logic.ValidationError,e:
log.error('Validation Error: %s' % str(e.error_summary))
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('config', help = 'path of configuration file')
parser.add_argument('-g', '--go', action = 'store_true', help = 'Change URLs of files')
parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
bad_resources_url = set()
while True:
model.repo.new_revision()
resources_found = False
resource_index = 0
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://www.data.gouv.fr/%'),
):
resource_url, error = conv.pipe(
conv.make_input_to_url(full = True),
conv.not_none,
)(resource.url, state = conv.default_state)
if error is not None:
continue
resource_url = resource_url.encode('utf-8')
if resource_url.startswith(('http://static.data.gouv.fr/', 'https://static.data.gouv.fr/')):
continue
if not resource_url.startswith(('http://www.data.gouv.fr/', 'https://www.data.gouv.fr/')):
continue
if resource_url in bad_resources_url:
continue
resource_url_path = urlparse.urlsplit(resource_url).path
print resource_url
try:
response = urllib2.urlopen(resource_url, timeout = 30)
except socket.timeout:
resources_found = True
continue
except urllib2.HTTPError:
bad_resources_url.add(resource_url)
continue
except urllib2.URLError:
bad_resources_url.add(resource_url)
continue
resources_found = True
resource_buffer = response.read()
resource_hash = hashlib.sha256(resource_buffer).hexdigest()
resource_url_path = '{}/{}{}'.format(resource_hash[:2], resource_hash[2:],
os.path.splitext(resource_url_path)[-1])
resource_path = '/tmp/resources/{}'.format(resource_url_path)
print ' ', resource_path
dir = os.path.dirname(resource_path)
if not os.path.exists(dir):
os.makedirs(dir)
with open(resource_path, 'w') as resource_file:
resource_file.write(resource_buffer)
if args.go:
resource.url = 'http://static.data.gouv.fr/{}'.format(resource_url_path)
resource_index += 1
if resource_index >= 5:
break
if resources_found:
model.repo.commit_and_remove()
else:
break
if not args.go:
print 'WARNING: URLs have not been modified. Transfer images then use the --go option.'
return 0
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument('-g',
'--go',
action='store_true',
help='Change URLs of files')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
args = parser.parse_args()
# logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
stream=sys.stdout)
site_conf = appconfig('config:{}'.format(os.path.abspath(args.config)))
load_environment(site_conf.global_conf, site_conf.local_conf)
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
plugins.load('synchronous_search')
bad_resources_url = set()
while True:
model.repo.new_revision()
resources_found = False
resource_index = 0
for resource in model.Session.query(model.Resource).filter(
model.Resource.url.like('http://www.data.gouv.fr/%'), ):
resource_url, error = conv.pipe(
conv.make_input_to_url(full=True),
conv.not_none,
)(resource.url, state=conv.default_state)
if error is not None:
continue
resource_url = resource_url.encode('utf-8')
if resource_url.startswith(('http://static.data.gouv.fr/',
'https://static.data.gouv.fr/')):
continue
if not resource_url.startswith(
('http://www.data.gouv.fr/', 'https://www.data.gouv.fr/')):
continue
if resource_url in bad_resources_url:
continue
resource_url_path = urlparse.urlsplit(resource_url).path
print resource_url
try:
response = urllib2.urlopen(resource_url, timeout=30)
except socket.timeout:
resources_found = True
continue
except urllib2.HTTPError:
bad_resources_url.add(resource_url)
continue
except urllib2.URLError:
bad_resources_url.add(resource_url)
continue
resources_found = True
resource_buffer = response.read()
resource_hash = hashlib.sha256(resource_buffer).hexdigest()
resource_url_path = '{}/{}{}'.format(
resource_hash[:2], resource_hash[2:],
os.path.splitext(resource_url_path)[-1])
resource_path = '/tmp/resources/{}'.format(resource_url_path)
print ' ', resource_path
dir = os.path.dirname(resource_path)
if not os.path.exists(dir):
os.makedirs(dir)
with open(resource_path, 'w') as resource_file:
resource_file.write(resource_buffer)
if args.go:
resource.url = 'http://static.data.gouv.fr/{}'.format(
resource_url_path)
resource_index += 1
if resource_index >= 5:
break
if resources_found:
model.repo.commit_and_remove()
else:
break
if not args.go:
print 'WARNING: URLs have not been modified. Transfer images then use the --go option.'
return 0
def migrate_v3_create_datasets(source_ids=None):
import pylons
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
sources = []
if not source_ids:
sources = model.Session.query(HarvestSource).all()
else:
sources = model.Session.query(HarvestSource).filter(HarvestSource.id.in_(source_ids)).all()
if not sources:
log.debug("No harvest sources to migrate")
return
site_user_name = logic.get_action("get_site_user")({"model": model, "ignore_auth": True}, {})["name"]
context = {
"model": model,
"session": model.Session,
"user": site_user_name, # TODO: auth of existing sources?
"return_id_only": True,
"extras_as_string": True,
}
def gen_new_name(title):
name = munge_title_to_name(title).replace("_", "-")
while "--" in name:
name = name.replace("--", "-")
pkg_obj = Session.query(Package).filter(Package.name == name).first()
if pkg_obj:
return name + str(uuid.uuid4())[:5]
else:
return name
for source in sources:
if "id" in context:
del context["id"]
if "package" in context:
del context["package"]
# Check if package already exists
try:
logic.get_action("package_show")(context, {"id": source.id})
continue
except logic.NotFound:
pass
package_dict = {
"id": source.id,
"name": gen_new_name(source.title) if source.title else source.id,
"title": source.title if source.title else source.url,
"notes": source.description,
"url": source.url,
"type": "harvest",
"source_type": source.type,
"config": source.config,
"frequency": source.frequency,
}
context["message"] = "Created package for harvest source {0}".format(source.id)
try:
new_package_id = logic.get_action("package_create")(context, package_dict)
if new_package_id != source.id or not context["return_id_only"]:
# this check only makes sense if we are sure we are returning
# the package id not the package object
raise PackageIdHarvestSourceIdMismatch
log.info("Created new package for source {0} ({1})".format(source.id, source.url))
except logic.ValidationError, e:
log.error("Validation Error: %s" % str(e.error_summary))
def migrate_v3_create_datasets(source_ids=None):
import pylons
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
sources = []
if not source_ids:
sources = model.Session.query(HarvestSource).all()
else:
sources = model.Session.query(HarvestSource) \
.filter(HarvestSource.id.in_(source_ids)) \
.all()
if not sources:
log.debug('No harvest sources to migrate')
return
site_user_name = logic.get_action('get_site_user')({
'model': model,
'ignore_auth': True
}, {})['name']
context = {
'model': model,
'session': model.Session,
'user': site_user_name, # TODO: auth of existing sources?
'return_id_only': True,
'extras_as_string': True,
}
def gen_new_name(title):
name = munge_title_to_name(title).replace('_', '-')
while '--' in name:
name = name.replace('--', '-')
pkg_obj = Session.query(Package).filter(Package.name == name).first()
if pkg_obj:
return name + str(uuid.uuid4())[:5]
else:
return name
for source in sources:
if 'id' in context:
del context['id']
if 'package' in context:
del context['package']
# Check if package already exists
try:
logic.get_action('package_show')(context, {'id': source.id})
continue
except logic.NotFound:
pass
package_dict = {
'id': source.id,
'name': gen_new_name(source.title) if source.title else source.id,
'title': source.title if source.title else source.url,
'notes': source.description,
'url': source.url,
'type': 'harvest',
'source_type': source.type,
'config': source.config,
'frequency': source.frequency,
}
context['message'] = 'Created package for harvest source {0}'.format(
source.id)
try:
new_package_id = logic.get_action('package_create')(context,
package_dict)
if new_package_id != source.id or not context['return_id_only']:
# this check only makes sense if we are sure we are returning
# the package id not the package object
raise PackageIdHarvestSourceIdMismatch
log.info('Created new package for source {0} ({1})'.format(
source.id, source.url))
except logic.ValidationError, e:
log.error('Validation Error: %s' % str(e.error_summary))
parser.add_argument('-s', '--subcatalogs', action='store_true', dest='subcatalogs',
default=False,
help="Enable subcatalogs handling (dct:hasPart support)")
args = parser.parse_args()
contents = args.file.read()
config.update({DCAT_EXPOSE_SUBCATALOGS: args.subcatalogs})
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
if args.mode == 'produce':
serializer = RDFSerializer(profiles=args.profile,
compatibility_mode=args.compat_mode)
dataset = json.loads(contents)
out = serializer.serialize_dataset(dataset, _format=args.format)
print(out)
else:
parser = RDFParser(profiles=args.profile,
compatibility_mode=args.compat_mode)
parser.parse(contents, _format=args.format)
ckan_datasets = [d for d in parser.datasets()]
