def setup_class(cls):
cls.registry = Registry()
cls.registry.prepare()
cls.context_obj = AttribSafeContextObj()
cls.registry.register(pylons.c, cls.context_obj)
cls.app_globals_obj = app_globals.Globals()
cls.registry.register(pylons.g, cls.app_globals_obj)
cls.request_obj = Request(dict(HTTP_HOST="nohost"))
cls.registry.register(pylons.request, cls.request_obj)
cls.translator_obj = MockTranslator()
cls.registry.register(pylons.translator, cls.translator_obj)
cls.buffet = pylons.templating.Buffet('genshi',
template_root='ckan.templates')
cls.registry.register(pylons.buffet, cls.buffet)
cls.registry.register(pylons.response, Response())
mapper = make_map()
cls.registry.register(pylons.url, URLGenerator(mapper, {}))
cls.registry.register(pylons.session, TestSession())
# Templates often want to find out the request's routes info, so put
# some dummy values into the routes_dict, so the templates that do
# this don't cause an exception.
pylons.request.environ.update({
'pylons.routes_dict': {
'action': 'test-action',
'controller': 'test-package::',
}
})
def mock_pylons():
from paste.registry import Registry
import pylons
from pylons.util import AttribSafeContextObj
import ckan.lib.app_globals as app_globals
from ckan.lib.cli import MockTranslator
from ckan.config.routing import make_map
from pylons.controllers.util import Request, Response
from routes.util import URLGenerator
class TestPylonsSession(dict):
last_accessed = None
def save(self):
pass
registry = Registry()
registry.prepare()
context_obj = AttribSafeContextObj()
registry.register(pylons.c, context_obj)
app_globals_obj = app_globals.app_globals
registry.register(pylons.g, app_globals_obj)
request_obj = Request(dict(HTTP_HOST="nohost", REQUEST_METHOD="GET"))
registry.register(pylons.request, request_obj)
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
registry.register(pylons.response, Response())
mapper = make_map()
registry.register(pylons.url, URLGenerator(mapper, {}))
registry.register(pylons.session, TestPylonsSession())
def register_translator():
# Workaround until core translation function defaults to Flask
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
from pylons import translator
registry = Registry()
registry.prepare()
registry.register(translator, MockTranslator())
def setup():
# Register a mock translator instead of having ckan domain translations defined
try:
from ckan.lib.cli import MockTranslator
patcher = mock.patch('pylons.i18n.translation._get_translator',
return_value=MockTranslator())
patcher.start()
except ImportError:
# if Pylons isn't present, we don't need it
pass
def register_translator():
# Register a translator in this thread so that
# the _() functions in logic layer can work
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
global registry
registry = Registry()
registry.prepare()
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def load_config(path):
import paste.deploy
conf = paste.deploy.appconfig('config:' + path)
import ckan, pylons
ckan.config.environment.load_environment(conf.global_conf, conf.local_conf)
from ckan.lib.cli import MockTranslator
from paste.registry import Registry
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(pylons.translator, translator_obj)
def command(dry_run=False):
from ckan import model
from ckanext.dgu.lib.resource_formats import match
from running_stats import StatsList
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
if not dry_run:
model.repo.new_revision()
# Add canonised formats to map
for format_ in res_type_map.keys():
res_type_map[canonise(format_)] = res_type_map[format_]
log.info('Tidying resource types')
stats = StatsList()
res_query = model.Session.query(model.Resource)
log.info('Tidying formats. Resources=%i Canonised formats=%i',
res_query.count(), len(set(res_type_map.values())))
for res in res_query:
canonised_fmt = canonise(res.format or '')
if canonised_fmt in res_type_map:
improved_fmt = res_type_map[canonised_fmt]
else:
improved_fmt = tidy(res.format)
match_ = match(improved_fmt)
if match_:
improved_fmt = match_
if (improved_fmt or '') != (res.format or ''):
if not dry_run:
res.format = improved_fmt
stats.add(improved_fmt, res.format)
else:
stats.add('No change', res.format)
if not dry_run:
model.repo.commit_and_remove()
log.info('Stats report: %r', stats.report())
print stats.report()
log.info('Warnings (%i): %r', len(warnings), warnings)
def __init__(self,
worker_id,
heartbeat_interval=60000,
action='update'):
super(UpdateIntentsWorker, self).__init__(worker_id, heartbeat_interval)
self.intents_worker = UserIntentsWorker(intents_extractor)
self.action = action
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
def register_translator():
# https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py
# If not set (in cli access), patch the a translator with a mock, so the
# _() functions in logic layer don't cause failure.
from paste.registry import Registry
from pylons import translator
from ckan.lib.cli import MockTranslator
if 'registery' not in globals():
global registry
registry = Registry()
registry.prepare()
if 'translator_obj' not in globals():
global translator_obj
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def _register_translator():
"""
Register a translator in this thread.
"""
global registry
try:
registry
except NameError:
registry = Registry()
registry.prepare()
global translator_obj
try:
translator_obj
except NameError:
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
def authenticate(self, environ, identity):
"""A username/password authenticator that throttles login request by IP."""
try:
login = identity['login']
except KeyError:
return None
environ['paste.registry'].register(pylons.translator, MockTranslator())
try:
remote_addr = Request(environ).headers['X-Forwarded-For']
except KeyError:
try:
remote_addr = environ['REMOTE_ADDR']
except KeyError:
log.critical(
'X-Forwarded-For header/REMOTE_ADDR missing from request.')
return None
throttle = LoginThrottle(User.by_name(login), remote_addr)
if not ('login' in identity and 'password' in identity):
return None
# Run through the CKAN auth sequence first, so we can hit the DB
# in every case and make timing attacks a little more difficult.
auth_user = super(CKANLoginThrottle,
self).authenticate(environ, identity)
# Check if there is a lock on the requested user, and return None if
# we have a lock.
if throttle.check_attempts() is False:
log.info('User %r (%s) locked out by brute force protection.' %
(login, remote_addr))
throttle.increment(
) # Increment so we only send an email the first time around
return None
# If the CKAN authenticator as successfully authenticated the request
# and the user wasn't locked out above, reset the throttle counter and
# return the user object.
if auth_user is not None:
throttle.reset()
return auth_user
# Increment the throttle counter if the login failed.
throttle.increment()
def command(config_ini, nodepublisher_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config,'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
global global_log
global_log = log
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
model.repo.new_revision()
log.info('Reading %s', nodepublisher_csv)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader( f)
for row in reader:
nid, title = row
publishers[ int(nid) ] = munge_title_to_name(title)
# Mappings where we are getting rid of duplicate publishers
publishers[16268] = publishers[11408] # UKSA -> ONS
publishers[11606] = publishers[11408] # ONS
publishers[20054] = publishers[16248] # Met Office
publishers[33036] = publishers[15255] # Windsor & Maidenhead
publishers[32619] = publishers[33245] # Monmouthshire
publishers[12662] = publishers[11567] # NHS
update_datasets()
generate_harvest_publishers()
log.info('Warnings: %r', warnings)
def authenticate(self, environ, identity):
"""A username/password authenticator that throttles login request by user name."""
try:
user_name = identity['login']
except KeyError:
return None
environ['paste.registry'].register(pylons.translator, MockTranslator())
if not ('login' in identity and 'password' in identity):
return None
# Run through the CKAN auth sequence first, so we can hit the DB
# in every case and make timing attacks a little more difficult.
auth_user_name = super(CKANLoginThrottle,
self).authenticate(environ, identity)
login_throttle_key = get_login_throttle_key(Request(environ),
user_name)
if login_throttle_key is None:
return None
throttle = LoginThrottle(User.by_name(user_name), login_throttle_key)
# Check if there is a lock on the requested user, and return None if
# we have a lock.
if throttle.is_locked():
return None
if auth_user_name is None:
# Increment the throttle counter if the login failed.
throttle.increment()
# if the CKAN authenticator has successfully authenticated the request and the user wasn't locked out above,
# then check the TOTP parameter to see if it is valid
if auth_user_name is not None:
totp_success = self.authenticate_totp(environ, auth_user_name)
if totp_success: # if TOTP was successful -- reset the log in throttle
throttle.reset()
return totp_success
def setup_class(cls):
cls.registry = Registry()
cls.registry.prepare()
cls.context_obj = AttribSafeContextObj()
cls.registry.register(pylons.c, cls.context_obj)
cls.app_globals_obj = app_globals.Globals()
cls.registry.register(pylons.g, cls.app_globals_obj)
cls.request_obj = Request(dict(HTTP_HOST="nohost"))
cls.registry.register(pylons.request, cls.request_obj)
cls.translator_obj = MockTranslator()
cls.registry.register(pylons.translator, cls.translator_obj)
cls.buffet = pylons.templating.Buffet('genshi',
template_root='ckan.templates')
cls.registry.register(pylons.buffet, cls.buffet)
cls.registry.register(pylons.response, Response())
mapper = make_map()
cls.registry.register(pylons.url, URLGenerator(mapper, {}))
cls.registry.register(pylons.session, TestSession())
dest='subcatalogs',
default=False,
help="Enable subcatalogs handling (dct:hasPart support)")
args = parser.parse_args()
contents = args.file.read()
config.update({DCAT_EXPOSE_SUBCATALOGS: args.subcatalogs})
# Workaround until the core translation function defaults to the Flask one
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
from pylons import translator
registry.register(translator, MockTranslator())
if args.mode == 'produce':
serializer = RDFSerializer(profiles=args.profile,
compatibility_mode=args.compat_mode)
dataset = json.loads(contents)
out = serializer.serialize_dataset(dataset, _format=args.format)
print(out)
else:
parser = RDFParser(profiles=args.profile,
compatibility_mode=args.compat_mode)
parser.parse(contents, _format=args.format)
ckan_datasets = [d for d in parser.datasets()]
def migrate_v3_create_datasets(source_ids=None):
import pylons
from paste.registry import Registry
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
sources = []
if not source_ids:
sources = model.Session.query(HarvestSource).all()
else:
sources = model.Session.query(HarvestSource) \
.filter(HarvestSource.id.in_(source_ids)) \
.all()
if not sources:
log.debug('No harvest sources to migrate')
return
site_user_name = logic.get_action('get_site_user')({
'model': model,
'ignore_auth': True
}, {})['name']
context = {
'model': model,
'session': model.Session,
'user': site_user_name, # TODO: auth of existing sources?
'return_id_only': True,
'extras_as_string': True,
}
def gen_new_name(title):
name = munge_title_to_name(title).replace('_', '-')
while '--' in name:
name = name.replace('--', '-')
pkg_obj = Session.query(Package).filter(Package.name == name).first()
if pkg_obj:
return name + str(uuid.uuid4())[:5]
else:
return name
for source in sources:
if 'id' in context:
del context['id']
if 'package' in context:
del context['package']
# Check if package already exists
try:
logic.get_action('package_show')(context, {'id': source.id})
continue
except logic.NotFound:
pass
package_dict = {
'id': source.id,
'name': gen_new_name(source.title) if source.title else source.id,
'title': source.title if source.title else source.url,
'notes': source.description,
'url': source.url,
'type': 'harvest',
'source_type': source.type,
'config': source.config,
'frequency': source.frequency,
}
context['message'] = 'Created package for harvest source {0}'.format(
source.id)
try:
new_package_id = logic.get_action('package_create')(context,
package_dict)
if new_package_id != source.id or not context['return_id_only']:
# this check only makes sense if we are sure we are returning
# the package id not the package object
raise PackageIdHarvestSourceIdMismatch
log.info('Created new package for source {0} ({1})'.format(
source.id, source.url))
except logic.ValidationError, e:
log.error('Validation Error: %s' % str(e.error_summary))
def mock_objects(config):
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
h.config = config
def command(config_ini, nodepublisher_csv, users_csv):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
from ckan import model
from ckan.lib.munge import munge_title_to_name
import ckanext.dgu.plugin
logging.config.fileConfig(config_ini_filepath)
global log
log = logging.getLogger(os.path.basename(__file__))
model.init_model(engine)
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
with open(nodepublisher_csv, 'rU') as f:
reader = csv.reader(f)
for row in reader:
publishers[int(row[0])] = munge_title_to_name(row[1])
log.info('Opened list of %i publishers', reader.line_num)
# get rid of flash message warnings
warnings_.filterwarnings('ignore', '.*flash message.*')
ckanext.dgu.plugin.log.disabled = True
with open(users_csv, 'rU') as f:
reader = csv.reader(f)
for row in reader:
model.repo.new_revision()
node_id, name, email, publisher_id = row
# create a new user
uname, user_id = _add_new_user(int(node_id), name, email)
if not user_id:
# validation error. warning already printed
continue
# Find the publisher and add them as editor
if node_id:
publisher_name = publishers[int(publisher_id)]
publisher = model.Group.by_name(publisher_name)
if not publisher:
warn(
'Could not find publisher %r so skipping making %r editor for it.',
publisher_name, name)
continue
capacity = 'editor'
# Check for Member where table_name is u['id']
res = model.Session.query(model.Member).\
from_statement(MEMBER_LOOKUP).\
params(userid=user_id, groupid=publisher.id).all()
if len(res) == 0:
m = model.Member(group_id=publisher.id,
table_id=user_id,
table_name='user',
capacity=capacity)
model.Session.add(m)
log.info('Made %r editor for %r', name, publisher_name)
else:
log.info('%r already editor for %r', name, publisher_name)
# Update harvest_source user_id field to new user id.
model.Session.execute(HARVEST_QUERY,
params={
'uid': user_id,
'node_id': str(node_id)
})
model.Session.commit()
log.info('Processed list of %i users', reader.line_num)
log.info('Warnings (%i): %r', len(warnings), warnings)
def command(dry_run=False):
from ckan import model
# Register a translator in this thread so that
# the _() functions in logic layer can work
from ckan.lib.cli import MockTranslator
registry=Registry()
registry.prepare()
translator_obj=MockTranslator()
registry.register(translator, translator_obj)
global_log.info('Tidying package fields')
stats = StatsList()
if not dry_run:
rev = model.repo.new_revision()
rev.message = 'Package fields migration'
for pkg in model.Session.query(model.Package)\
.filter_by(state='active')\
.order_by(model.Package.name):
# field map
for existing_fields, destination_field in field_map.items():
value = pkg.extras.get(destination_field)
if value:
continue
for existing_field in existing_fields:
if hasattr(pkg, existing_field):
value = getattr(pkg, existing_field)
else:
value = pkg.extras.get(existing_field)
if value:
value = value.strip()
if value:
# take the first hit
continue
if not dry_run:
pkg.extras[destination_field] = value or ''
# delete existing field values
for existing_field in existing_fields:
if hasattr(pkg, existing_field):
setattr(pkg, existing_field, '')
elif existing_field in pkg.extras:
del pkg.extras[existing_field]
if value:
stats.add('Merged to field "%s"' % destination_field, pkg.name)
else:
stats.add('Not merged to field "%s"' % destination_field, pkg.name)
# move url to additional resource
if pkg.url:
stats.add('Url moved to additional resource', value)
if not dry_run:
if not pkg.resource_groups:
res_group = model.ResourceGroup(label="default")
pkg.resource_groups.append(res_group)
res_group = pkg.resource_groups[0]
res = model.Resource(format='HTML', resource_type='documentation',
url=pkg.url, description='Web page about the data')
res_group.resources.append(res)
model.Session.add(res)
#pkg.url = ''
stats.add('URL moved to additional resource', pkg.name)
else:
stats.add('No URL to move to additional resource', pkg.name)
# delete fields
for field in delete_fields:
if field in pkg.extras:
if not dry_run:
del pkg.extras[field]
stats.add('Deleted field "%s"' % field, pkg.name)
else:
stats.add('No field to delete "%s"' % field, pkg.name)
if not dry_run:
model.repo.commit_and_remove()
global_log.info(stats.report())
def _get_or_create_user(self, env):
#WSGI Variables
#Shib-Application-ID 'default'
#Shib-Authentication-Instant '2012-08-13T12:04:22.492Z'
#Shib-Authentication-Method 'urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport'
#Shib-AuthnContext-Class 'urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport'
#Shib-Identity-Provider 'https://idp.example.com/idp/shibboleth'
#Shib-Session-ID '_7ec5a681e6dbae627c1cefcc7cb4d56a'
#Shib-Session-Index '39dafd8477850f5e0b968e3561570197f2109948c1d374a7a2b4c9a7adbf8628'
#cn 'My Other Self'
#givenName 'My Other Self'
#mail '*****@*****.**'
eppn = env.get(self.eppn, None)
fullname = env.get(self.fullname, None)
email = env.get(self.mail, None)
if not eppn or not fullname:
log.debug(
'Environ does not contain eppn or cn attributes, user not loaded.')
return None
user = model.Session.query(model.User).autoflush(False) \
.filter_by(name=eppn).first()
# Check if user information from shibboleth has changed
if user:
if (user.fullname != fullname or user.email != email):
log.debug('User attributes modified, updating.')
user.fullname = fullname
user.email = email
else: # user is None:
log.debug('User does not exists, creating new one.')
basename = eppn
username = basename
suffix = 0
while not model.User.check_name_available(username):
suffix += 1
username = basename + str(suffix)
user = model.User(name=username,
fullname=fullname,
email=email,
openid=eppn)
model.Session.add(user)
model.Session.flush()
log.info('Created new user {usr}'.format(usr=fullname))
groups = env.get(self.groups, None)
if groups:
groups = groups.split(";")
log.debug("groups: {}".format(sorted(groups)))
orgs = toolkit.get_action('group_list')(data_dict={
'all_fields': True,
'include_extras': True
})
log.debug("orgs: {}".format(orgs))
add_member = toolkit.get_action('group_member_create')
# Ensure there's a pylons.translator object
registry = Registry()
registry.prepare()
registry.register(pylons.translator, MockTranslator())
for o in orgs:
for e in o['extras']:
if e['key'] == 'ecgroup' and e['value'] in groups:
log.debug("Adding {} to {}".format(user.name, o['name']))
add_member(context={
'user': '******'
}, data_dict={
'id': o['name'],
'username': user.name,
'role': 'member'
})
model.Session.commit()
model.Session.remove()
return user
def setup():
# Register a mock translator instead of having ckan domain translations defined
patcher = mock.patch('pylons.i18n.translation._get_translator',
return_value=MockTranslator())
patcher.start()
def command(input_csv, config_ini, commit=False):
config_ini_filepath = os.path.abspath(config_ini)
load_config(config_ini_filepath)
engine = engine_from_config(config, 'sqlalchemy.')
logging.config.fileConfig(config_ini_filepath)
log = logging.getLogger(os.path.basename(__file__))
from ckan import model
from ckan.logic import get_action
from ckan.lib.cli import MockTranslator
model.init_model(engine)
registry = Registry()
registry.prepare()
translator_obj = MockTranslator()
registry.register(translator, translator_obj)
ctx = {
'model':
model,
'session':
model.Session,
'user':
get_action('get_site_user')({
'model': model,
'ignore_auth': True
}, {})['name']
}
if commit:
rev = model.repo.new_revision()
packages_to_check = set()
reader = csv.reader(open(input_csv, 'r'))
for row in reader:
# For each URL in the csv, get the list of resources referencing
# that URL
resources = model.Session.query(model.Resource)\
.filter(model.Resource.state=='active')\
.filter(model.Resource.url==row[0]).all()
for resource in resources:
# For each resource, add the package to the list
packages_to_check.add(resource.get_package_id())
# Delete the resource
resource.state = 'deleted'
model.Session.add(resource)
if commit:
model.Session.commit()
print "Deleted resource: {0}".format(resource.id)
stats.increment("Deleted resource")
for pid in packages_to_check:
# for each package we need to check, see if it has any
# resources left, it not, delete it.
pkg = model.Package.get(pid)
if len(pkg.resources) == 0:
pkg.state = 'deleted'
model.Session.add(pkg)
if commit:
model.Session.commit()
stats.increment('Deleted packages')
print "Deleted package: {0}".format(pkg.name)
if commit:
model.repo.commit_and_remove()
else:
print ""
print '*' * 60
print "DON'T PANIC, this was a dry run, nothing was committed"
print '*' * 60
print ''
print '*' * 60, 'Deletion Report'
print stats.report(order_by_title=True)